Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion CONTEXT.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ _Avoid_: Sub-agent, handoff

**Conversation**:
The accumulated model context across runs, including user messages, assistant
messages, tool exchanges, and runtime observations.
messages, tool exchanges, and runtime observations. A run saves to it while it
runs, after each model call and tool batch, so a running turn can be read; a
failed run saves what it committed, with a note on the failure.
_Avoid_: Public transcript, session

**Session**:
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ repeat a completed model call.
| ---------------------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------- |
| Tools with journaled results | `asyncTool`, `tool` | [02](packages/examples/src/lessons/02_tools.ts) |
| Conversation history | `run(input, {messages})`, `run(input, {conversation})` | [03](packages/examples/src/lessons/03_conversation.ts), 09 |
| Live and failed-turn history | `{conversation}` saves as the run goes; `messagesOf(error)` | [09](packages/examples/src/lessons/09_support_session.ts) |
| Typed output | `agent({output})` | [04](packages/examples/src/lessons/04_structured_output.ts) |
| Human approval | `needsApproval`, `onApproval`, `humanApprovalTool` | [06](packages/examples/src/lessons/06_human_approval.ts) |
| Steering and interruption | `run(input, {steeringSignal, interruptSignal})` or `{controls}` | [09](packages/examples/src/lessons/09_support_session.ts) |
Expand Down
77 changes: 68 additions & 9 deletions packages/core/src/agent-run.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import {CancelledError, TerminalError} from "@restatedev/restate-sdk";
import {
handlerRequest,
InterruptedError,
Expand All @@ -21,7 +22,8 @@ import {
ToolError,
toolErrorCode,
} from "./errors.js";
import {RunHistory, type BranchPoint} from "./run-history.js";
import {ConversationConflictError} from "./conversation.js";
import {recordFailedHistory, RunHistory, type BranchPoint} from "./run-history.js";
import {RunTasks, type FutureBranches} from "./run-tasks.js";
import {RunTools} from "./run-tools.js";
import {PARENT_USAGE, recordFailedRun, RunUsageTracker} from "./run-usage.js";
Expand Down Expand Up @@ -179,14 +181,19 @@ function resolveControls<C>(options?: RunOptions<C>): RunControls {
* - Background results completed during a model request also invalidate its
* response: the model must see them before acting.
*
* Nothing is saved until finish() persists the turn; failures save nothing. */
* With a conversation, the run saves as it goes, so a shared handler can read its
* progress: after each model call, each tool batch, and accepted steering or
* background results, then once more when finish() commits the answer. A failed
* run saves its committed history with a note on the failure; see fail(). */
class AgentRun<C, O> {
private step = 0;
private steeringSource: SteeringSource | undefined;
private steering: Future<string | SteeringUpdate> | undefined;
private interrupt: Future<string> | undefined;
/** Set once a summary failed; the run truncates from then on. */
private compactionFailed = false;
/** Set once `beforeRun` accepted the input; a fork's input needs no check. */
private inputChecked: boolean;
/** The last request without its messages: the prefix a provider cached.
* Its instructions may come from a function. */
private lastRequest: Pick<ModelRequest, "instructions" | "tools"> | undefined;
Expand All @@ -213,6 +220,7 @@ class AgentRun<C, O> {
private readonly origin?: ForkOrigin<C>,
) {
this.active = origin?.active ?? root;
this.inputChecked = origin !== undefined;
this.skills = origin?.skills;
this.catalog = createToolCatalog(this.active.tools, this.runTools());
for (const name of origin?.loaded ?? []) this.catalog.search.loaded.add(name);
Expand Down Expand Up @@ -252,6 +260,10 @@ class AgentRun<C, O> {
}
// Outside the catch: a failing stop propagates instead of stopping twice.
return yield* this.stop(stop);
} catch (error) {
yield* this.fail(error);
// The same value: Restate's retry and cancellation handling depend on its type.
throw error;
} finally {
yield* this.tasks.close();
}
Expand All @@ -263,6 +275,7 @@ class AgentRun<C, O> {
private *start(): Operation<void> {
if (this.origin) return;
const rejected = yield* this.checkInput();
this.inputChecked = true;
const source = this.root.skills;
const listed = source ? yield* listSkills(source) : [];
if (source && listed.length) {
Expand Down Expand Up @@ -313,11 +326,13 @@ class AgentRun<C, O> {
yield* this.emit({type: "step", step: this.step});
yield* this.compactContext();
const response = yield* this.ask();
if (!response || this.collectBackground()) continue;
if (!response || (yield* this.collectBackground())) continue;
if (response.type === "tool_calls") {
yield* this.toolPhase(response);
continue;
}
// The answer itself waits for review; this saves what came before it, such as the input.
yield* this.history.save();
const result = yield* this.complete(response);
if (result) return result;
}
Expand Down Expand Up @@ -608,17 +623,20 @@ class AgentRun<C, O> {
loaded: [...this.catalog.search.loaded],
});
const foreground = this.tools.startBatch(action, this.step, this.active, this.catalog);
// The calls are saved before their results, so a shared handler can show them.
yield* this.history.save();
while (Object.keys(foreground).length > 0) {
const {tag} = yield* this.race({...foreground, ...this.tools.pendingTasks()});
if (tag === "steering") {
for (const name of this.tools.handOffPrograms()) delete foreground[name];
continue;
}
delete foreground[tag];
this.collectBackground();
yield* this.collectBackground();
}
const transfer = this.tools.commitBatch();
this.collectBackground();
yield* this.collectBackground();
yield* this.history.save();
if (transfer) {
this.active = transfer();
yield* this.emit({type: "handoff", agent: this.active.name});
Expand All @@ -628,11 +646,13 @@ class AgentRun<C, O> {
}
}

/** Moves finished background results into history; true when any arrived. */
private collectBackground(): boolean {
/** Moves finished background results into history and saves them; true when any arrived. */
private *collectBackground(): Operation<boolean> {
const completed = this.tools.drainCompleted();
if (!completed.length) return false;
this.history.append(...completed);
return completed.length > 0;
yield* this.history.save();
return true;
}

private *complete(action: ModelFinalResponse): Operation<RunResult<O> | undefined> {
Expand All @@ -641,7 +661,7 @@ class AgentRun<C, O> {
// Its result, or steering, gives the model a reason to reconsider.
if (this.step < this.config.maxSteps) {
yield* this.race(this.tools.pendingTasks());
this.collectBackground();
yield* this.collectBackground();
}
return undefined;
}
Expand Down Expand Up @@ -751,6 +771,8 @@ class AgentRun<C, O> {
}
}
this.history.appendSteering(message);
// Saved before it is reported. Mid-batch, the note waits for the batch's results.
yield* this.history.save();
yield* this.emit({type: "steering", message, ...withData});
return true;
}
Expand Down Expand Up @@ -796,6 +818,43 @@ class AgentRun<C, O> {
return yield* this.finish(yield* this.review(result));
}

/** Records the history of a run that `error` ends, for `messagesOf`, and saves it
* when the error fails the invocation, so the next run knows what happened.
* Running work stops first: calls that never finished get cancelled results, and
* a note tells the model why the run ended. A retryable error saves nothing:
* Restate retries the invocation, and a journaled save would not match a retry
* that takes another path. A conflict means another writer changed the
* conversation, so saving again would fail the same way. A failure to save is
* logged; the caller still receives the run's own error.
*
* A run that fails before `beforeRun` accepted its input records nothing:
* throwing `AgentTerminalError` from `beforeRun` keeps the input out of the
* history. */
private *fail(error: unknown): Operation<void> {
if (!this.inputChecked) return;
try {
yield* this.tasks.close();
const reason = error instanceof Error ? error.message : String(error);
const cancelled = error instanceof CancelledError;
const closed = `${cancelled ? "Run cancelled" : "Run failed"}: ${reason}`;
this.tools.commitBatch(closed);
this.history.append(...this.tools.settlePending(new InterruptedError(closed)));
this.history.append({
role: "user",
content: JSON.stringify({
source: "agent-runtime",
type: cancelled ? "run-cancelled" : "run-failed",
message: reason,
}),
});
recordFailedHistory(error, this.history.committed());
if (error instanceof TerminalError && !(error instanceof ConversationConflictError))
yield* this.history.save();
} catch (failure) {
logger().warn("Could not record the failed run's history:", failure);
}
}

/** The schema the run's answer must match. A fork answers in plain text: its
* reply is a tool result for the parent, not the run's output. */
private get output(): AgentOptions<C, O>["output"] {
Expand Down
82 changes: 68 additions & 14 deletions packages/core/src/conversation.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
import {TerminalError} from "@restatedev/restate-sdk";
import {sharedState, state, type Operation} from "@restatedev/restate-sdk-gen";
import {contextLocal, sharedState, state, type Operation} from "@restatedev/restate-sdk-gen";
import {readable} from "./context-size.js";
import type {Message, Model} from "./types.js";
import {history, modelResponse} from "./validation.js";
import {checkProtocol, history, modelResponse, parseMessages, type OpenCall} from "./validation.js";

export type ConversationSnapshot = {messages: Message[]; version: number};
export type ConversationCompactionStatus = "skipped" | "applied" | "stale";
/** Where a run loads its history and saves its progress.
*
* A run saves while it runs, not only at its end. Each `append` continues the
* messages before it, so it can start with results for tool calls saved by an
* earlier append, and end with tool calls whose results a later append adds:
* the run saves a batch's calls as soon as the model makes them. Between those
* two appends, `load` returns a history that ends with unfinished calls. */
export interface Conversation {
load(): Operation<ConversationSnapshot>;
append(messages: readonly Message[], expectedVersion: number): Operation<number>;
Expand All @@ -23,7 +30,12 @@ export type CompactionPlan = {
messages: Message[];
};
export type CompactionResult = {plan: CompactionPlan; summary: string};
type Meta = {count: number; summary?: {through: number; text: string}};
type Meta = {
count: number;
summary?: {through: number; text: string};
/** Calls at the end of the stored messages that have no result yet. */
open?: OpenCall[];
};
/** Another writer changed the conversation since this run loaded it. */
export class ConversationConflictError extends TerminalError {
constructor(message: string) {
Expand All @@ -32,6 +44,8 @@ export class ConversationConflictError extends TerminalError {
}
}
const CHUNK_SIZE = 32;
/** The value of each key this invocation read or wrote, per store. */
const known = contextLocal<WeakMap<ConversationStore, Map<string, unknown>>>();
const restateStore: ConversationStore = {
*get<T>(key: string) {
return (yield* sharedState().get<T>(key)) ?? undefined;
Expand All @@ -42,7 +56,15 @@ const restateStore: ConversationStore = {
};

/** Append-only chunks in an application-owned exclusive Virtual Object handler.
* No constructor I/O and no process-local cache. Use one writer per conversation. */
* No constructor I/O. Use one writer per conversation.
*
* Within one invocation, each key is read from the store at most once. Restate
* journals every state read with its value, so a run that loads, compacts, and
* saves would otherwise journal the same chunks and metadata several times. The
* values are kept per invocation, not per instance, so every instance on the
* same store sees the writes of the others, and a replay reads them from the
* journal again. A handler that reads within one invocation does not see
* writes from other invocations after its first read of a key. */
export class LazyStateConversation implements Conversation {
private readonly store: ConversationStore;
private readonly metaKey: string;
Expand All @@ -61,8 +83,32 @@ export class LazyStateConversation implements Conversation {
this.metaKey = options.metaKey ?? `${this.stateKey}/meta`;
}

private known(): Map<string, unknown> {
let stores = known.get();
if (!stores) known.set((stores = new WeakMap()));
let values = stores.get(this.store);
if (!values) stores.set(this.store, (values = new Map()));
return values;
}

/** The key's value as this invocation last read or wrote it. Callers clone
* what they change: the value is shared for the rest of the invocation. */
private *read<T>(key: string): Operation<T | undefined> {
if (this.known().has(key)) return this.known().get(key) as T | undefined;
const value = yield* this.store.get<T>(key);
// A write during the read is newer than the value read.
const values = this.known();
if (!values.has(key)) values.set(key, value);
return values.get(key) as T | undefined;
}

private *write<T>(key: string, value: T): Operation<void> {
yield* this.store.set(key, value);
this.known().set(key, value);
}

private *meta(): Operation<Meta> {
return structuredClone((yield* this.store.get<Meta>(this.metaKey)) ?? {count: 0});
return structuredClone((yield* this.read<Meta>(this.metaKey)) ?? {count: 0});
}

private key(index: number) {
Expand All @@ -73,17 +119,20 @@ export class LazyStateConversation implements Conversation {
private *range(from: number, through: number): Operation<Message[]> {
const messages: Message[] = [];
for (let i = Math.floor(from / CHUNK_SIZE); i < Math.ceil(through / CHUNK_SIZE); i++) {
const chunk = yield* this.store.get<Message[]>(this.key(i));
const chunk = yield* this.read<Message[]>(this.key(i));
if (!chunk) throw new ConversationConflictError(`Missing conversation chunk: ${i}`);
const offset = i * CHUNK_SIZE;
messages.push(...chunk.slice(Math.max(0, from - offset), through - offset));
}
return structuredClone(messages);
}

/** The history a run continues. It ends with unfinished tool calls while a run's
* batch is running, or after that run was killed; the next run closes them. */
*load(): Operation<ConversationSnapshot> {
const meta = yield* this.meta();
const messages = history(yield* this.range(meta.summary?.through ?? 0, meta.count));
const range = yield* this.range(meta.summary?.through ?? 0, meta.count);
const messages = history(range, {unfinished: true});
if (meta.summary)
messages.unshift({
role: "user",
Expand All @@ -93,24 +142,28 @@ export class LazyStateConversation implements Conversation {
}

*append(messages: readonly Message[], expectedVersion: number): Operation<number> {
const additions = history(messages);
const additions = parseMessages(messages);
const meta = yield* this.meta();
if (meta.count !== expectedVersion)
throw new ConversationConflictError("Conversation changed since it was loaded");
if (!additions.length) return meta.count;
// The additions may first answer the calls the last append left open.
const open = checkProtocol(additions, {unanswered: meta.open ?? []});
if (open.length) meta.open = open;
else delete meta.open;
let offset = 0;
while (offset < additions.length) {
const index = Math.floor(meta.count / CHUNK_SIZE);
const used = meta.count % CHUNK_SIZE;
const chunk = used ? yield* this.store.get<Message[]>(this.key(index)) : [];
const chunk = used ? yield* this.read<Message[]>(this.key(index)) : [];
if (!chunk || chunk.length !== used)
throw new ConversationConflictError("Invalid conversation tail");
const next = additions.slice(offset, offset + CHUNK_SIZE - used);
yield* this.store.set(this.key(index), [...chunk, ...next]);
yield* this.write(this.key(index), [...chunk, ...next]);
offset += next.length;
meta.count += next.length;
}
yield* this.store.set(this.metaKey, meta);
yield* this.write(this.metaKey, meta);
return meta.count;
}

Expand All @@ -130,7 +183,8 @@ export class LazyStateConversation implements Conversation {
throw new TypeError("Compaction limits must be positive safe integers");
const meta = yield* this.meta();
const baseThrough = meta.summary?.through ?? 0;
const messages = history(yield* this.range(baseThrough, meta.count));
// A run may be in the middle of a batch: the boundary stays before its calls.
const messages = history(yield* this.range(baseThrough, meta.count), {unfinished: true});
// Measured as a summarizer reads them: encrypted reasoning would inflate the count.
const chars = JSON.stringify(messages.map(readable)).length;
if (messages.length <= maxMessages && chars <= maxChars) return undefined;
Expand Down Expand Up @@ -170,8 +224,8 @@ export class LazyStateConversation implements Conversation {
if (meta.summary?.through === plan.through && meta.summary.text === summary) return true;
if ((meta.summary?.through ?? 0) !== plan.baseThrough) return false;
// Verify the retained suffix independently before committing the new boundary.
history(yield* this.range(plan.through, meta.count));
yield* this.store.set(this.metaKey, {...meta, summary: {through: plan.through, text: summary}});
history(yield* this.range(plan.through, meta.count), {unfinished: true});
yield* this.write(this.metaKey, {...meta, summary: {through: plan.through, text: summary}});
return true;
}

Expand Down
1 change: 1 addition & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export {
isContextOverflow,
} from "./errors.js";
export {usageOf} from "./run-usage.js";
export {messagesOf} from "./run-history.js";
export type {
Agent,
AgentOptions,
Expand Down
Loading