diff --git a/packages/examples/src/lessons/09_support_session.ts b/packages/examples/src/lessons/09_support_session.ts index 9f3efc1..6661a3d 100644 --- a/packages/examples/src/lessons/09_support_session.ts +++ b/packages/examples/src/lessons/09_support_session.ts @@ -125,6 +125,9 @@ export const aiSession = restate.object({ ), }, options: { + // Preload only the small keys that handlers read on every call. Conversation + // chunks are read when needed instead of sent with every invocation. + state: {preload: ["conversation/meta", "active", "approval"]}, // Shared handlers run while doTurn holds the object's lock. handlers: { status: {shared: true}, diff --git a/plugins/restate-agents/skills/restate-agents/SKILL.md b/plugins/restate-agents/skills/restate-agents/SKILL.md index a3d5902..d7b79c5 100644 --- a/plugins/restate-agents/skills/restate-agents/SKILL.md +++ b/plugins/restate-agents/skills/restate-agents/SKILL.md @@ -139,5 +139,12 @@ Restate journals. Move on only when the learner is ready. never from model-supplied tool arguments. - [ ] Approval requests and active-turn IDs stored in state are cleared in a `finally`. +- [ ] An agent with long or large conversations in a `LazyStateConversation` + preloads only small keys on its object, such as + `options.state: {preload: ["conversation/meta", "active"]}`. Otherwise + Restate sends the full history with every invocation of the object. + On serverless platforms, each lazy read adds a suspension, so keep the + default for short conversations. See + `references/conversations-and-controls.md`. - [ ] Tests use a fixture model (`journaledModel`) against `RestateContainer().alwaysReplay().disableRetries()`. No paid model calls. diff --git a/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md b/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md index 4172fe3..de02990 100644 --- a/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md +++ b/plugins/restate-agents/skills/restate-agents/references/conversations-and-controls.md @@ -45,6 +45,48 @@ const result = yield* support.run(message, {conversation}); Read it from a shared handler with `yield* new LazyStateConversation().page()`. +**Preload only small keys for long or large conversations.** By default, +Restate sends a snapshot of all of the object's state when each invocation +starts, including invocations of shared handlers such as `steer` and +`status`. Compaction changes what the model reads, not what is stored: every +chunk stays in state so `page()` can return the full history. Once a +conversation has many turns or large messages, such as long tool results, +every invocation of the object receives the entire history, although a run +loads only the chunks after the last summary. Short conversations do not +need this; keep the default for them. For agents with long or large +conversations, list the keys to preload: the conversation's metadata key and +your own small keys. Restate fetches every other key, including each chunk, +when it is read: + +```ts +export const aiSession = restate.object({ + name: "AiSession", + handlers: {doTurn, history, steer, interrupt}, + options: { + state: {preload: ["conversation/meta", "active"]}, // chunks are read on demand + handlers: {history: {shared: true}, steer: {shared: true}, interrupt: {shared: true}}, + }, +}); +``` + +- `conversation/meta` holds the message count and the latest summary. The + chunk keys (`conversation/0`, `conversation/1`, ...) follow from the + count, and every load, append and `page()` reads the metadata key first, + so always preload it. With a custom `stateKey`, the key is + `/meta`; with a custom `metaKey`, list that key. +- Each key that is not preloaded costs a round trip to Restate when it is + read. Loading the conversation reads one chunk key per 32 messages since + the last summary. +- On AWS Lambda and other serverless platforms without bidirectional + streaming, each lazy read adds a suspension: the invocation suspends and + replays once Restate sends the requested state. +- A list of keys requires Restate 1.8 or later: an older server rejects the + deployment at registration because it does not support + `eagerStateKeysWhitelist`. On older servers, use + `state: {preload: false}`; the metadata key is then read lazily as well. +- To set it for one handler only, put `state` in `options.handlers`. The + deprecated `enableLazyState: true` equals `state: {preload: false}`. + When a run starts and the conversation compaction fails, the run logs a warning and continues with the history it loaded; the next run tries again. An `AgentTerminalError` from the compaction model fails the run instead. diff --git a/plugins/restate-agents/skills/restate-gen-sdk/references/handlers-and-state.md b/plugins/restate-agents/skills/restate-gen-sdk/references/handlers-and-state.md index b32f88e..b3d105c 100644 --- a/plugins/restate-agents/skills/restate-gen-sdk/references/handlers-and-state.md +++ b/plugins/restate-agents/skills/restate-gen-sdk/references/handlers-and-state.md @@ -120,6 +120,12 @@ export const onboarding = restate.workflow({ `state: {preload}` chooses which state keys are loaded eagerly when the handler starts: `true` for all, `false` for none, or a list of key names. `enableLazyState: true` is deprecated and equals `state: {preload: false}`. +Set `state` at the object level in `options` to apply it to every handler. +Use `preload: false`, or a list of the small keys every invocation reads, +when an object stores much more state than one invocation reads, such as a +long agent conversation kept in chunks. On +serverless platforms without bidirectional streaming, each lazy read adds a +suspension, so keep the default for small state. Use `retryPolicy` with `onMaxAttempts: "kill"` or `"pause"` to decide what happens to an invocation after its retries are exhausted.