Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/core/src/context-compaction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ function noteRequest(kept: number): NoteMessage {
};
}

// Both compactor calls write the handoff note, so they share one journal name.
const JOURNAL_NAME = "Compact context";

// Each message the compactor reads is clipped, which keeps its own input
// bounded; a long payload's detail rarely survives a summary anyway.
const MAX_MESSAGE_CHARS = 16_000;
Expand Down Expand Up @@ -93,6 +96,7 @@ export function* summarizeContext(
yield* settings.model({
...settings.prefix,
messages: [...history.snapshot(), noteRequest(events.length - cut)],
name: JOURNAL_NAME,
}),
);
if (response.type === "final" && response.text.trim()) text = response.text;
Expand All @@ -112,6 +116,7 @@ export function* summarizeContext(
},
],
tools: [],
name: JOURNAL_NAME,
}),
);
if (response.type !== "final" || !response.text.trim())
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/conversation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ export function* summarizeConversation(
},
],
tools: [],
name: "Summarize conversation",
}),
);
if (response.type !== "final")
Expand Down
5 changes: 3 additions & 2 deletions packages/core/src/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@ import {run, type RetryOptions} from "@restatedev/restate-sdk-gen";
import type {Model, ModelRequest, ModelResponse} from "./types.js";
import {modelResponse} from "./validation.js";

/** Adapt asynchronous model I/O into one journaled, cancellable operation. */
/** Adapt asynchronous model I/O into one journaled, cancellable operation. The
* journal entry takes the request's name, then `options.name`, then "Ask model". */
export function journaledModel(
complete: (request: ModelRequest, options: {signal: AbortSignal}) => Promise<ModelResponse>,
options: {name?: string; retry?: RetryOptions} = {},
): Model {
return function* (request) {
return yield* run(async ({signal}) => modelResponse(await complete(request, {signal})), {
name: options.name ?? "Ask model",
name: request.name ?? options.name ?? "Ask model",
Comment thread
gvdongen marked this conversation as resolved.
Resolved
retry: options.retry ?? {maxAttempts: 3},
});
};
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ export function aiModel(options: ProviderOptions): Model {
}
return journaledModel(
(request, {signal}) => traceModel(options.tracer, request, () => complete(request, signal)),
{name: "Ask model", retry: options.retry},
{retry: options.retry},
);
}
/** Requests already contain JSON schemas, so no Zod objects cross this RPC. */
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/run-output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export function* summarizeStoppedRun<O>(
},
],
tools: [],
name: "Summarize stopped run",
};
let text: string | undefined;
try {
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ export type ModelRequest = {
outputSchema?: Record<string, unknown>;
maxOutputTokens?: number;
step?: number;
/** Journal entry name for this call, such as "Compact context". Steps leave
* it unset, and `journaledModel` falls back to its own name. */
name?: string;
};
export type ModelChunk = {type: "text-delta" | "reasoning-delta"; text: string};
/** Tokens a model call used, as the provider reported them. `inputTokens`
Expand Down
12 changes: 12 additions & 0 deletions packages/tests/src/replay-harness.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,18 @@ function frame(type, body) {
return cat(h, body);
}

/**
* The names of the run entries in a recorded journal, in journal order.
* @param {Uint8Array[]} journal
* @returns {string[]}
*/
export function runNames(journal) {
return journal
.map((f) => Buffer.from(f))
.filter((f) => f.readUInt16BE(0) === 0x411)
.map((f) => decode(f.subarray(8)).get(12)?.toString() ?? "");
}

/**
* @param {(ctx: import('@restatedev/restate-sdk').Context) => Promise<unknown>} handler
* @param {{replay?: Uint8Array[], onProposal?: (value: unknown) => void}} options
Expand Down
70 changes: 68 additions & 2 deletions packages/tests/src/replay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,16 @@ import assert from "node:assert/strict";
import {test} from "node:test";
import {execute, run} from "@restatedev/restate-sdk-gen";
import {z} from "zod";
import {agent, journaledModel, tool, type ModelResponse} from "@restatedev/agentlib";
import {runHandler} from "./replay-harness.mjs";
import {
agent,
journaledModel,
LazyStateConversation,
tool,
type ModelRequest,
type ModelResponse,
} from "@restatedev/agentlib";
import {memoryStore} from "./helpers.js";
import {runHandler, runNames} from "./replay-harness.mjs";

void test("model and tool effects are reused on journal replay", {timeout: 5000}, async () => {
const effects = {models: 0, tools: 0};
Expand Down Expand Up @@ -82,3 +90,61 @@ void test(
assert.equal(models, 3); // Two original calls; only the unfinished final call repeats.
},
);

void test("compaction and summary calls are journaled under their own names", async () => {
const lastText = (request: ModelRequest) => {
const last = request.messages.at(-1);
return last?.role === "user" ? last.content : "";
};
let reads = 0;
// One model serves every call, so each journal name comes from its request.
const model = journaledModel(async (request): Promise<ModelResponse> => {
if (request.instructions?.startsWith("Summarize the supplied conversation"))
return {type: "final", text: "earlier questions answered"};
if (lastText(request).includes('"type":"handoff-note-request"'))
return {type: "final", text: "read some pages"};
if (lastText(request).startsWith("The run stopped"))
return {type: "final", text: "Stopped after reading"};
return {type: "tool_calls", calls: [{id: `r${++reads}`, name: "read", input: {}}]};
});
const read = tool({
description: "Read a page",
input: z.object({}),
*execute() {
return "x".repeat(400);
},
});
const earlier = Array.from({length: 10}, (_, i) => [
{role: "user" as const, content: `question ${i}`},
{role: "assistant" as const, content: `answer ${i}`},
]).flat();
const handler = (ctx: Parameters<typeof execute>[0]) =>
execute(
ctx,
(function* () {
const {store} = memoryStore();
const conversation = new LazyStateConversation({
store,
compaction: {model, policy: {maxMessages: 8, retainMessages: 4}},
});
yield* conversation.append(earlier, 0);
const a = agent({
tools: {read},
codeMode: false,
maxSteps: 6,
compaction: {maxTokens: 650, retainTokens: 300},
model,
});
return yield* a.run("Go", {conversation});
})(),
);
const live = await runHandler(handler);
assert.equal(live.output.status, "stopped");
assert.equal(live.output.output, "Stopped after reading");
assert.deepEqual(
[...new Set(runNames(live.journal))],
["Summarize conversation", "Ask model", "Compact context", "Summarize stopped run"],
);
const replay = await runHandler(handler, {replay: live.journal});
assert.deepEqual(replay.output, live.output);
});