diff --git a/.agent/domain-map.json b/.agent/domain-map.json index 9ba603e418..1da5026ded 100644 --- a/.agent/domain-map.json +++ b/.agent/domain-map.json @@ -44,9 +44,11 @@ "packages/ai-claude-code", "packages/ai-codex", "packages/ai-opencode", - "packages/ai-grok-build" + "packages/ai-grok-build", + "packages/ai-harness", + "packages/ai-harness-cli" ], - "talksTo": ["core", "sandbox"] + "talksTo": ["core", "sandbox", "persistence"] }, { "id": "client", diff --git a/.changeset/routed-agent-history.md b/.changeset/routed-agent-history.md new file mode 100644 index 0000000000..9d52032223 --- /dev/null +++ b/.changeset/routed-agent-history.md @@ -0,0 +1,15 @@ +--- +'@tanstack/ai-harness': minor +--- + +Routed agents now remember their own tool calls, you can pass your own handoff tool, and a `harnessAgent` can be a root agent. + +- `routing.history` sets what each model call gets of the routed runs of earlier turns. With `'own'` (the default), a routed agent gets its own earlier runs in full, with tool calls and results. It gets the other agents and the main model as labeled text in a user message, for example `main:\n...`. The main model gets the routed answers the same way, so no model reads the words of another speaker as its own. `'all'` gives every speaker every routed run in full. `'text'` gives routed runs as text only, as before. +- `routing.handoffTool` makes the handoff tool of each speaker. It gets `from`, `targets`, `allowHuman`, and `handoff`. The tool can have any name and its own fields. +- A `harnessAgent` in `agents` with `routing` keeps one child session across turns. Each turn sends it what happened in the thread since its last turn. Its model gets the `handoff` tool, so it can hand back. + +The default `history: 'own'` changes the messages that routed agents and handoff targets get. A target gets the turns of the main model as labeled text. To keep the earlier messages, and the prompt cache of the provider for a handoff target, set `routing: { router, history: 'text' }`. + +An app tool with the name of the handoff tool now replaces the harness tool. Before, the turn failed with a duplicate tool error. + +A handoff from the `handoff` tool of a root agent now adds the note messages to the thread, as a router or code handoff does. Before, the next speaker did not see that the agent handed off. On models that reject a context that ends on an assistant message (new Claude models), the main model failed after a hand-back. diff --git a/docs/config.json b/docs/config.json index 3cc74c896a..f2022b3e60 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1312,13 +1312,13 @@ "label": "Run agents from a harness", "to": "harness/subagents", "addedAt": "2026-10-13", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-12" }, { "label": "Hand off a conversation", "to": "harness/handoffs", "addedAt": "2026-10-18", - "updatedAt": "2026-10-20" + "updatedAt": "2026-11-12" }, { "label": "Add skills", diff --git a/docs/harness/handoffs.md b/docs/harness/handoffs.md index 0a4bd2b7d0..0bcaacb937 100644 --- a/docs/harness/handoffs.md +++ b/docs/harness/handoffs.md @@ -58,10 +58,63 @@ Send `I want a refund for order 42.` The main model calls its `handoff` tool wit What the handoff does: - The log keeps the active agent. Later turns go to billing until it hands off again, or until the router picks another agent. -- The main model and each root agent get a `handoff` tool. Its `to` lists the root agents. A root agent can also hand back to `'main'`, the main model. +- The main model and each root agent get a `handoff` tool. Its `to` lists the root agents. A root agent can also hand back to `'main'`, the main model. A whole harness in `agents` can hand back too. See [Make a harness a root agent](./subagents#make-a-harness-a-root-agent). - One turn can hand off 10 times at most. - An agent that is only in `subagents` cannot be a target. A child that an agent starts cannot hand off either. -- If the app has its own tool named `handoff`, the turn fails with a duplicate tool error. Set `routing: { router, giveModelsHandoffTool: false }` to give no model the `handoff` tool. The router and your code can still hand off. +- An app tool named `handoff` replaces the harness tool. See [Use your own handoff tool](#use-your-own-handoff-tool). To give no model the `handoff` tool, set `routing: { router, giveModelsHandoffTool: false }`. The router and your code can still hand off. + +### Use your own handoff tool + +The `handoff` tool takes only `to` and a short `note`. The target then often asks the user again for facts that the first agent had. Pass `routing.handoffTool` to set the description, the fields, and the note of the tool: + +```ts +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { defineHarness } from '@tanstack/ai-harness' +import { openaiText } from '@tanstack/ai-openai' +import { z } from 'zod' + +const billing = defineAgent({ + name: 'billing', + description: 'Handles refunds, invoices, and payments', + run: (ctx) => + ctx.chat({ + adapter: openaiText('gpt-6.1-sol'), + systemPrompts: ['You handle refunds, invoices, and payments.'], + }), +}) + +export const support = defineHarness({ + name: 'acme/support', + adapter: openaiText('gpt-6.1-sol'), + agents: [billing], + routing: { + router: () => undefined, + handoffTool: ({ from, targets, handoff }) => + toolDefinition({ + name: 'handoff', + description: 'Hand the user to another agent. Say what you did and what the user still needs.', + inputSchema: z.object({ + to: z.enum(targets), + done: z.string().describe('What you did for the user'), + next: z.string().describe('What the user still needs'), + orderId: z.string().optional(), + }), + }).server(async ({ to, done, next, orderId }) => { + const order = orderId ? ` Order: ${orderId}.` : '' + await handoff({ to, note: `From ${from ?? 'main'}. Done: ${done}. Next: ${next}.${order}` }) + return `Handed off to ${to}.` + }), + }, +}) +``` + +- The harness calls `handoffTool` once per turn for each speaker: the main model (`from: null`) and each root agent. +- `targets` holds the names that this speaker can hand off to. `allowHuman` is the value of `routing.allowHandoffToHuman`. +- `handoff({ to, note })` hands off. For the main model, the tool call is already in the transcript. The harness adds no note messages. +- A tool handoff from a root agent still adds the two note messages. +- The tool can have any name. With `giveModelsHandoffTool: false`, the harness does not call `handoffTool`. + +Give one speaker an app tool with the same name as the tool that `handoffTool` returns. It replaces the harness tool. Its code hands off with `ctx.context.run.handoff` (main model) or `ctx.harness.handoff` (root agent), as in [Hand off from code](#2-hand-off-from-code). ## 2. Hand off from code @@ -105,16 +158,22 @@ The router can hand off too. Return `{ handoff: 'billing', note }` from `routing ## What the target gets - Its own system prompts and tools. The tools of the agent that handed off do not go with it. -- The saved transcript of the thread, the same messages that the members see. +- The saved transcript of the thread. `routing.history` sets how it sees the turns of the other speakers. See [Choose what later turns see](./subagents#choose-what-later-turns-see). - The note, in the transcript, in the log, and in the `harness.handoff` event. - The provider keys of the user who sent the turn. -The `handoff` tool call of a model is in the transcript already. A handoff from the router or from code adds the same two messages before the target answers: +The `handoff` tool call of the main model is in the transcript already. A root agent keeps its own tool calls in its own transcript. So a handoff from a root agent, from the router, or from code adds the same two messages before the target answers: - An assistant message with a `handoff` tool call. Its arguments are `{ to, note }`. - The tool result, for example `Handed off to billing. billing answers next.` -Both messages have `metadata.harness.handoff.by` (`'router'` or `'code'`). When an agent handed off, they also have its name in `metadata.harness.agent`. The messages go after the messages that the earlier model call got, so the prompt cache of the provider still holds. +Both messages have `metadata.harness.handoff.by` (`'tool'`, `'router'`, or `'code'`). When an agent handed off, they also have its name in `metadata.harness.agent`. + +What the model of the target gets depends on `routing.history`: + +- `'own'` (default): the turns of the other speakers are labeled text in a user message, for example `main:\n...`. The handoff is one text line, for example `main handed off to billing. Note: Refund for order 42`. When billing hands back, the main model gets `billing handed off to main.` the same way. +- `'text'`: the messages of the main model stay as they are, and the two handoff messages come after them. So the prompt cache of the provider still holds. +- `'all'`: each routed host message becomes the child transcripts of that run, with their tool calls. With `routing.giveModelsHandoffTool: false`, the models have no `handoff` tool. Then the harness adds one assistant message with the same text and the note. @@ -323,7 +382,7 @@ A [fork](./fork-and-reset) stays in its harness. `host.fork` with a thread of an ## What you have now -- Agents that hand the conversation to each other, from a tool, from code, or from the router. +- Agents that hand the conversation to each other, from a tool, from code, or from the router. The tool can be your own, with the fields that you need. - A log that keeps who answers, also after a restart. - A UI that shows the name of the agent on each message. - A pause for a person, operator replies, and a way to give the thread back. diff --git a/docs/harness/subagents.md b/docs/harness/subagents.md index b32ff72e94..13ef18dbdb 100644 --- a/docs/harness/subagents.md +++ b/docs/harness/subagents.md @@ -669,7 +669,7 @@ export const studio = defineHarness({ The router can also return a pick of its own: `'main'`, a name, a list of names, `{ names, order }`, or `{ steps }`. Anywhere a name goes, `{ name, input }` can go too, for example `{ name: 'pricer', input: { vendor: 'Acme' } }`. If a picked agent has an `inputSchema` and no input, the turn fails with `Agent "pricer" needs input. Return { name: 'pricer', input } from the router.` See [Route, or let the model pick](../chat/subagents#route-or-let-the-model-pick) for each pick. -The router also gets `session`, the `input` of the turn as the user sent it, `operationId`, `inputId`, `abortSignal`, `activeAgent`, and `model`. Next to `router`, `routing` takes `order`, `limits`, `sandbox`, `afterAgentsAnswer`, `allowHandoffToHuman`, and `giveModelsHandoffTool`. The first three work the same as in `subagents`. `giveModelsHandoffTool: false` gives no model the `handoff` tool. See [Hand off a conversation](./handoffs). +The router also gets `session`, the `input` of the turn as the user sent it, `operationId`, `inputId`, `abortSignal`, `activeAgent`, and `model`. Next to `router`, `routing` takes `order`, `limits`, `sandbox`, `afterAgentsAnswer`, `history`, `allowHandoffToHuman`, `giveModelsHandoffTool`, and `handoffTool`. The first three work the same as in `subagents`. For `history`, see [Choose what later turns see](#choose-what-later-turns-see). For the last three, see [Hand off a conversation](./handoffs). ### Count the model calls of the router @@ -751,6 +751,40 @@ How each pick changes the active agent: Agents can also hand the turn to each other, or to a person. See [Hand off a conversation](./handoffs). +### Choose what later turns see + +On turn 1, billing calls its `refund` tool and answers `I refunded order 42.` On turn 3, billing answers again. If billing reads only that text, with no tool call behind it, it can doubt the text and refund again. `routing.history` sets what each model call gets of the routed runs of earlier turns: + +```ts group=harness-routing +const billing = defineAgent({ + name: 'billing', + description: 'Handles refunds', + run: (ctx) => ctx.chat({ adapter: openaiText('gpt-6.1-sol'), stream: false }), +}) + +export const support = defineHarness({ + name: 'acme/support', + adapter: openaiText('gpt-6.1-sol'), + agents: [billing], + routing: { + router: ({ input }) => (typeof input === 'string' && input.includes('refund') ? 'billing' : undefined), + history: 'own', + }, +}) +``` + +| Value | A routed agent gets | The main model gets | +| --- | --- | --- | +| `'own'` (default) | Its own earlier runs in full, with tool calls and results. Other agents as `name:\ntext`, and the main model as `main:\ntext`. A handoff is one text line. | Its own tool calls in full. Routed runs as `billing:\n...` text, and a hand-back as one text line. | +| `'all'` | Each routed host message becomes the child transcripts of that run. | Each routed host message becomes the child transcripts of that run. | +| `'text'` | Every routed run as `name:\ntext`. | Every routed run as `name:\ntext`. | + +- With `'own'`, the text of another speaker reaches the model as a user message. So no model reads the words of another speaker as its own. +- The router gets the same messages with each value. +- The stored transcript does not change. Only the messages of each model call change. +- With `'text'`, a handoff target keeps the prompt cache of the provider. See [What the target gets](./handoffs#what-the-target-gets). +- The harness keeps the run records of the last 100 runs that ended. A routed run that is older reaches the model as text. + ## Call a harness as a child `harnessAgent` turns a harness into an agent. Put it in `subagents.agents`, and the main model calls it as a tool. The child harness keeps its own tools, plugins, and history. @@ -787,6 +821,33 @@ What you see in the parent session: Outside a session, for example in a plain `chat()`, `harnessAgent` runs the child harness on the host that you pass as `host`. Without `host`, it makes a host with a memory log the first time it runs. +### Make a harness a root agent + +The billing team has its own harness. Your front desk wants billing to answer whole turns and keep the conversation, not only answer one tool call. Put `harnessAgent` in `agents`, and add `routing`: + +```ts group=harness-subagents +const billingTeam = defineHarness({ + name: 'acme/billing', + description: 'Handles refunds, invoices, and payments', + adapter: openaiText('gpt-6.1-sol'), +}) + +export const frontDesk = defineHarness({ + name: 'acme/front-desk', + adapter: openaiText('gpt-6.1-sol'), + agents: [harnessAgent(billingTeam, { name: 'billing' })], + routing: { router: () => undefined }, +}) +``` + +- The agent keeps one child session for all turns, on the thread `/agent/billing`. So turn 2 sees turn 1 and its own tool calls. +- Each turn sends the child what happened in the thread since its last turn, as labeled text. Then it sends the latest user message. +- The model of the child gets the `handoff` tool of the outer turn, so it can hand back to `'main'`. A tool of the child can call `ctx.context.run.handoff`. See [Hand off a conversation](./handoffs). +- The child harness cannot have its own `routing`. If it has one, the turn fails. +- `routing.history` does not apply to this child. It always gets the `'own'` fold of labeled text. + +In `subagents.agents`, the same `harnessAgent` stays a tool of the main model, with a new child session for each call. + ## Stay within limits A harness always limits its children. Without `subagents.limits`, it uses a depth of 2, 3 children at once, and 12 children per tree. Agents started from code count against the same limits: @@ -894,5 +955,5 @@ const reader = defineAgent({ - Plugins, commands, and agents that start typed agents, in parallel or in series, and message them while they run. - Background agents that continue on the next host after a crash, with steps and children that do not run twice. - Runs with their own budget and time limit, in a tree that stays within limits. -- A router that sends each turn to the right agent, and a routed agent that takes steers like the main model. -- Harnesses that call other harnesses as child runs, and agents that share state by key. +- A router that sends each turn to the right agent, and a routed agent that takes steers like the main model and keeps its own tool calls. +- Harnesses that call other harnesses as child runs or as root agents, and agents that share state by key. diff --git a/packages/ai-harness/src/define.ts b/packages/ai-harness/src/define.ts index 197e688463..fd33a0f5af 100644 --- a/packages/ai-harness/src/define.ts +++ b/packages/ai-harness/src/define.ts @@ -16,6 +16,7 @@ import type { SystemPrompt, } from '@tanstack/ai' import type { AnyAgent } from './agents' +import type { HandoffOptions } from './handoff' import type { ProjectOptions, ReduceOptions } from './log' import type { MediaOptions } from './media' import type { HarnessPlugin, PluginMetadataCheck, PluginModel } from './plugins' @@ -126,6 +127,27 @@ export type HarnessRouterPick = | { handoff: string; note?: string } | undefined +/** + * What `routing.handoffTool` gets for one speaker of the turn. + */ +export interface HandoffToolContext { + /** The speaker: `null` for the main model, else the root agent name. */ + from: string | null + /** + * The names this speaker can hand off to: the root agents other than + * itself, plus `'main'` for an agent. + */ + targets: Array + /** The value of `routing.allowHandoffToHuman`. */ + allowHuman: boolean + /** + * Record a handoff with `by: 'tool'` and end the part. For the main + * model, the tool call is the record, so this adds no note messages. + * A root-agent tool handoff still adds the two note messages. + */ + handoff: (options: HandoffOptions) => Promise +} + /** * The `routing` option. The other fields mean the same as in `subagents`. * `limits` defaults to `subagents.limits`. The `strategy` of `subagents` is @@ -166,6 +188,41 @@ export interface HarnessRouting extends Pick< * hand off. Default `true`. */ giveModelsHandoffTool?: boolean + /** + * How later turns see the routed runs of earlier turns. The router gets + * the same messages in every mode. + * - `'own'`: each speaker sees only its own tool calls. A routed agent + * sees its own earlier routed runs in full. It sees the other speakers + * as labeled text (`name:\ntext`, `main:\ntext`), and a handoff call as + * a text line with the note. The main model sees routed runs as + * `name:\ntext`. + * - `'all'`: each routed host message becomes the child transcripts of + * that run. A root harnessAgent always gets the `'own'` fold. + * - `'text'`: every speaker sees routed runs as `name:\ntext`. + * + * Default `'own'`. + */ + history?: 'own' | 'all' | 'text' + /** + * Make the handoff tool of each speaker. The harness calls it once per + * speaker (the main model and each root agent) per turn. The returned + * tool replaces the default `handoff` tool, and it can have any name. + * Not called when `giveModelsHandoffTool` is false. + * + * @example + * ```ts + * handoffTool: ({ from, targets, handoff }) => + * toolDefinition({ + * name: 'handoff', + * description: 'Hand the user to another agent', + * inputSchema: z.object({ to: z.enum(targets), note: z.string() }), + * }).server(async ({ to, note }) => { + * await handoff({ to, note: `${from ?? 'main'}: ${note}` }) + * return `Handed off to ${to}.` + * }) + * ``` + */ + handoffTool?: (ctx: HandoffToolContext) => AnyTool } /** diff --git a/packages/ai-harness/src/handoff.ts b/packages/ai-harness/src/handoff.ts index f7d795bac3..f0139eb503 100644 --- a/packages/ai-harness/src/handoff.ts +++ b/packages/ai-harness/src/handoff.ts @@ -1,14 +1,20 @@ import { toolDefinition } from '@tanstack/ai' import { isHumanTarget } from './log-records' +import { isHandoffNote, routedRunId } from './routed-history' import { isRecord } from './utils' import type { + AfterToolCallInfo, AnyChatMiddleware, + AnyTool, + ChatMiddlewareConfig, + ChatMiddlewareContext, ModelMessage, SubagentRouterPick, + ToolCallHookContext, } from '@tanstack/ai' import type { MessageStore } from '@tanstack/ai-persistence' import type { AnyAgent } from './agents' -import type { HarnessRouterPick } from './define' +import type { HandoffToolContext, HarnessRouterPick } from './define' import type { HandoffRecord, HumanTarget } from './log-records' /** What `ctx.harness.handoff` takes. */ @@ -53,6 +59,53 @@ function handoffText(to: string | HumanTarget) { return `Handed off to ${to}. ${to} answers next.` } +/** The middleware context fields that tell which chat runs. */ +type ChatOf = Pick< + ChatMiddlewareContext, + 'subagentName' | 'parentSubagentRunId' +> + +/** + * The built-in `handoff` tool. Its `to` lists `targets`, and + * `{ human: true }` when `allowHuman` is set. + */ +function defaultHandoffTool({ + targets, + allowHuman, + handoff, +}: HandoffToolContext) { + const agents = { type: 'string', enum: targets } + const person = { + type: 'object', + properties: { human: { type: 'boolean', const: true } }, + required: ['human'], + } + return toolDefinition({ + name: HANDOFF_TOOL, + description: `Hand the conversation to another agent. That agent answers the user next, in this turn and in later turns.${allowHuman ? ' Use { human: true } only when the user needs a person.' : ''} Stop after this call.`, + inputSchema: { + type: 'object', + properties: { + to: allowHuman ? { anyOf: [agents, person] } : agents, + note: { + type: 'string', + description: 'What the next agent must know.', + }, + }, + required: ['to'], + }, + }).server(async (args: unknown) => { + const to = isRecord(args) ? args.to : undefined + const note = isRecord(args) ? args.note : undefined + const target = isHumanTarget(to) ? HUMAN : to + if (typeof target !== 'string' && !isHumanTarget(target)) { + throw new Error('A handoff needs `to`: the name of an agent.') + } + await handoff({ to: target, ...(typeof note === 'string' ? { note } : {}) }) + return handoffText(target) + }) +} + /** * The messages that show a router or code handoff in the transcript, so the * target knows that it got the conversation, and why. They look like a @@ -116,6 +169,10 @@ export class TurnHandoffs { /** The handoff that ends the current part. */ private pending: Handoff | undefined private count = 0 + /** The handoff tool of each speaker, made once per turn. */ + private readonly tools = new Map() + /** The ids of the handoff tool calls that run now. */ + private readonly activeCalls = new Set() constructor( private readonly options: { @@ -134,6 +191,8 @@ export class TurnHandoffs { humans: boolean /** `routing.giveModelsHandoffTool`: the root agents get the `handoff` tool. */ tool: boolean + /** `routing.handoffTool`: makes the handoff tool of each speaker. */ + handoffTool?: (ctx: HandoffToolContext) => AnyTool /** * Append the record. A handoff to a person opens its `human` wait in * the same append. Resolves with the record as the log keeps it. @@ -163,11 +222,13 @@ export class TurnHandoffs { } /** - * The messages that show a router or code handoff in the transcript. None - * for a `handoff` tool call: the call of the model is there already. + * The messages that show a handoff in the transcript. None for a tool + * call of the main model: its call is in the thread already. A root agent + * keeps its tool calls in its own transcript, so its handoff gets them, + * and the next speaker reads who handed off. */ messages(handoff: Handoff): Array { - if (handoff.by !== 'router' && handoff.by !== 'code') return [] + if (handoff.by === 'tool' && handoff.from === null) return [] // A turn that runs again after a crash makes the same ids, so the store // keeps each note once. const id = `handoff_${this.options.noteKey}_${this.count}` @@ -180,12 +241,18 @@ export class TurnHandoffs { * A handoff to a person ends the turn, and the session pauses. Throws for * a target that is not a root agent, for a person without * `routing.allowHandoffToHuman`, and for a second handoff in the same part. + * A code handoff inside a call of the handoff tool counts as a tool + * handoff: the tool call is in the transcript. */ async handoff( options: HandoffOptions, from: string | null, by: 'tool' | 'code', ): Promise { + // ponytail: any running handoff tool call of the turn counts. A parallel + // tool that calls `run.handoff` while a handoff call runs is also 'tool'. + // Keep the speaker of each call if that matters. + const recordedBy = this.activeCalls.size > 0 ? 'tool' : by const { to, note } = options const isHuman = isHumanTarget(to) if (isHuman && !this.options.humans) { @@ -227,7 +294,7 @@ export class TurnHandoffs { from, to: target, ...(typeof note === 'string' ? { note } : {}), - by, + by: recordedBy, // The agents an operator can hand back to: the ones this turn offers. ...(isHuman ? { agents: names } : {}), } @@ -252,82 +319,81 @@ export class TurnHandoffs { } /** - * The `handoff` tool of `from`: the main model (`null`) or a root agent. - * Its `to` lists the root agents, and `'main'` for an agent, but not - * `from`. With `routing.allowHandoffToHuman`, `{ human: true }` hands the conversation - * to a person. + * The handoff tool of `from`: the main model (`null`) or a root agent. + * `routing.handoffTool` makes it, else the built-in `handoff` tool. Its + * targets are the root agents, and `'main'` for an agent, but not `from`. + * With `routing.allowHandoffToHuman`, `{ human: true }` hands the + * conversation to a person. Each speaker gets one tool per turn. */ tool(from: string | null) { - const { humans } = this.options + const made = this.tools.get(from) + if (made) return made const targets = [ ...this.options.rootAgents.map((agent) => agent.name), ...(from === null ? [] : ['main']), ].filter((name) => name !== from) - const agents = { type: 'string', enum: targets } - const person = { - type: 'object', - properties: { human: { type: 'boolean', const: true } }, - required: ['human'], + const ctx: HandoffToolContext = { + from, + targets, + allowHuman: this.options.humans, + handoff: (options) => this.handoff(options, from, 'tool'), } - return toolDefinition({ - name: HANDOFF_TOOL, - description: `Hand the conversation to another agent. That agent answers the user next, in this turn and in later turns.${humans ? ' Use { human: true } only when the user needs a person.' : ''} Stop after this call.`, - inputSchema: { - type: 'object', - properties: { - to: humans ? { anyOf: [agents, person] } : agents, - note: { - type: 'string', - description: 'What the next agent must know.', - }, - }, - required: ['to'], - }, - }).server(async (args: unknown) => { - const to = isRecord(args) ? args.to : undefined - const note = isRecord(args) ? args.note : undefined - const target = isHumanTarget(to) ? HUMAN : to - if (typeof target !== 'string' && !isHumanTarget(target)) { - throw new Error('A handoff needs `to`: the name of an agent.') - } - await this.handoff( - { to: target, ...(typeof note === 'string' ? { note } : {}) }, - from, - 'tool', - ) - return handoffText(target) - }) + const tool = this.options.handoffTool?.(ctx) ?? defaultHandoffTool(ctx) + this.tools.set(from, tool) + return tool } /** * Middleware for the chat calls of a turn: a pending handoff stops the * tool loop of the chat that made it, so the target answers next. Other * chats of the part (a sibling root agent, a tool-mode child) go on. With - * `roots`, it also gives each routed root agent its `handoff` tool, unless - * `routing.giveModelsHandoffTool` is false. A nested child of a root agent gets no - * tool, so it cannot move the whole session. + * `roots`, it also gives each routed root agent its handoff tool, unless + * `routing.giveModelsHandoffTool` is false. A tool in `config.tools` with + * the same name wins. A nested child of a root agent gets no tool, so it + * cannot move the whole session. It also tracks the calls of the handoff + * tool, so a code handoff inside one records `by: 'tool'`. */ - middleware(options: { roots?: boolean } = {}): AnyChatMiddleware { + middleware(options: { roots?: boolean } = {}) { + /** The name of the handoff tool of a chat. Undefined when it has none. */ + const toolName = (ctx: ChatOf) => + this.options.tool && ctx.parentSubagentRunId === undefined + ? this.tool(ctx.subagentName ?? null).name + : undefined return { name: 'harness:handoff', - onConfig: (ctx, config) => { - const isRoot = - options.roots === true && - this.options.tool && - ctx.subagentName !== undefined && - ctx.parentSubagentRunId === undefined - if (!isRoot) return - if (config.tools.some((tool) => tool.name === HANDOFF_TOOL)) return + onConfig: (ctx: ChatOf, config: Pick) => { + const isRoot = options.roots === true && ctx.subagentName !== undefined + const name = isRoot ? toolName(ctx) : undefined + if (name === undefined) return undefined + if (config.tools.some((tool) => tool.name === name)) return undefined return { tools: [...config.tools, this.tool(ctx.subagentName ?? null)] } }, - onShouldContinue: (ctx) => { + // The guard tier, not onBeforeToolCall: the first onBeforeToolCall + // decision (such as changed args) stops the hooks after it. + onGuardToolCall: ( + ctx: ChatOf, + call: Pick, + ) => { + if (call.toolName === toolName(ctx)) { + this.activeCalls.add(call.toolCallId) + } + return undefined + }, + onAfterToolCall: ( + _ctx: ChatOf, + info: Pick, + ) => { + this.activeCalls.delete(info.toolCallId) + return undefined + }, + onShouldContinue: (ctx: ChatOf) => { const { pending } = this if (!pending || ctx.parentSubagentRunId !== undefined) return undefined // ponytail: a chat is known by its agent name ('main' has none). The // root agents of one pick have different names. return (ctx.subagentName ?? null) === pending.from ? false : undefined }, - } + } satisfies AnyChatMiddleware } private async write(handoff: Handoff) { @@ -397,25 +463,64 @@ export function answererOf(pick: SubagentRouterPick | undefined) { * root agents of run `runId` in one assistant message. */ function routedIndex(messages: ReadonlyArray, runId: string) { - return messages.findIndex((message) => { - const tanstack = message.metadata?.tanstack - return isRecord(tanstack) && tanstack.runId === runId - }) + return messages.findIndex((message) => routedRunId(message) === runId) +} + +/** + * Put the recorded note of a main-model tool handoff on that tool call. + * A custom `handoffTool` can pass a different note than the model put in + * the call. `'own'` then folds that note for the target. The stored call + * arguments stay as the model wrote them. No extra note messages. + */ +export async function stampHandoffNote( + store: MessageStore, + threadId: string, + toolName: string, + note: string, +) { + const history = await store.loadThread(threadId) + for (let index = history.length - 1; index >= 0; index -= 1) { + const message = history[index] + if ( + message?.role !== 'assistant' || + isHandoffNote(message) || + !message.toolCalls?.some((call) => call.function.name === toolName) + ) { + continue + } + const harness = isRecord(message.metadata?.harness) + ? message.metadata.harness + : {} + if (harness.handoffNote === note) return + const updated: ModelMessage = { + ...message, + metadata: { + ...message.metadata, + harness: { ...harness, handoffNote: note }, + }, + } + await store.saveThread(threadId, history.with(index, updated)) + return + } + throw new Error( + `No ${toolName} call in the thread to stamp with the handoff note.`, + ) } /** * Mark the routed message of run `runId` with the agent that wrote it, in - * `metadata.harness.agent`. A routed pass that runs again on the same run id - * (a handoff to another root agent, a late join, a hook) first keeps this - * message under its own id, so each pass keeps its own message. + * `metadata.harness.agent`, and the child run of that pass, in + * `metadata.harness.subagentRunId`. A routed pass that runs again on the + * same run id (a handoff to another root agent, a late join, a hook) first + * keeps this message under its own id, so each pass keeps its own message. */ export async function markRouted( store: MessageStore, threadId: string, runId: string, - mark: { agent?: string }, + mark: { agent?: string; subagentRunId?: string }, ) { - const { agent } = mark + const { agent, subagentRunId } = mark if (agent === undefined) return const history = await store.loadThread(threadId) const index = routedIndex(history, runId) @@ -428,7 +533,11 @@ export async function markRouted( ...message, metadata: { ...message.metadata, - harness: { ...harness, agent }, + harness: { + ...harness, + agent, + ...(subagentRunId !== undefined ? { subagentRunId } : {}), + }, }, } await store.saveThread(threadId, history.with(index, updated)) diff --git a/packages/ai-harness/src/harness-agent.ts b/packages/ai-harness/src/harness-agent.ts index 8f7d7ec532..9cec92c77e 100644 --- a/packages/ai-harness/src/harness-agent.ts +++ b/packages/ai-harness/src/harness-agent.ts @@ -6,11 +6,21 @@ import type { AnyHarness } from './define' import type { HarnessHost } from './host' /** - * Use a whole harness as a child agent: put it in `subagents.agents` and the - * main model can call it as a tool, with its own tools, plugins, and history. + * Use a whole harness as a child agent, with its own tools, plugins, and + * history. It works in two places: * - * In a harness session, the child is a session of `harness` on the thread - * `${rootThreadId}/agent/${subagentRunId}`, in the log of the root thread. + * - In `subagents.agents`: the main model calls it as a tool. + * - In `agents`, with `routing`: it is a root agent that answers turns of + * the thread. It keeps one child session per agent across turns, on the + * thread `${rootThreadId}/agent/${name}`. Each turn sends it the latest + * user message, after what happened in the thread since its last turn. + * Its model gets the `handoff` tool of the outer turn, so it can hand + * back. A tool of the child can call `ctx.context.run.handoff`. The child + * harness cannot have its own `routing`: that turn fails. + * + * In a harness session, the child is a session of `harness` in the log of + * the root thread. As a tool, its thread is + * `${rootThreadId}/agent/${subagentRunId}`, one per call. * It runs for the user of the run that starts it, with the keys of that * user. Its model calls count in the usage and the budget of that run, and * the snapshot of the session lists its waits. An answer there goes to the @@ -28,6 +38,13 @@ import type { HarnessHost } from './host' * adapter, * subagents: { agents: [harnessAgent(reviewer)] }, * }) + * + * defineHarness({ + * name: 'acme/front-desk', + * adapter, + * agents: [harnessAgent(billing, { name: 'billing' })], + * routing: { router: () => 'billing' }, + * }) * ``` */ export function harnessAgent( diff --git a/packages/ai-harness/src/index.ts b/packages/ai-harness/src/index.ts index a8f0029384..700f2d0df3 100644 --- a/packages/ai-harness/src/index.ts +++ b/packages/ai-harness/src/index.ts @@ -1,6 +1,7 @@ export { defineHarness, isHarnessDefinition } from './define' export type { AnyHarness, + HandoffToolContext, HarnessAgentsOf, HarnessConfig, HarnessDefinition, diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts new file mode 100644 index 0000000000..04f412e638 --- /dev/null +++ b/packages/ai-harness/src/routed-history.ts @@ -0,0 +1,337 @@ +import { textOf } from './first-party/session-tools' +import { isHumanTarget } from './log-records' +import { isRecord } from './utils' +import type { + AnyChatMiddleware, + ChatMiddlewareConfig, + ChatMiddlewareContext, + ModelMessage, +} from '@tanstack/ai' +import type { HarnessRouting } from './define' + +/** A routed child of a run: the agent that answered, and its transcript id. */ +type RoutedChild = { name: string; subagentRunId: string } + +/** Where the routed runs of a thread and their transcripts come from. */ +interface HistorySource { + mode: NonNullable + /** The name of the handoff tool. `'handoff'` when not set. */ + handoffToolName?: string + /** The routed children of run `runId` for this host message, in order. */ + childrenOf: (runId: string, message: ModelMessage) => Array + /** The stored transcript of one routed child. */ + transcriptOf: (subagentRunId: string) => Promise> + /** + * True for a message of the shared thread. An assistant message that is + * not (the messages of the current chat, or of a resumed run) stays as it + * is. Every message counts when not set. + */ + isThreadMessage?: (message: ModelMessage) => boolean +} + +/** The run id of a routed host message: `metadata.tanstack.runId`. */ +export function routedRunId(message: ModelMessage) { + const tanstack = message.metadata?.tanstack + return isRecord(tanstack) && typeof tanstack.runId === 'string' + ? tanstack.runId + : undefined +} + +/** + * The run id of a routed pass. An earlier pass of a run with several passes + * keeps it in `metadata.harness.routedRunId`, because its `tanstack.runId` + * is gone. The last pass has `metadata.tanstack.runId`. + */ +function passRunId(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.routedRunId === 'string' + ? harness.routedRunId + : routedRunId(message) +} + +/** The root agent that wrote a message: `metadata.harness.agent`. */ +export function routedAgent(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.agent === 'string' + ? harness.agent + : undefined +} + +/** The child run of a routed pass: `metadata.harness.subagentRunId`. */ +export function routedChildId(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.subagentRunId === 'string' + ? harness.subagentRunId + : undefined +} + +/** True for a router or code handoff note: `metadata.harness.handoff`. */ +export function isHandoffNote(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && harness.handoff !== undefined +} + +/** The line that tells a reader about one handoff call. */ +function handoffLine(from: string, args: string) { + let parsed: unknown + try { + parsed = JSON.parse(args) + } catch { + parsed = undefined + } + if (!isRecord(parsed)) return `${from} handed off.` + const { to, note } = parsed + const target = isHumanTarget(to) ? 'a person' : to + const head = + typeof target === 'string' + ? `${from} handed off to ${target}.` + : `${from} handed off.` + return typeof note === 'string' ? `${head} Note: ${note}` : head +} + +/** The note a custom handoff tool recorded on this call, when it set one. */ +function recordedHandoffNote(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.handoffNote === 'string' + ? harness.handoffNote + : undefined +} + +/** `args` with `note` set, so a fold reads the recorded note. */ +function argumentsWithNote(args: string, note: string) { + try { + const parsed = JSON.parse(args) + if (!isRecord(parsed)) return JSON.stringify({ note }) + if (parsed.note === note) return args + return JSON.stringify({ ...parsed, note }) + } catch { + return JSON.stringify({ note }) + } +} + +/** The joined text of the assistant messages of a transcript. */ +function assistantText(messages: ReadonlyArray) { + return messages + .filter((message) => message.role === 'assistant') + .map((message) => textOf(message).trim()) + .filter((text) => text !== '') + .join('\n\n') +} + +/** + * The messages of a routed child for the model: its transcript without the + * task it started from, and without an empty placeholder message. + */ +function childMessages(transcript: Array) { + const rest = transcript[0]?.role === 'user' ? transcript.slice(1) : transcript + return rest.filter( + (message) => + message.role !== 'assistant' || + textOf(message) !== '' || + (message.toolCalls?.length ?? 0) > 0, + ) +} + +/** + * The text of another speaker as one user message, with the id and the + * metadata of the message it replaces. A user message: the words are not + * the reader's own, and the context still ends on a user turn after a + * handoff (some models, new Claude models among them, reject a context that + * ends on an assistant message). The fields of an assistant message + * (thinking, tool calls) do not go with it. + */ +function otherSpeaker(message: ModelMessage | undefined, content: string) { + const user: ModelMessage = { + ...(message?.id !== undefined && { id: message.id }), + ...(message?.metadata !== undefined && { metadata: message.metadata }), + role: 'user', + content, + } + return user +} + +/** + * Replace a routed host message with the transcripts of its children. A + * child that `wanted` picks comes in full. Another child comes as its + * `name:\ntext`. Undefined when no picked child has a stored transcript. + */ +async function expandHost( + message: ModelMessage, + children: Array, + wanted: (name: string) => boolean, + transcriptOf: HistorySource['transcriptOf'], +) { + // A host message of one agent holds only that agent's pass of the run. + // A pass-specific child id wins over agent name: two passes of the same + // agent in one turn share the outer run id and the agent name. + const childId = routedChildId(message) + const agent = routedAgent(message) + const own = + childId !== undefined + ? children.filter((child) => child.subagentRunId === childId) + : agent === undefined + ? children + : children.filter((child) => child.name === agent) + if (!own.some((child) => wanted(child.name))) return undefined + const parts: Array = [] + let expanded = false + for (const child of own) { + const body = childMessages(await transcriptOf(child.subagentRunId)) + if (wanted(child.name) && body.length > 0) { + parts.push(...body) + expanded = true + continue + } + const text = assistantText(body) + if (text !== '') { + parts.push(otherSpeaker(undefined, `${child.name}:\n${text}`)) + } + } + return expanded ? parts : undefined +} + +/** + * Fold one message of the thread for `speaker` (`null` for the main model) + * in `'own'` mode. A message of the speaker stays, except a fabricated + * handoff note (`metadata.harness.handoff`): those fold for every speaker, + * including the owner, because they are not a model call. A message of + * another speaker becomes text in one user message, so the speaker does + * not read it as its own: the main model's text as `main:\ntext`, a routed + * answer as its `name:\ntext`, and each handoff call as one line. Other + * tool calls of another speaker go. `folded` collects the ids of the tool + * calls that this removes, so their results go too. + */ +function foldFor( + message: ModelMessage, + speaker: string | null, + handoffToolName: string, + folded: Set, +) { + if (message.role === 'tool') { + const answersFolded = + message.toolCallId !== undefined && folded.has(message.toolCallId) + return answersFolded ? [] : [message] + } + // A note names the agent that handed off. A router note of the main + // model names none. Notes are fabricated: fold them for every speaker + // so a later turn does not keep a second handoff pair it never produced. + const owner = routedAgent(message) ?? null + const isNote = isHandoffNote(message) + if (message.role !== 'assistant' || (owner === speaker && !isNote)) { + return [message] + } + const from = owner ?? 'main' + const text = textOf(message).trim() + const lines: Array = [] + const calls = message.toolCalls ?? [] + if (isNote && calls.length === 0 && text !== '') { + // A plain note (no handoff tool): its text says who answers next. + lines.push(text) + } else if (!isNote && text !== '') { + // A routed answer has its `name:` label already. + lines.push(owner === null ? `main:\n${text}` : text) + } + for (const call of message.toolCalls ?? []) { + folded.add(call.id) + // A note pair always uses the built-in tool name. + if (isNote || call.function.name === handoffToolName) { + // A custom handoff tool can record a different note than the model + // put in the call. The target reads that note; the stored call stays. + const stamped = isNote ? undefined : recordedHandoffNote(message) + lines.push( + handoffLine( + from, + stamped === undefined + ? call.function.arguments + : argumentsWithNote(call.function.arguments, stamped), + ), + ) + } + } + return lines.length === 0 ? [] : [otherSpeaker(message, lines.join('\n\n'))] +} + +/** + * The history that `speaker` sees (`null` for the main model). + * + * - `'text'`: the messages as they are. + * - `'all'`: every routed host message becomes the transcripts of its run. + * - `'own'`: each speaker gets its own messages and its own routed runs in + * full, and the words of the other speakers as labeled text. + */ +export async function historyFor( + options: HistorySource & { + messages: Array + speaker: string | null + }, +) { + const { messages, speaker, mode, childrenOf, transcriptOf } = options + if (mode === 'text') return messages + const handoffToolName = options.handoffToolName ?? 'handoff' + const wanted = (name: string) => mode === 'all' || name === speaker + const folded = new Set() + const result: Array = [] + for (const message of messages) { + const isOutsideThread = + message.role === 'assistant' && + options.isThreadMessage?.(message) === false + if (isOutsideThread) { + result.push(message) + continue + } + const runId = passRunId(message) + const children = + message.role === 'assistant' && runId !== undefined + ? childrenOf(runId, message) + : [] + const expanded = + children.length > 0 + ? await expandHost(message, children, wanted, transcriptOf) + : undefined + // Unstamped multi-agent host (a list / { names } / { steps } pick + // never sets metadata.harness.agent). Fold as user text, even when + // this pass has no children left, and keep the existing name: labels. + // A stamped host still folds: its owner is the agent name. + const unstamped = + message.role === 'assistant' && + runId !== undefined && + routedAgent(message) === undefined && + !children.some((child) => wanted(child.name)) + if (expanded) result.push(...expanded) + else if (mode === 'all') result.push(message) + else if (unstamped) { + const text = textOf(message).trim() + if (text !== '') result.push(otherSpeaker(message, text)) + } else result.push(...foldFor(message, speaker, handoffToolName, folded)) + } + return result +} + +/** + * Middleware that gives each routed root agent its history for + * `routing.history`. It sets `providerMessages`, so the stored thread does + * not change. A nested child of a root agent keeps its own messages. + */ +export function routedHistoryMiddleware(options: HistorySource) { + return { + name: 'harness:routed-history', + onConfig: async ( + ctx: Pick, + config: Pick, + ) => { + if (options.mode === 'text' || ctx.parentSubagentRunId !== undefined) { + return undefined + } + const source = config.providerMessages ?? config.messages + const providerMessages = await historyFor({ + ...options, + messages: source, + speaker: ctx.subagentName ?? null, + }) + const isSame = + providerMessages.length === source.length && + providerMessages.every((message, index) => message === source[index]) + return isSame ? undefined : { providerMessages } + }, + } satisfies AnyChatMiddleware +} diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index fe4859fbce..4cb0f4a2d6 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -7,10 +7,11 @@ import { validateWithStandardSchema, } from '@tanstack/ai' import { withPersistence } from '@tanstack/ai-persistence' +import type { BlobBody } from '@tanstack/ai-persistence' import { redactMiddleware } from '../auth' import { BOUNDARY_INTERRUPTS, withoutBoundaryWaits } from '../boundary-waits' import { errorsOnlyDebug } from '../logger' -import { mediaCapture, mediaMiddleware } from '../media' +import { MediaError, mediaCapture, mediaMiddleware } from '../media' import { OperationImpl } from '../operation' import { checkpointMiddleware, @@ -55,6 +56,7 @@ import type { HeldPlugins } from './helpers' import { SessionTurnParts, stepWaitPrefix } from './turn-parts' import type { ChatMiddleware, + ContentPart, SubagentBudget, Interrupt, ModelMessage, @@ -69,6 +71,9 @@ import type { InnerTurnEnd } from '../harness-text' import type { AgentResultRecord, RunTreeNode } from '../log-records' import { NO_SESSION_CONTEXT, withHarnessChild } from '../agents' import { forwardTurn, lastUserInput } from '../harness-text' +import { textOf } from '../first-party/session-tools' +import { mediaIdOf, mediaPart } from '../media-ref' +import { historyFor, isHandoffNote, routedAgent } from '../routed-history' import { wait as backoff, isTransientModelError } from '../turn' import { addCounts, emptyCounts } from '../usage' import { isRecord } from '../utils' @@ -537,15 +542,15 @@ export abstract class SessionAgents< const extend: SubagentBinding['extend'] = (turn || handoffs) && ((child: SubagentRunContext) => { - const handoff = + // A routed root agent: a `harnessAgent` there keeps its child session. + const root = handoffs && child.parentSubagentRunId === undefined - ? (handoffOptions: HandoffOptions) => - handoffs.handoff( - handoffOptions, - child.forward.subagentName, - 'code', - ) - : NO_SESSION_CONTEXT.handoff + ? { name: child.forward.subagentName, handoffs } + : undefined + const handoff = root + ? (handoffOptions: HandoffOptions) => + root.handoffs.handoff(handoffOptions, root.name, 'code') + : NO_SESSION_CONTEXT.handoff if (!turn) return { harness: { ...NO_SESSION_CONTEXT, handoff } } // The steps of the child are in the log under its run id: the same // id when it goes on after a wait of the turn. @@ -561,7 +566,12 @@ export abstract class SessionAgents< state: (stateOptions) => this.sharedState(stateOptions), handoff, }, - (inner, ctx) => this.runHarnessChild(inner, ctx, { operation, user }), + (inner, ctx) => + this.runHarnessChild(inner, ctx, { + operation, + user, + ...(root && { root }), + }), run, ) return { harness } @@ -829,6 +839,12 @@ export abstract class SessionAgents< * child turns with `parentRunId`, then a RUN_FINISHED with the usage of * the child. A stop of the parent run cancels the child, and a recoverable * close stops it as on a crash. The child closes when the run ends. + * + * With `parent.root` (a routed root agent), the thread is + * `${rootThreadId}/agent/${name}`: one child session per agent, kept + * across turns. Each turn sends what happened in the shared thread since + * the last answer of the agent. Its main model gets the `handoff` tool of + * the outer turn, unless `routing.giveModelsHandoffTool` is false. */ protected async *runHarnessChild( harness: AnyHarness, @@ -836,13 +852,21 @@ export abstract class SessionAgents< parent: { operation: OperationImpl | OperationImpl user: HarnessUser | undefined + root?: { name: string; handoffs: TurnHandoffs } }, ): AsyncGenerator { const open = this.openChild if (!open) throw new Error('This session cannot open a child session.') + const { root } = parent const parentRunId = parent.operation.id - const threadId = `${this.rootThreadId}/agent/${ctx.subagentRunId}` + const threadId = `${this.rootThreadId}/agent/${root?.name ?? ctx.subagentRunId}` const used = emptyCounts() + // With the handoff tool off, the child still hands off from code. + const withHandoffTool = + this.harness.routing?.giveModelsHandoffTool !== false + const handoff = root + ? childHandoff(root.name, root.handoffs, withHandoffTool) + : undefined const session = await open(harness, { threadId, user: parent.user, @@ -857,6 +881,7 @@ export abstract class SessionAgents< }) this.checkBudget(parent.operation) }, + ...(handoff && { handoff }), }, }) let operation: Operation | undefined @@ -899,8 +924,22 @@ export abstract class SessionAgents< }) signal.addEventListener('abort', onAbort, { once: true }) try { + const messages = convertMessagesToModelMessages(ctx.messages) operation = session.send( - lastUserInput(convertMessagesToModelMessages(ctx.messages)), + root + ? await copyThreadMedia( + { + getMedia: (id) => this.getMedia({ id }), + open: (id) => this.mediaStore.open(id), + }, + session, + await rootInput( + messages, + root.name, + withHandoffTool ? root.handoffs.tool(null).name : undefined, + ), + ) + : lastUserInput(messages), ) if (signal.aborted) onAbort() let finished: InnerTurnEnd['finished'] = 'stop' @@ -2777,6 +2816,159 @@ function withParentRun(chunk: StreamChunk, parentRunId: string): StreamChunk { }) } +/** + * The link that lets the child session of the routed root agent `from` hand + * off in the outer turn: the outer `handoff` tool, `run.handoff`, and a check + * that stops the tool loop of the child after a handoff. + */ +function childHandoff(from: string, handoffs: TurnHandoffs, withTool: boolean) { + // The outer middleware stops the chat of `from` after its handoff. The + // child asks it the same question. + const { onShouldContinue } = handoffs.middleware() + return { + from, + ...(withTool ? { tool: handoffs.tool(from) } : {}), + call: (options: HandoffOptions) => handoffs.handoff(options, from, 'code'), + pending: () => onShouldContinue({ subagentName: from }) === false, + } +} + +/** + * The input of a turn of the routed root `harnessAgent` named `name`: + * what happened in the shared thread since the last answer of `name`, then + * the latest user message when that message is new (it comes after the last + * answer). Other speakers come with their labels, as `routing.history: 'own'` + * folds them. The child session keeps its own turns, so they are + * not sent again. A second pass in the same outer turn does not resend the + * already-answered request: only the context since the last answer, or a + * short continue prompt when that context is empty. On its first turn, the + * context is the conversation so far. + */ +async function rootInput( + messages: Array, + name: string, + handoffToolName?: string, +) { + // A handoff note names its source agent too, but it is no answer. + const isAnswer = (message: ModelMessage) => + routedAgent(message) === name && !isHandoffNote(message) + const lastAnswer = messages.findLastIndex(isAnswer) + const latest = messages.findLastIndex((message) => message.role === 'user') + // The latest user message is new input only when it comes after this + // agent last answered. A second pass in the same outer turn (desk → + // main → desk) has lastAnswer after latest, so the kept child session + // must not get that request again. + const hasNewUser = latest > lastAnswer + const since = messages.filter( + (message, index) => + index > lastAnswer && + index !== latest && + // This agent's own handoff is in its child session, not this context. + !(isHandoffNote(message) && routedAgent(message) === name), + ) + // No transcripts: other runs stay as their `name:\ntext` host message. + const folded = await historyFor({ + messages: since, + speaker: name, + mode: 'own', + childrenOf: () => [], + transcriptOf: async () => [], + ...(handoffToolName !== undefined && { handoffToolName }), + }) + // A message of the user comes back as the same object. A folded message + // of another speaker is a new user message that has its label already. + const fromUser = new Set(since.filter((message) => message.role === 'user')) + const lines: Array = [] + const earlierMedia: Array = [] + for (const message of folded) { + if (message.role === 'tool') continue + const line = textOf(message).trim() + if (fromUser.has(message)) { + if (line !== '') lines.push(`user:\n${line}`) + if (Array.isArray(message.content)) { + for (const part of message.content) { + if (part.type !== 'text') earlierMedia.push(part) + } + } + continue + } + if (line !== '') lines.push(line) + } + const input = hasNewUser ? lastUserInput(messages) : undefined + const sent: Array = + input === undefined + ? [] + : typeof input === 'string' + ? [{ type: 'text', content: input }] + : input + if (lines.length === 0 && earlierMedia.length === 0) { + return input ?? 'Continue.' + } + const head = + lastAnswer === -1 ? 'The conversation so far:' : 'Since your last turn:' + const context: Array = [ + { + type: 'text', + content: lines.length === 0 ? head : [head, ...lines].join('\n\n'), + }, + ...earlierMedia, + ] + return sent.length === 0 ? context : [...context, ...sent] +} + +/** + * Copy parent-thread `harness-media:` parts in `input` into `dest`, so the + * child session can read them. Same copy as `adoptMessages`: each file gets + * a new id on the child thread. + */ +async function copyThreadMedia( + source: { + getMedia: (id: string) => Promise + open: (id: string) => Promise + }, + dest: { + putMedia: (upload: { + body: BlobBody + mimeType: string + name: string + }) => Promise + }, + input: UserInput, +): Promise { + if (typeof input === 'string') return input + const copies = new Map() + const copy = async (id: string) => { + if (!copies.has(id)) { + const record = await source.getMedia(id) + let copied: MediaRecord | null = null + if (record) { + try { + copied = await dest.putMedia({ + body: await source.open(id), + mimeType: record.mimeType, + name: record.name, + }) + } catch (error) { + // A missing blob must not abort the child turn: keep the original + // part so the child's media middleware can emit a not-found note. + if (!(error instanceof MediaError && error.status === 404)) { + throw error + } + } + } + copies.set(id, copied) + } + return copies.get(id) ?? undefined + } + return Promise.all( + input.map(async (part) => { + const id = mediaIdOf(part) + const made = id === undefined ? undefined : await copy(id) + return made ? mediaPart(made) : part + }), + ) +} + /** * The admission of agent runs: at most `max()` holders at once, no limit * when it is `undefined`. `busy` names more holders that the gate does not diff --git a/packages/ai-harness/src/session/turn.ts b/packages/ai-harness/src/session/turn.ts index 2aa178562b..d41563e45e 100644 --- a/packages/ai-harness/src/session/turn.ts +++ b/packages/ai-harness/src/session/turn.ts @@ -60,7 +60,13 @@ import { import { SessionAgents } from './agents' import { commandStepScope, stepWaitPrefix } from './turn-parts' import { ToolTurnAdapter } from '../tool-turn' -import { HANDOFF_TOOL, TurnHandoffs, answererOf, markRouted } from '../handoff' +import { + TurnHandoffs, + answererOf, + markRouted, + stampHandoffNote, +} from '../handoff' +import { routedChildId, routedHistoryMiddleware } from '../routed-history' import type { AnyChatMiddleware, AnyTextAdapter, @@ -717,7 +723,15 @@ export abstract class SessionTurn< run, ...(attempt ? { attempt } : {}), } + /** The link of the child session of a routed root `harnessAgent`. */ + const childHandoff = this.childLink?.handoff try { + // Two handoff scopes in one session: a handoff of the child is unclear. + if (childHandoff && this.harness.routing) { + throw new Error( + `A harnessAgent that is a root agent cannot have its own routing. Remove routing from "${this.harness.name}".`, + ) + } await this.flushNotes() // The stored settings of the thread, as they are when the turn starts. const stored = this.currentSettings @@ -799,6 +813,9 @@ export abstract class SessionTurn< subagents: subagentList.map((agent) => agent.name), humans: this.harness.routing.allowHandoffToHuman === true, tool: this.harness.routing.giveModelsHandoffTool !== false, + ...(this.harness.routing.handoffTool + ? { handoffTool: this.harness.routing.handoffTool } + : {}), append: (record) => this.appendHandoff(record, storedUser(sender).user), publish: (record) => @@ -894,27 +911,19 @@ export abstract class SessionTurn< // here, so a middleware wrapper wraps the bound tool, // and again at each model call, for tools that a middleware returns. const bindTool = this.durableBinder(run) - // `routing.giveModelsHandoffTool: false` gives the models no `handoff` tool. + // `routing.giveModelsHandoffTool: false` gives the models no handoff tool. + // A child of a routed root agent gets the handoff tool of the outer turn. const handoffTool = handoffs && this.harness.routing?.giveModelsHandoffTool !== false ? handoffs.tool(null) - : undefined - const clash = handoffTool - ? prepared.find((tool) => tool.name === HANDOFF_TOOL) - : undefined - if (clash) { - const owner = - [ - ...(session?.owners.tools ?? []), - ...(runPlugins?.owners.tools ?? []), - ].find((entry) => entry.name === HANDOFF_TOOL)?.owner ?? 'the turn' - throw new Error( - `Duplicate tool "${HANDOFF_TOOL}": first owner routing (the handoff tool), second owner ${owner}. Rename the tool.`, - ) - } + : childHandoff?.tool + // An app tool with the same name wins, as it does for a root agent. + const isHandoffTaken = prepared.some( + (tool) => tool.name === handoffTool?.name, + ) const tools = [ ...prepared.map((tool) => bindDurable(tool, bindTool)), - ...(handoffTool ? [handoffTool] : []), + ...(handoffTool && !isHandoffTaken ? [handoffTool] : []), ] // The override tools as the turn has them: prepared and bound. const keptTools = tools.filter((tool) => turnToolNames.has(tool.name)) @@ -957,6 +966,50 @@ export abstract class SessionTurn< const durableTools = this.durableTools(bindTool) const signal = operation.abortController.signal const { routing } = this.harness + // How a model call sees the routed runs of the thread. The transcripts + // do not change in a turn, and onConfig runs before each model call, + // so the turn reads each one once. + const historyMode = routing?.history ?? 'own' + const transcripts = new Map>>() + const routedHistory = + handoffs && historyMode !== 'text' + ? routedHistoryMiddleware({ + mode: historyMode, + ...(handoffTool ? { handoffToolName: handoffTool.name } : {}), + // ponytail: the log fold (the run store too) keeps the last + // `KEPT_ENDED` ended runs. An older routed run has no children + // there, so the model gets it as text. Keep the routed children + // in the fold if that matters. + childrenOf: (runId, message) => { + const childId = routedChildId(message) + return [...this.writer.state.thread.runs.values()].flatMap( + ({ parentRunId: parent, name, subagentRunId }) => + parent === runId && + name !== undefined && + subagentRunId !== undefined && + // Only routed root agents: a tool-mode child of the main + // model in the same run reports through its tool result. + rootAgents.some((agent) => agent.name === name) && + (childId === undefined || subagentRunId === childId) + ? [{ name, subagentRunId }] + : [], + ) + }, + transcriptOf: (subagentRunId) => { + const transcript = + transcripts.get(subagentRunId) ?? + this.agentTranscript({ subagentRunId }) + transcripts.set(subagentRunId, transcript) + return transcript + }, + // A routed agent's own messages of its chat, and of a resumed + // run, are in its own thread. ponytail: a scan of the thread + // for each message. A set of ids if long threads get slow. + isThreadMessage: (entry) => + entry.id !== undefined && + this.writer.state.messages.some(({ id }) => id === entry.id), + }) + : undefined /** * The messages that the next chat() call adds after `message`: the * note of a handoff, or the partial answer and the note that a @@ -1038,6 +1091,8 @@ export abstract class SessionTurn< let answerer = isRootPart ? answererOf(pick) : null // A tool of the main model hands off as the main model. A tool of a // root agent hands off as that agent. + // The child of a routed root agent hands off in the outer turn. + if (childHandoff) turnHandoff = childHandoff.call if (handoffs) { turnHandoff = (options) => handoffs.handoff( @@ -1135,10 +1190,13 @@ export abstract class SessionTurn< // The root agents of a routed part take steers at each model // call, as the main model does, and get the ephemeral messages // of a turn hook. Those go after the reset cut, so it keeps them. + // The routed history goes after the reset cut too, so it + // changes only the context after a reset. const { chatMiddleware } = binding chatMiddleware.splice( chatMiddleware.indexOf(resetCut) + 1, 0, + ...(routedHistory ? [routedHistory] : []), ephemeralCall(ephemeral), ) if (!isToolTurn) { @@ -1190,10 +1248,23 @@ export abstract class SessionTurn< durableTools, this.detachTools(operation.id, sender), ...(handoffs ? [handoffs.middleware()] : []), + // After a handoff of the child, the outer turn runs the target. + ...(childHandoff + ? [ + { + name: 'harness:child-handoff', + onShouldContinue: () => + childHandoff.pending() ? false : undefined, + }, + ] + : []), ...(rootBag ? [] : [this.syncFold()]), // Again: a middleware that returns `messages` (as `syncFold` // does) resets what the model gets to the whole transcript. resetCut, + // After the reset cut. In `'own'` the main model gets the + // answers of the routed agents as labeled user text. + ...(routedHistory ? [routedHistory] : []), // After steering and the reset, so neither drops them. ...(start && turn.ephemeral ? [turnEphemeral(turn.ephemeral, start)] @@ -1503,8 +1574,18 @@ export abstract class SessionTurn< // The routed message of this part gets the name of its agent. A part // that waits for an answer gets it too. if (routed && failure === undefined && !signal.aborted) { + const childId = + typeof answerer === 'string' + ? [...this.writer.state.thread.runs.values()].findLast( + (run) => + run.parentRunId === operation.id && + run.name === answerer && + run.subagentRunId !== undefined, + )?.subagentRunId + : undefined await markRouted(this.messages, this.threadId, operation.id, { ...(answerer ? { agent: answerer } : {}), + ...(childId !== undefined ? { subagentRunId: childId } : {}), }) } const isFinished = @@ -1514,6 +1595,22 @@ export abstract class SessionTurn< if (isFinished) break const handoff = handoffs?.take() const target = handoff?.to + // A custom handoff tool can rewrite the note. Stamp it on the main + // model's call so the target reads it. No extra note messages: the + // call is already in the thread. + if ( + handoff?.by === 'tool' && + handoff.from === null && + typeof handoff.note === 'string' && + handoffs + ) { + await stampHandoffNote( + this.messages, + this.threadId, + handoffs.tool(null).name, + handoff.note, + ) + } // The note of a code handoff, for the transcript. const note = handoff ? (handoffs?.messages(handoff) ?? []) : [] if (isHumanTarget(target)) { @@ -1914,18 +2011,24 @@ const attachmentId = (id: string) => `${id}:media` /** * The answer message of a routed run, kept under the id `${id}:${index}`. * It loses its run tag, so the store writes the next answer of the run as a - * new message. + * new message. The run id goes to `metadata.harness.routedRunId`, so the + * routed history of a later turn still finds the children of the pass. */ -function keptAnswer(message: ModelMessage, index: number): ModelMessage { +function keptAnswer(message: ModelMessage, index: number) { const { metadata } = message const tanstack = isRecord(metadata) ? metadata.tanstack : undefined if (!isRecord(metadata) || !isRecord(tanstack)) { return { ...message, id: `${message.id}:${index}` } } - const { runId: _runId, ...rest } = tanstack + const { runId, ...rest } = tanstack + const harness = isRecord(metadata.harness) ? metadata.harness : {} return { ...message, id: `${message.id}:${index}`, - metadata: { ...metadata, tanstack: rest }, + metadata: { + ...metadata, + tanstack: rest, + harness: { ...harness, routedRunId: runId }, + }, } } diff --git a/packages/ai-harness/src/session/types.ts b/packages/ai-harness/src/session/types.ts index 1d4a1c6ad9..b9cbf47364 100644 --- a/packages/ai-harness/src/session/types.ts +++ b/packages/ai-harness/src/session/types.ts @@ -1,5 +1,6 @@ import type { CredentialStore } from '@tanstack/ai/byok' import type { + AnyTool, SubagentBudget, Interrupt, Logger, @@ -29,6 +30,7 @@ import type { SessionImpl } from './harness-session' import type { AgentInputOf, AgentResultOf, AnyAgent } from '../agents' import type { PendingWait } from '../waits' import type { HandoffRecord } from '../log-records' +import type { HandoffOptions } from '../handoff' import type { AnyHarness, HarnessAgentsOf } from '../define' import type { HarnessPersistence, @@ -472,6 +474,23 @@ export interface ChildSessionLink { * the budget of that run. The child keeps no usage of its own. */ countUsage: (call: UsageCall) => Promise + /** + * Set only for the child session of a `harnessAgent` that is a routed + * root agent. It lets the child hand off in the outer turn. + */ + handoff?: { + /** The name of the agent in the outer turn. */ + from: string + /** + * The outer handoff tool, for the main model of the child. None when + * `routing.giveModelsHandoffTool` is false. + */ + tool?: AnyTool + /** Hand off in the outer turn. */ + call: (options: HandoffOptions) => Promise + /** True after a handoff. Then the child stops its tool loop. */ + pending: () => boolean + } } /** A child session of `harnessAgent` that a run of a session has open. */ diff --git a/packages/ai-harness/tests/handoff-tool.test.ts b/packages/ai-harness/tests/handoff-tool.test.ts new file mode 100644 index 0000000000..38ba4b13a0 --- /dev/null +++ b/packages/ai-harness/tests/handoff-tool.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { TurnHandoffs, stampHandoffNote } from '../src/handoff' +import type { AnyTool } from '@tanstack/ai' +import type { HandoffToolContext } from '../src' +import type { HandoffRecord } from '../src/log-records' + +function agent(name: string) { + return defineAgent({ + name, + description: `The ${name} desk`, + run: async () => name, + }) +} + +const ROOTS = [agent('billing'), agent('triage')] + +/** The middleware context of the main model's chat. */ +const MAIN = {} +/** The middleware context of the chat of root agent `name`. */ +const rootChat = (name: string) => ({ subagentName: name }) + +/** `TurnHandoffs` with a fake log: `records` gets each published record. */ +function turnHandoffs( + options: { + humans?: boolean + tool?: boolean + handoffTool?: (ctx: HandoffToolContext) => AnyTool + } = {}, +) { + const records: Array = [] + const handoffs = new TurnHandoffs({ + runId: 'run_1', + noteKey: 'input_1', + rootAgents: ROOTS, + subagents: [], + humans: options.humans ?? false, + tool: options.tool ?? true, + append: async (record) => record, + publish: (record) => { + records.push(record) + }, + ...(options.handoffTool ? { handoffTool: options.handoffTool } : {}), + }) + return { handoffs, records } +} + +/** A `handoffTool` factory that keeps each context it gets. */ +function transferFactory(name = 'transfer') { + const contexts: Array = [] + const factory = (ctx: HandoffToolContext) => { + contexts.push(ctx) + return toolDefinition({ + name, + description: 'Move the user to another desk.', + inputSchema: z.object({ to: z.string(), note: z.string() }), + }).server(async ({ to, note }) => { + await ctx.handoff({ to, note }) + return `Moved to ${to}.` + }) + } + return { contexts, factory } +} + +/** An app tool named `handoff`. */ +const appHandoff = toolDefinition({ + name: 'handoff', + description: 'The app hands off.', + inputSchema: z.object({ to: z.string() }), +}).server(async ({ to }) => `App moved to ${to}.`) + +/** The one record of `records`. */ +function onlyRecord(records: Array) { + expect(records).toHaveLength(1) + const [record] = records + if (!record) throw new Error('No handoff record.') + return record +} + +describe('routing.handoffTool factory', () => { + it.each([true, false])( + 'gets the speaker, its targets, and allowHuman %s', + (humans) => { + const { contexts, factory } = transferFactory() + const { handoffs } = turnHandoffs({ humans, handoffTool: factory }) + + handoffs.tool(null) + handoffs.tool('billing') + + expect( + contexts.map(({ from, targets, allowHuman }) => ({ + from, + targets, + allowHuman, + })), + ).toEqual([ + { from: null, targets: ['billing', 'triage'], allowHuman: humans }, + { from: 'billing', targets: ['triage', 'main'], allowHuman: humans }, + ]) + }, + ) + + it('records a handoff of the factory tool with by tool, and adds a note only for a root agent', async () => { + const { factory } = transferFactory() + const { handoffs, records } = turnHandoffs({ handoffTool: factory }) + + const result = await handoffs + .tool('billing') + .execute?.({ to: 'triage', note: 'Needs a ticket' }) + + expect(result).toBe('Moved to triage.') + const record = onlyRecord(records) + expect(record).toEqual({ + runId: 'run_1', + from: 'billing', + to: 'triage', + note: 'Needs a ticket', + by: 'tool', + }) + // A root agent keeps its tool calls in its own transcript, so the shared + // thread gets the note pair. The main model's call is in the thread. + expect(handoffs.messages(record)).toMatchObject([ + { + role: 'assistant', + toolCalls: [{ function: { name: 'handoff' } }], + metadata: { harness: { agent: 'billing', handoff: { by: 'tool' } } }, + }, + { role: 'tool', content: 'Handed off to triage. triage answers next.' }, + ]) + expect(handoffs.messages({ ...record, from: null })).toEqual([]) + }) + + it('keeps the default handoff tool when no factory is set', async () => { + const { handoffs, records } = turnHandoffs() + const tool = handoffs.tool(null) + + const result = await tool.execute?.({ to: 'billing' }) + + expect(tool.name).toBe('handoff') + expect(result).toBe('Handed off to billing. billing answers next.') + expect(onlyRecord(records)).toEqual({ + runId: 'run_1', + from: null, + to: 'billing', + by: 'tool', + }) + }) +}) + +describe('a code handoff inside a handoff tool call', () => { + it('records by tool while a handoff-named call runs', async () => { + const { handoffs, records } = turnHandoffs() + const middleware = handoffs.middleware() + + await middleware.onGuardToolCall(MAIN, { + toolName: 'handoff', + toolCallId: 'call_1', + }) + await handoffs.handoff({ to: 'billing', note: 'Refund' }, null, 'code') + + expect(onlyRecord(records).by).toBe('tool') + }) + + it('records by code after the handoff-named call ends', async () => { + const { handoffs, records } = turnHandoffs() + const middleware = handoffs.middleware() + + await middleware.onGuardToolCall(MAIN, { + toolName: 'handoff', + toolCallId: 'call_1', + }) + await middleware.onAfterToolCall(MAIN, { toolCallId: 'call_1' }) + await handoffs.handoff({ to: 'billing' }, null, 'code') + + expect(onlyRecord(records).by).toBe('code') + }) + + it('records by code inside a call of another tool', async () => { + const { handoffs, records } = turnHandoffs() + const middleware = handoffs.middleware() + + await middleware.onGuardToolCall(MAIN, { + toolName: 'search', + toolCallId: 'call_1', + }) + await handoffs.handoff({ to: 'billing' }, null, 'code') + + expect(onlyRecord(records).by).toBe('code') + }) +}) + +describe('the handoff tool of a root agent', () => { + it('adds no tool and never calls the factory with giveModelsHandoffTool false', async () => { + const { contexts, factory } = transferFactory() + const { handoffs } = turnHandoffs({ tool: false, handoffTool: factory }) + const middleware = handoffs.middleware({ roots: true }) + + const change = middleware.onConfig(rootChat('billing'), { tools: [] }) + await middleware.onGuardToolCall(rootChat('billing'), { + toolName: 'transfer', + toolCallId: 'call_1', + }) + + expect(change).toBeUndefined() + expect(contexts).toEqual([]) + }) + + it('keeps an app tool with the handoff name only for the agent that has it', () => { + const { factory } = transferFactory('handoff') + const { handoffs } = turnHandoffs({ handoffTool: factory }) + const middleware = handoffs.middleware({ roots: true }) + + const billing = middleware.onConfig(rootChat('billing'), { + tools: [appHandoff], + }) + const triage = middleware.onConfig(rootChat('triage'), { tools: [] }) + + expect(billing).toBeUndefined() + expect(triage).toEqual({ tools: [handoffs.tool('triage')] }) + // The next model call of triage has the tool already. + expect( + middleware.onConfig(rootChat('triage'), { tools: triage?.tools ?? [] }), + ).toBeUndefined() + }) +}) + +describe('stampHandoffNote', () => { + it('throws when the thread has no call of that tool', async () => { + const store = { + loadThread: async () => [], + saveThread: async () => {}, + } + await expect( + stampHandoffNote(store, 'thread_1', 'handoff', 'Refund for order 42'), + ).rejects.toThrow( + 'No handoff call in the thread to stamp with the handoff note.', + ) + }) +}) diff --git a/packages/ai-harness/tests/handoff.test.ts b/packages/ai-harness/tests/handoff.test.ts index 74d8d8b6c5..fa815d367c 100644 --- a/packages/ai-harness/tests/handoff.test.ts +++ b/packages/ai-harness/tests/handoff.test.ts @@ -55,6 +55,7 @@ async function openRouting(options: { humans?: boolean log?: LogStore ownership?: OwnershipOptions + handoffTool?: HarnessRouting['handoffTool'] }) { const { metadata } = memoryPersistence().stores const log = options.log ?? memoryLogStore() @@ -79,6 +80,7 @@ async function openRouting(options: { ? { giveModelsHandoffTool: options.handoffs } : {}), ...(options.humans ? { allowHandoffToHuman: true } : {}), + ...(options.handoffTool ? { handoffTool: options.handoffTool } : {}), }, }), { threadId: THREAD, user: USER }, @@ -381,17 +383,67 @@ describe('handoffs', () => { await host.close() }) - it('fails the turn when an app tool is named handoff', async () => { + it('an app tool named handoff replaces the harness tool for main', async () => { const billing = modelAgent('billing', []) - const { host, session } = await openRouting({ + const { host, session, main } = await openRouting({ agents: [billing.agent], tools: [plainTool('handoff')], - replies: [() => text('never')], + replies: [() => text('Main')], + }) + + expect((await session.send('hi')).text).toBe('Main') + // Only the app tool, once. + const tools = main.calls[0].tools + expect(toolNames(main.calls[0])).toEqual(['handoff']) + expect(tools[0].description).toBe('The handoff tool') + await host.close() + }) + + it('routing.handoffTool gives each speaker its own tool', async () => { + const billing = modelAgent('billing', [() => text('Refunded')]) + const { host, session, main, log } = await openRouting({ + agents: [billing.agent], + handoffTool: ({ from, handoff }) => + toolDefinition({ + name: 'transfer', + description: `Transfer from ${from ?? 'main'}`, + inputSchema: z.object({ to: z.string(), note: z.string() }), + }).server(async ({ to, note }) => { + await handoff({ to, note: `${from ?? 'main'}: ${note}` }) + return `Moved to ${to}.` + }), + replies: [ + () => + toolCall('transfer', { to: 'billing', note: 'order 42' }, 'call_t'), + ], }) - await expect(session.send('hi')).rejects.toThrow( - 'Duplicate tool "handoff": first owner routing (the handoff tool), second owner the harness.', + expect((await session.send('refund please')).text).toBe( + 'billing:\nRefunded', ) + const description = (call: { tools: Array }) => + call.tools.map((tool) => tool.description) + expect(toolNames(main.calls[0])).toEqual(['transfer']) + expect(description(main.calls[0])).toEqual(['Transfer from main']) + expect(toolNames(billing.model.calls[0])).toEqual(['transfer']) + expect(description(billing.model.calls[0])).toEqual([ + 'Transfer from billing', + ]) + expect( + (await handoffRecords(log)).map(({ from, to, note, by }) => ({ + from, + to, + note, + by, + })), + ).toEqual([ + { from: null, to: 'billing', note: 'main: order 42', by: 'tool' }, + ]) + // billing gets the rewritten note from the tool call, once. + expect(billing.model.calls[0].messages.at(-1)).toMatchObject({ + role: 'user', + content: 'main handed off to billing. Note: main: order 42', + }) await host.close() }) @@ -559,11 +611,14 @@ describe('handoffs', () => { ...call(1, undefined, { to: 'triage' }), ...call(2, 'triage', { to: 'billing', note: 'A refund' }), ]) - // The target gets the note of its handoff last. - expect(billing.model.calls[0].messages.slice(-2)).toMatchObject([ - { role: 'assistant', toolCalls: [{ id: `handoff_${inputId}_2` }] }, - { role: 'tool', toolCallId: `handoff_${inputId}_2` }, - ]) + // The target gets the note of its handoff last, as a text line in a user + // message: with `routing.history: 'own'`, a call of another speaker is + // text, and the context ends on a user turn. + expect(billing.model.calls[0].messages.at(-1)).toMatchObject({ + id: `handoff_${inputId}_2`, + role: 'user', + content: 'triage handed off to billing. Note: A refund', + }) await host.close() }) @@ -605,7 +660,11 @@ describe('handoffs', () => { role: 'assistant', content: 'Handed off to billing. billing answers next.', } - expect(billing.model.calls[0].messages.at(-1)).toMatchObject(note) + // The target gets the plain note as text of another speaker. + expect(billing.model.calls[0].messages.at(-1)).toMatchObject({ + ...note, + role: 'user', + }) expect( (await session.transcript()).filter((message) => message.id?.startsWith('handoff_'), diff --git a/packages/ai-harness/tests/harness-agent-root.test.ts b/packages/ai-harness/tests/harness-agent-root.test.ts new file mode 100644 index 0000000000..425d68a4a3 --- /dev/null +++ b/packages/ai-harness/tests/harness-agent-root.test.ts @@ -0,0 +1,395 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { memoryLogStore, memoryPersistence } from '@tanstack/ai-persistence' +import { createHarnessHost, defineHarness, harnessAgent } from '../src' +import { mediaPart } from '../src/media-entry' +import { mockAdapter, storedThreadState, text, toolCall } from './helpers' +import type { ModelMessage } from '@tanstack/ai' +import type { AnyHarness, HarnessRouting } from '../src' +import type { Reply } from './helpers' + +const THREAD = 't1' + +/** A tool of the child harness, so its model has a tool call to keep. */ +const lookup = toolDefinition({ + name: 'lookup', + description: 'Looks a thing up', + inputSchema: z.object({}), +}).server(async () => 'found') + +/** The child harness of the `desk` root agent, and its model. */ +function deskHarness(replies: Array) { + const model = mockAdapter(replies) + const harness = defineHarness({ + name: 'test/desk', + adapter: model.adapter, + tools: [lookup], + }) + return { harness, model } +} + +/** The stores of a host. A second host on the same stores reads the same log. */ +function stores() { + return { + log: memoryLogStore(), + metadata: memoryPersistence().stores.metadata, + } +} + +/** + * Open the routed thread on a new host. Its only root agent is the + * harnessAgent `desk` of `child`. + */ +async function openRouted(options: { + child: AnyHarness + router: HarnessRouting['router'] + mainReplies?: Array + stores?: ReturnType +}) { + const { log, metadata } = options.stores ?? stores() + const main = mockAdapter(options.mainReplies ?? []) + const host = createHarnessHost({ persistence: { stores: { log, metadata } } }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: main.adapter, + agents: [harnessAgent(options.child, { name: 'desk' })], + routing: { router: options.router }, + }), + { threadId: THREAD }, + ) + return { host, session, main, log } +} + +/** + * The text that the user sent in each user message of one model call: the + * string, or the last text part. The parts before it are the context. + */ +const userTexts = (call: { messages: Array }) => + call.messages + .filter((message) => message.role === 'user') + .map((message) => + typeof message.content === 'string' + ? message.content + : (message.content ?? []).findLast((part) => part.type === 'text') + ?.content, + ) + +/** The names of the tool calls in the messages of one model call. */ +const calledTools = (call: { messages: Array }) => + call.messages.flatMap((message) => + (message.toolCalls ?? []).map((entry) => entry.function.name), + ) + +/** A router that picks `picks[n]` on turn `n`. */ +function picksInOrder(picks: Array) { + let turn = 0 + return () => picks[turn++] +} + +describe('harnessAgent as a routed root agent', () => { + it('keeps one child session, so turn 2 sees turn 1 and its own tool call', async () => { + const desk = deskHarness([ + () => toolCall('lookup', {}, 'call-lookup'), + () => text('Looked it up.'), + () => text('Second answer.'), + ]) + const { host, session } = await openRouted({ + child: desk.harness, + router: () => 'desk', + }) + + await session.send('first question') + await session.send('second question') + + const turn2 = desk.model.calls[2] + expect(userTexts(turn2)).toEqual(['first question', 'second question']) + expect(calledTools(turn2)).toEqual(['lookup']) + expect( + turn2.messages.some( + (message: ModelMessage) => + message.role === 'tool' && message.content === 'found', + ), + ).toBe(true) + await host.close() + }) + + it('sends what main said since its last turn, labeled main:', async () => { + const desk = deskHarness([ + () => text('Desk one.'), + () => text('Desk three.'), + ]) + const { host, session } = await openRouted({ + child: desk.harness, + router: picksInOrder(['desk', 'main', 'desk']), + mainReplies: [() => text('Main two.')], + }) + + await session.send('first') + await session.send('second') + await session.send('third') + + const turn3 = desk.model.calls[1] + const input = turn3.messages.at(-1) + expect(input.role).toBe('user') + expect(input.content).toEqual([ + { + type: 'text', + content: [ + 'Since your last turn:', + // Its own hand-back is in its own session, so it is not here. + 'user:\nsecond', + 'main:\nMain two.', + 'main handed off to desk.', + ].join('\n\n'), + }, + { type: 'text', content: 'third' }, + ]) + // The earlier turn of the desk stays in its own session once. + expect(userTexts(turn3).filter((entry) => entry === 'first')).toHaveLength( + 1, + ) + await host.close() + }) + + it('hands back to main with the handoff tool, and main stays active', async () => { + const desk = deskHarness([() => toolCall('handoff', { to: 'main' })]) + const { host, session, main, log } = await openRouted({ + child: desk.harness, + router: picksInOrder(['desk', undefined]), + mainReplies: [() => text('Main took over.'), () => text('Main again.')], + }) + + const first = await session.send('help') + + // The child model has the outer handoff tool, to main only. + const tool = desk.model.calls[0].tools.find( + (entry: { name: string }) => entry.name === 'handoff', + ) + expect(tool.inputSchema.properties.to).toEqual({ + type: 'string', + enum: ['main'], + }) + // The child stopped after the handoff, and main answered in the turn. + expect(desk.model.calls).toHaveLength(1) + expect(first.text).toContain('Main took over.') + // The call is in the child thread, so the shared thread gets the note. + // Main reads who handed the conversation back, as text of another + // speaker, not as a call of its own. + const transcript = await session.transcript() + expect( + transcript.find( + (message) => + message.role === 'tool' && + message.content === 'Handed off to main. main answers next.', + )?.metadata?.harness, + ).toEqual({ agent: 'desk', handoff: { by: 'tool' } }) + expect(main.calls[0].messages.at(-1)).toMatchObject({ + role: 'user', + content: 'desk handed off to main.', + }) + expect( + (await storedThreadState(log, { threadId: THREAD })).activeAgent, + ).toBeNull() + + expect((await session.send('more')).text).toBe('Main again.') + expect(desk.model.calls).toHaveLength(1) + expect(main.calls).toHaveLength(2) + await host.close() + }) + + it('gives a routed root harnessAgent the bytes of parent-thread media', async () => { + const encoder = new TextEncoder() + const model = mockAdapter([() => text('A cat.')]) + const child = defineHarness({ + name: 'test/desk', + adapter: { ...model.adapter, inputModalities: ['text', 'image'] }, + }) + const main = mockAdapter([]) + const host = createHarnessHost({ persistence: memoryPersistence() }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: { ...main.adapter, inputModalities: ['text', 'image'] }, + agents: [harnessAgent(child, { name: 'desk' })], + routing: { router: () => ({ name: 'desk' }) }, + }), + { threadId: THREAD }, + ) + const record = await session.putMedia({ + body: encoder.encode('hello'), + mimeType: 'image/png', + name: 'cat.png', + }) + await session.send([ + { type: 'text', content: 'What is this?' }, + mediaPart(record), + ]) + + const last = model.calls[0].messages.at(-1) + expect(last.role).toBe('user') + const parts = Array.isArray(last.content) ? last.content : [] + expect(parts).toEqual( + expect.arrayContaining([ + { + type: 'image', + source: { type: 'data', value: 'aGVsbG8=', mimeType: 'image/png' }, + }, + ]), + ) + expect(JSON.stringify(last.content)).not.toContain('not found') + await host.close() + }) + + it('completes a routed root harnessAgent turn when the parent blob is missing', async () => { + const encoder = new TextEncoder() + const model = mockAdapter([() => text('No image.')]) + const child = defineHarness({ + name: 'test/desk', + adapter: { ...model.adapter, inputModalities: ['text', 'image'] }, + }) + const main = mockAdapter([]) + const persistence = memoryPersistence() + const host = createHarnessHost({ persistence }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: { ...main.adapter, inputModalities: ['text', 'image'] }, + agents: [harnessAgent(child, { name: 'desk' })], + routing: { router: () => ({ name: 'desk' }) }, + }), + { threadId: THREAD }, + ) + const record = await session.putMedia({ + body: encoder.encode('hello'), + mimeType: 'image/png', + name: 'cat.png', + }) + const blobKey = (await persistence.stores.artifacts.get(record.id))?.blobKey + await persistence.stores.blobs.delete(blobKey ?? '') + await session.send([ + { type: 'text', content: 'What is this?' }, + mediaPart(record), + ]) + + expect(model.calls).toHaveLength(1) + const last = model.calls[0].messages.at(-1) + expect(last.role).toBe('user') + expect(JSON.stringify(last.content)).toContain('not found') + await host.close() + }) + + it('keeps an earlier user image URL in the conversation so far for a routed root harnessAgent', async () => { + const model = mockAdapter([() => text('A cat.')]) + const child = defineHarness({ + name: 'test/desk', + adapter: { ...model.adapter, inputModalities: ['text', 'image'] }, + }) + const main = mockAdapter([() => text('Main.')]) + const host = createHarnessHost({ persistence: memoryPersistence() }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: { ...main.adapter, inputModalities: ['text', 'image'] }, + agents: [harnessAgent(child, { name: 'desk' })], + routing: { router: picksInOrder([undefined, 'desk']) }, + }), + { threadId: THREAD }, + ) + const image = { + type: 'image', + source: { type: 'url', value: 'https://tanstack.com/cat.png' }, + } as const + await session.send([{ type: 'text', content: 'Look at this.' }, image]) + await session.send('What is in the image?') + + const last = model.calls[0].messages.at(-1) + expect(last.role).toBe('user') + const parts = Array.isArray(last.content) ? last.content : [] + expect(parts).toEqual(expect.arrayContaining([image])) + await host.close() + }) + + it('does not resend the original user message when invoked again in the same turn', async () => { + const desk = deskHarness([ + () => toolCall('handoff', { to: 'main' }), + () => text('Desk again.'), + ]) + const { host, session } = await openRouted({ + child: desk.harness, + router: () => 'desk', + mainReplies: [() => toolCall('handoff', { to: 'desk' })], + }) + + await session.send('refund please') + + expect(desk.model.calls).toHaveLength(2) + const second = desk.model.calls[1] + expect( + userTexts(second).filter((entry) => entry === 'refund please'), + ).toHaveLength(1) + expect(userTexts(second).at(-1)).not.toBe('refund please') + await host.close() + }) + + it('fails the turn when the child harness has its own routing', async () => { + const model = mockAdapter([() => text('Never.')]) + const child = defineHarness({ + name: 'test/routed-child', + adapter: model.adapter, + agents: [ + defineAgent({ + name: 'inner', + description: 'An inner agent', + run: async () => 'Inner.', + }), + ], + routing: { router: () => undefined }, + }) + const { host, session } = await openRouted({ + child, + router: () => 'desk', + }) + + await expect(session.send('hi')).rejects.toThrow( + 'A harnessAgent that is a root agent cannot have its own routing. Remove routing from "test/routed-child".', + ) + expect(model.calls).toHaveLength(0) + await host.close() + }) + + it('opens the kept child thread again on a new host', async () => { + const desk = deskHarness([ + () => toolCall('lookup', {}, 'call-lookup'), + () => text('Looked it up.'), + () => text('Second answer.'), + () => text('Third answer.'), + ]) + const kept = stores() + const before = await openRouted({ + child: desk.harness, + router: () => 'desk', + stores: kept, + }) + await before.session.send('first question') + await before.session.send('second question') + await before.host.close() + + const after = await openRouted({ + child: desk.harness, + router: () => 'desk', + stores: kept, + }) + await after.session.send('third question') + + const turn3 = desk.model.calls[3] + expect(userTexts(turn3)).toEqual([ + 'first question', + 'second question', + 'third question', + ]) + expect(calledTools(turn3)).toEqual(['lookup']) + await after.host.close() + }) +}) diff --git a/packages/ai-harness/tests/prompt-cache.test.ts b/packages/ai-harness/tests/prompt-cache.test.ts index 07c6df4b46..12ee718b7a 100644 --- a/packages/ai-harness/tests/prompt-cache.test.ts +++ b/packages/ai-harness/tests/prompt-cache.test.ts @@ -117,7 +117,9 @@ describe('prompt cache', () => { adapter: main.adapter, tools: [escalate], agents: [billing], - routing: { router: () => undefined }, + // 'text' keeps the main model's messages as they are, so the target + // has them as its prefix. 'own' folds them to text for the target. + routing: { router: () => undefined, history: 'text' }, }), { threadId: 't' }, ) diff --git a/packages/ai-harness/tests/routed-history.test.ts b/packages/ai-harness/tests/routed-history.test.ts new file mode 100644 index 0000000000..187662b864 --- /dev/null +++ b/packages/ai-harness/tests/routed-history.test.ts @@ -0,0 +1,425 @@ +import { describe, expect, it } from 'vitest' +import { historyFor, routedHistoryMiddleware } from '../src/routed-history' +import type { ModelMessage } from '@tanstack/ai' + +type Child = { name: string; subagentRunId: string } + +const CHILDREN: Record> = { + r1: [{ name: 'billing', subagentRunId: 's1' }], + r2: [{ name: 'research', subagentRunId: 's2' }], +} + +const BILLING_TRANSCRIPT: Array = [ + { id: 'task1', role: 'user', content: 'Refund order 42' }, + { + id: 'b1', + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'call_refund', + type: 'function', + function: { name: 'refund', arguments: '{"order":42}' }, + }, + ], + }, + { id: 'b2', role: 'tool', toolCallId: 'call_refund', content: 'refunded' }, + { id: 'b3', role: 'assistant', content: 'Order 42 is refunded.' }, +] + +const TRANSCRIPTS: Record> = { + s1: BILLING_TRANSCRIPT, + s2: [ + { id: 'task2', role: 'user', content: 'Find papers' }, + { id: 'c1', role: 'assistant', content: 'Found 3 papers.' }, + ], +} + +function host(runId: string, agent: string, text: string): ModelMessage { + return { + id: `assistant:${runId}`, + role: 'assistant', + content: `${agent}:\n${text}`, + metadata: { tanstack: { runId }, harness: { agent } }, + } +} + +const USER: ModelMessage = { id: 'u1', role: 'user', content: 'Hi' } +const BILLING_HOST = host('r1', 'billing', 'Order 42 is refunded.') +const RESEARCH_HOST = host('r2', 'research', 'Found 3 papers.') + +function handoffCall( + id: string, + name: string, + args: Record, +): ModelMessage { + return { + id: `m_${id}`, + role: 'assistant', + content: 'Let me pass you on.', + toolCalls: [ + { + id, + type: 'function', + function: { name, arguments: JSON.stringify(args) }, + }, + ], + } +} + +function toolResult(toolCallId: string): ModelMessage { + return { + id: `${toolCallId}_result`, + role: 'tool', + toolCallId, + content: 'Handed off to billing. billing answers next.', + } +} + +function history(options: { + messages: Array + speaker: string | null + mode: 'own' | 'all' | 'text' + handoffToolName?: string + transcripts?: Record> + children?: Record> + isThreadMessage?: (message: ModelMessage) => boolean +}) { + const transcripts = options.transcripts ?? TRANSCRIPTS + const children = options.children ?? CHILDREN + return historyFor({ + ...options, + childrenOf: (runId) => children[runId] ?? [], + transcriptOf: (subagentRunId) => + Promise.resolve(transcripts[subagentRunId] ?? []), + }) +} + +describe('historyFor', () => { + it("returns the messages unchanged in 'text' mode", async () => { + const messages = [USER, BILLING_HOST, RESEARCH_HOST] + const result = await history({ messages, speaker: 'billing', mode: 'text' }) + expect(result).toEqual([USER, BILLING_HOST, RESEARCH_HOST]) + }) + + it("gives a routed agent its own runs in full and others as text in 'own' mode", async () => { + const result = await history({ + messages: [USER, BILLING_HOST, RESEARCH_HOST], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + USER, + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + // Another agent's answer: a user message, so billing does not read + // it as its own. + { ...RESEARCH_HOST, role: 'user' }, + ]) + }) + + it("folds a main handoff call to text for a routed agent in 'own' mode", async () => { + const call = handoffCall('call_h', 'handoff', { + to: 'billing', + note: 'refund for order 42', + }) + const result = await history({ + messages: [USER, call, toolResult('call_h')], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + USER, + { + id: 'm_call_h', + role: 'user', + content: + 'main:\nLet me pass you on.\n\nmain handed off to billing. Note: refund for order 42', + }, + ]) + }) + + it('folds a handoff to a person and drops the other tool calls of main', async () => { + const call: ModelMessage = { + id: 'm2', + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'call_w', + type: 'function', + function: { name: 'weather', arguments: '{}' }, + }, + { + id: 'call_p', + type: 'function', + function: { + name: 'handoff', + arguments: JSON.stringify({ to: { human: true } }), + }, + }, + ], + } + const result = await history({ + messages: [call, toolResult('call_w'), toolResult('call_p')], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + { id: 'm2', role: 'user', content: 'main handed off to a person.' }, + ]) + }) + + it('folds a stamped recorded note instead of the model call args', async () => { + const call = { + ...handoffCall('call_h', 'handoff', { + to: 'billing', + note: 'Refund for order 42.', + }), + metadata: { + harness: { handoffNote: 'from main: Refund for order 42.' }, + }, + } + const result = await history({ + messages: [USER, call, toolResult('call_h')], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + USER, + { + id: 'm_call_h', + role: 'user', + content: + 'main:\nLet me pass you on.\n\nmain handed off to billing. Note: from main: Refund for order 42.', + metadata: { + harness: { handoffNote: 'from main: Refund for order 42.' }, + }, + }, + ]) + }) + + it('reads the handoff tool name from handoffToolName', async () => { + const custom = handoffCall('call_t', 'transfer', { to: 'billing' }) + const builtIn = handoffCall('call_b', 'handoff', { to: 'research' }) + const result = await history({ + messages: [custom, toolResult('call_t'), builtIn, toolResult('call_b')], + speaker: 'billing', + mode: 'own', + handoffToolName: 'transfer', + }) + expect(result).toEqual([ + { + id: 'm_call_t', + role: 'user', + content: 'main:\nLet me pass you on.\n\nmain handed off to billing.', + }, + { + id: 'm_call_b', + role: 'user', + content: 'main:\nLet me pass you on.', + }, + ]) + }) + + it("folds an unstamped multi-agent host as user text for the main model in 'own' mode", async () => { + const parallel: ModelMessage = { + id: 'assistant:r3', + role: 'assistant', + content: 'billing:\nRefunded\n\nshipping:\nShipped', + metadata: { tanstack: { runId: 'r3' } }, + } + const result = await history({ + messages: [USER, parallel], + speaker: null, + mode: 'own', + children: { + r3: [ + { name: 'billing', subagentRunId: 's1' }, + { name: 'shipping', subagentRunId: 's2' }, + ], + }, + }) + expect(result).toEqual([USER, { ...parallel, role: 'user' }]) + }) + + it('folds an unstamped host as user text when the run has no children', async () => { + const parallel: ModelMessage = { + id: 'assistant:r3', + role: 'assistant', + content: 'billing:\nRefunded\n\nshipping:\nShipped', + metadata: { tanstack: { runId: 'r3' } }, + } + const asUser = { ...parallel, role: 'user' as const } + const seen = async (speaker: string | null) => + history({ + messages: [USER, parallel], + speaker, + mode: 'own', + children: {}, + }) + expect(await seen(null)).toEqual([USER, asUser]) + expect(await seen('billing')).toEqual([USER, asUser]) + }) + + it("keeps the main model's own calls and gives it routed answers as user text in 'own' mode", async () => { + const call = handoffCall('call_h', 'handoff', { to: 'billing' }) + const messages = [USER, call, toolResult('call_h'), BILLING_HOST] + const result = await history({ messages, speaker: null, mode: 'own' }) + expect(result).toEqual([ + USER, + call, + toolResult('call_h'), + { ...BILLING_HOST, role: 'user' }, + ]) + }) + + it("expands every routed run for the main model in 'all' mode", async () => { + const call = handoffCall('call_h', 'handoff', { to: 'billing' }) + const result = await history({ + messages: [USER, call, toolResult('call_h'), BILLING_HOST, RESEARCH_HOST], + speaker: null, + mode: 'all', + }) + expect(result).toEqual([ + USER, + call, + toolResult('call_h'), + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + TRANSCRIPTS.s2?.[1], + ]) + }) + + it('expands a kept earlier pass of a run by its harness routedRunId', async () => { + // One run with two passes: billing's pass moved to its own id and lost + // its tanstack run id. research wrote the last pass. + const keptPass: ModelMessage = { + id: 'assistant:r3:1', + role: 'assistant', + content: 'billing:\nOrder 42 is refunded.', + metadata: { + tanstack: {}, + harness: { agent: 'billing', routedRunId: 'r3' }, + }, + } + const lastPass = host('r3', 'research', 'Found 3 papers.') + const result = await history({ + messages: [USER, keptPass, lastPass], + speaker: 'billing', + mode: 'own', + children: { + r3: [ + { name: 'billing', subagentRunId: 's1' }, + { name: 'research', subagentRunId: 's2' }, + ], + }, + }) + expect(result).toEqual([ + USER, + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + { ...lastPass, role: 'user' }, + ]) + }) + + it('keeps the messages that are not in the thread as they are', async () => { + const ownCall = handoffCall('call_own', 'lookup', {}) + const result = await history({ + messages: [USER, ownCall, toolResult('call_own')], + speaker: 'billing', + mode: 'own', + isThreadMessage: (message) => message.id === USER.id, + }) + expect(result).toEqual([USER, ownCall, toolResult('call_own')]) + }) + + it('keeps the host message when the run has no stored transcript', async () => { + const result = await history({ + messages: [USER, BILLING_HOST], + speaker: 'billing', + mode: 'own', + transcripts: {}, + }) + expect(result).toEqual([USER, BILLING_HOST]) + }) + + it('folds a router note pair of another agent to one text line', async () => { + const metadata = { + harness: { agent: 'billing', handoff: { by: 'router' } }, + } + const note: ModelMessage = { + id: 'n1', + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'n1', + type: 'function', + function: { + name: 'handoff', + arguments: JSON.stringify({ to: 'main', note: 'needs a human' }), + }, + }, + ], + metadata, + } + const result = await history({ + messages: [USER, note, { ...toolResult('n1'), metadata }], + speaker: 'research', + mode: 'own', + }) + expect(result).toEqual([ + USER, + { + id: 'n1', + role: 'user', + content: 'billing handed off to main. Note: needs a human', + metadata, + }, + ]) + }) +}) + +describe('routedHistoryMiddleware', () => { + const middleware = routedHistoryMiddleware({ + mode: 'own', + childrenOf: (runId) => CHILDREN[runId] ?? [], + transcriptOf: (subagentRunId) => + Promise.resolve(TRANSCRIPTS[subagentRunId] ?? []), + }) + const config = { messages: [USER, BILLING_HOST] } + + it('sets providerMessages for a routed agent', async () => { + const change = await middleware.onConfig( + { subagentName: 'billing' }, + config, + ) + expect(change).toEqual({ + providerMessages: [ + USER, + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + ], + }) + }) + + it("gives the main model a routed answer as user text in 'own' mode", async () => { + const change = await middleware.onConfig({}, config) + expect(change).toEqual({ + providerMessages: [USER, { ...BILLING_HOST, role: 'user' }], + }) + }) + + it('changes nothing for a nested child of a routed agent', async () => { + const change = await middleware.onConfig( + { subagentName: 'billing', parentSubagentRunId: 's1' }, + config, + ) + expect(change).toBeUndefined() + }) +}) diff --git a/packages/ai-harness/tests/routing-history.test.ts b/packages/ai-harness/tests/routing-history.test.ts new file mode 100644 index 0000000000..270e8844d3 --- /dev/null +++ b/packages/ai-harness/tests/routing-history.test.ts @@ -0,0 +1,393 @@ +import { memoryCredentialStore } from '@tanstack/ai/byok/server' +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { memoryLogStore, memoryPersistence } from '@tanstack/ai-persistence' +import { createHarnessHost, defineHarness } from '../src' +import { messageTexts, mockAdapter, text, toolCall } from './helpers' +import type { AnyTool, ModelMessage } from '@tanstack/ai' +import type { AnyAgent, HarnessRouting } from '../src' +import type { Reply } from './helpers' + +/** A session whose harness routes, with `routing.history`. */ +async function openRouting(options: { + agents: ReadonlyArray + router: HarnessRouting['router'] + history?: HarnessRouting['history'] + replies?: Array +}) { + const { metadata } = memoryPersistence().stores + const main = mockAdapter(options.replies ?? []) + const host = createHarnessHost({ + persistence: { stores: { log: memoryLogStore(), metadata } }, + credentials: memoryCredentialStore(), + }) + const session = await host.open( + defineHarness({ + name: 'test/routing-history', + adapter: main.adapter, + agents: options.agents, + routing: { + router: options.router, + ...(options.history ? { history: options.history } : {}), + }, + }), + { threadId: 't1', user: { id: 'user-1' } }, + ) + return { host, session, main } +} + +/** An agent that answers with its own model and tools. */ +function modelAgent( + name: string, + replies: Array, + tools: ReadonlyArray = [], +) { + const model = mockAdapter(replies) + const agent = defineAgent({ + name, + description: `The ${name} desk`, + run: (ctx) => ctx.chat({ adapter: model.adapter, tools: [...tools] }), + }) + return { agent, model } +} + +/** A tool that answers `ok`. */ +function plainTool(name: string) { + return toolDefinition({ + name, + description: `The ${name} tool`, + inputSchema: z.object({}), + }).server(async () => 'ok') +} + +/** A router that picks an agent for some inputs, else the active agent. */ +const routeBy = + (picks: Record): HarnessRouting['router'] => + ({ input }) => + typeof input === 'string' ? picks[input] : undefined + +/** The names of the tool calls in the messages of one model call. */ +function calledTools(call: { messages: Array }) { + return call.messages.flatMap((message) => + (message.toolCalls ?? []).map((made) => made.function.name), + ) +} + +/** The tool call ids that have a tool result in one model call. */ +function answeredCalls(call: { messages: Array }) { + return call.messages.flatMap((message) => + message.role === 'tool' && message.toolCallId !== undefined + ? [message.toolCallId] + : [], + ) +} + +/** billing calls `refund`, then answers. A later turn answers `Welcome`. */ +function refundingBilling() { + return modelAgent( + 'billing', + [ + () => toolCall('refund', {}, 'call_r'), + () => text('Refunded'), + () => text('Welcome'), + ], + [plainTool('refund')], + ) +} + +describe('routing.history', () => { + it("'own' gives a routed agent its own earlier tool calls", async () => { + const billing = refundingBilling() + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing' }), + }) + + await session.send('refund please') + expect((await session.send('thanks')).text).toBe('billing:\nWelcome') + + const later = billing.model.calls[2] + expect(calledTools(later)).toContain('refund') + expect(answeredCalls(later)).toContain('call_r') + expect(messageTexts(later)).toContain('Refunded') + expect(messageTexts(later)).not.toContain('billing:\nRefunded') + await host.close() + }) + + it("'own' gives the main model a routed run as text only", async () => { + const billing = refundingBilling() + const { host, session, main } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing', 'ask main': 'main' }), + replies: [() => text('Main here')], + }) + + await session.send('refund please') + expect((await session.send('ask main')).text).toBe('Main here') + + const [call] = main.calls + expect(calledTools(call)).not.toContain('refund') + expect(messageTexts(call)).toContain('billing:\nRefunded') + await host.close() + }) + + it("'all' gives the main model the tool calls of a routed run", async () => { + const billing = refundingBilling() + const { host, session, main } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing', 'ask main': 'main' }), + history: 'all', + replies: [() => text('Main here')], + }) + + await session.send('refund please') + await session.send('ask main') + + const [call] = main.calls + expect(calledTools(call)).toContain('refund') + expect(answeredCalls(call)).toContain('call_r') + expect(messageTexts(call)).not.toContain('billing:\nRefunded') + await host.close() + }) + + it("'text' gives a routed agent its earlier run as text only", async () => { + const billing = refundingBilling() + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing' }), + history: 'text', + }) + + await session.send('refund please') + await session.send('thanks') + + const later = billing.model.calls[2] + expect(calledTools(later)).not.toContain('refund') + expect(messageTexts(later)).toContain('billing:\nRefunded') + await host.close() + }) + + it("'own' gives the main model a parallel run as user text, not assistant", async () => { + const billing = modelAgent('billing', [() => text('Refunded')]) + const shipping = modelAgent('shipping', [() => text('Shipped')]) + const { host, session, main } = await openRouting({ + agents: [billing.agent, shipping.agent], + router: ({ input }) => + input === 'both' + ? { names: ['billing', 'shipping'], order: 'parallel' } + : 'main', + replies: [() => text('Main here')], + }) + + await session.send('both') + expect((await session.send('ask main')).text).toBe('Main here') + + const [call] = main.calls + const labeled = call.messages.filter( + (message: ModelMessage) => + typeof message.content === 'string' && + message.content.includes('billing:') && + message.content.includes('shipping:'), + ) + expect(labeled.length).toBeGreaterThan(0) + expect( + labeled.every((message: ModelMessage) => message.role === 'user'), + ).toBe(true) + expect( + call.messages.some( + (message: ModelMessage) => + message.role === 'assistant' && + typeof message.content === 'string' && + (message.content.includes('billing:') || + message.content.includes('Refunded')), + ), + ).toBe(false) + await host.close() + }) + + it("'own' gives a routed agent the handoff of main as a text line", async () => { + const billing = modelAgent('billing', [ + () => text('Refunded'), + () => text('Welcome'), + ]) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: () => undefined, + replies: [ + () => + toolCall('handoff', { to: 'billing', note: 'order 42' }, 'call_h'), + ], + }) + + expect((await session.send('refund please')).text).toBe( + 'billing:\nRefunded', + ) + await session.send('thanks') + + const later = billing.model.calls[1] + expect(answeredCalls(later)).not.toContain('call_h') + expect(messageTexts(later)).toContain( + 'main handed off to billing. Note: order 42', + ) + await host.close() + }) + + it("'own' folds a root agent's handoff note so a later turn keeps only its real handoff", async () => { + const billing = modelAgent('billing', [ + () => [ + ...text('Refunded'), + ...toolCall('handoff', { to: 'main' }, 'call_hb'), + ], + () => text('Welcome'), + ]) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ + 'refund please': 'billing', + thanks: 'billing', + }), + replies: [() => text('Main here')], + }) + + expect((await session.send('refund please')).text).toBe('Main here') + await session.send('thanks') + + const later = billing.model.calls[1] + expect(calledTools(later).filter((name) => name === 'handoff')).toEqual([ + 'handoff', + ]) + expect(answeredCalls(later).filter((id) => id === 'call_hb')).toEqual([ + 'call_hb', + ]) + expect( + later.messages.some( + (message: ModelMessage) => + message.role === 'assistant' && + (message.id?.startsWith('handoff_') || + (message.toolCalls ?? []).some((call) => + call.id.startsWith('handoff_'), + )), + ), + ).toBe(false) + await host.close() + }) + + it('gives each pass of a turn with two routed passes to its own agent', async () => { + const billing = modelAgent( + 'billing', + [ + () => toolCall('refund', {}, 'call_r'), + () => [ + ...text('Passing you to research.'), + ...toolCall('handoff', { to: 'research' }, 'call_hb'), + ], + () => text('Back in billing'), + ], + [plainTool('refund')], + ) + const research = modelAgent( + 'research', + [() => toolCall('search', {}, 'call_s'), () => text('Found 3 papers.')], + [plainTool('search')], + ) + const { host, session } = await openRouting({ + agents: [billing.agent, research.agent], + router: routeBy({ + 'refund please': 'billing', + 'back to billing': 'billing', + }), + }) + + expect((await session.send('refund please')).text).toBe( + 'research:\nFound 3 papers.', + ) + await session.send('back to billing') + + const later = billing.model.calls[2] + expect(calledTools(later)).toContain('refund') + expect(calledTools(later)).not.toContain('search') + expect(answeredCalls(later)).toEqual( + expect.arrayContaining(['call_r', 'call_hb']), + ) + expect(messageTexts(later)).toContain('research:\nFound 3 papers.') + await host.close() + }) + + it('gives a later billing turn each tool call once after billing → main → billing in one turn', async () => { + const billing = modelAgent( + 'billing', + [ + () => toolCall('refund', {}, 'call_r'), + () => [ + ...text('Refunded'), + ...toolCall('handoff', { to: 'main' }, 'call_hb'), + ], + () => toolCall('lookup', {}, 'call_l'), + () => text('Looked up.'), + () => text('Welcome'), + ], + [plainTool('refund'), plainTool('lookup')], + ) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ + 'refund please': 'billing', + thanks: 'billing', + }), + replies: [() => toolCall('handoff', { to: 'billing' }, 'call_hm')], + }) + + expect((await session.send('refund please')).text).toBe( + 'billing:\nLooked up.', + ) + await session.send('thanks') + + const later = billing.model.calls[4] + expect(calledTools(later).filter((name) => name === 'refund')).toEqual([ + 'refund', + ]) + expect(calledTools(later).filter((name) => name === 'lookup')).toEqual([ + 'lookup', + ]) + expect(answeredCalls(later).filter((id) => id === 'call_r')).toHaveLength(1) + expect(answeredCalls(later).filter((id) => id === 'call_l')).toHaveLength(1) + await host.close() + }) + + it('gives a resumed routed agent the messages of its paused run', async () => { + // No server implementation: the turn waits for the result of the client. + const openScreen = toolDefinition({ + name: 'openScreen', + description: 'Open a screen', + inputSchema: z.object({}), + }) + const billing = modelAgent( + 'billing', + [() => toolCall('openScreen', {}, 'call_o'), () => text('Screen open')], + [openScreen], + ) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'open it': 'billing' }), + }) + + const [interrupt] = (await session.send('open it')).interrupts ?? [] + if (!interrupt) throw new Error('No client tool interrupt.') + await session.resolve({ + resume: [{ interruptId: interrupt.id, status: 'resolved', payload: {} }], + inputId: 'opened', + }) + expect(await session.settled({ inputId: 'opened' })).toMatchObject({ + outcome: 'completed', + }) + + const resumed = billing.model.calls[1] + const opened = calledTools(resumed).filter((name) => name === 'openScreen') + expect(opened).toEqual(['openScreen']) + const answered = answeredCalls(resumed).filter((id) => id === 'call_o') + expect(answered).toEqual(['call_o']) + expect(messageTexts(resumed)).toContain('open it') + await host.close() + }) +}) diff --git a/packages/ai-harness/tests/routing.test.ts b/packages/ai-harness/tests/routing.test.ts index b9f9eeb54b..2cb146c1c9 100644 --- a/packages/ai-harness/tests/routing.test.ts +++ b/packages/ai-harness/tests/routing.test.ts @@ -355,7 +355,7 @@ describe('routing.router', () => { expect(main.calls).toHaveLength(1) expect(messageTexts(main.calls[0])).toEqual([ 'write it', - ...routedTo('writer'), + 'main handed off to writer.', 'writer:\nDraft', ]) expect( diff --git a/packages/ai-harness/tests/workspace-tools.test.ts b/packages/ai-harness/tests/workspace-tools.test.ts index 31d2017a1a..2198915b5b 100644 --- a/packages/ai-harness/tests/workspace-tools.test.ts +++ b/packages/ai-harness/tests/workspace-tools.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs' import { mkdtemp, readFile, realpath, rm, symlink } from 'node:fs/promises' import { tmpdir } from 'node:os' import * as nodePath from 'node:path' @@ -173,10 +174,18 @@ const toolNames = (call: { tools: Array<{ name: string }> }) => function isRunning(pid: number) { try { process.kill(pid, 0) - return true } catch { return false } + // A zombie still answers `kill(pid, 0)`. In a CI container whose PID 1 + // never reaps it (TanStack Source), a killed child stays a zombie, so read + // its state on Linux. With no /proc, the process counts as running. + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8') + return !stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z') + } catch { + return true + } } describe.each(fixtures)('workspace tools on the $name backend', (fixture) => { diff --git a/testing/e2e/src/routes/api.harness-handoff.$.ts b/testing/e2e/src/routes/api.harness-handoff.$.ts index b80998cc15..55b067f7ab 100644 --- a/testing/e2e/src/routes/api.harness-handoff.$.ts +++ b/testing/e2e/src/routes/api.harness-handoff.$.ts @@ -1,5 +1,6 @@ import { createFileRoute } from '@tanstack/react-router' -import { defineAgent } from '@tanstack/ai' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' import { createAnthropicChat } from '@tanstack/ai-anthropic' import { HARNESS_EVENTS, @@ -12,6 +13,7 @@ import { createSessionView } from '@tanstack/ai-harness/view' import { memoryPersistence } from '@tanstack/ai-persistence' import type { HarnessHost, + HarnessRouting, HarnessSession, HarnessUser, } from '@tanstack/ai-harness' @@ -19,8 +21,8 @@ import type { SessionView } from '@tanstack/ai-harness/view' /** * Handoffs inside one session, over the harness protocol. The harness has - * `routing` and one root agent, `billing`. The host has `membersOnly()` - * access: olga owns the thread, and bea is an operator. + * `routing` and one root agent, `billing`, with a `refund` tool. The host + * has `membersOnly()` access: olga owns the thread, and bea is an operator. * * - `POST .../start` with `{ scenario }` opens the thread for olga and * shares it with bea. Returns the run id and the thread id. @@ -29,6 +31,10 @@ import type { SessionView } from '@tanstack/ai-harness/view' * the `handoff` tool to a person. * - `'code'`: the router picks the `triage` agent, and its code hands off * to `billing` with `ctx.harness.handoff`. The main model never runs. + * - `'history'`: the main model hands off to `billing`. On turn 1, + * `billing` calls `refund`, then answers. On turn 2, `billing` answers. + * - `'custom-tool'`: the harness has `routing.handoffTool`. Its tool adds + * `from : ` before the note, and the main model calls it. * - `POST .../control?runId=` and `GET .../snapshot?runId=` go to the * harness handler. The `authorization` header names the user. * - `GET .../settled?runId=&inputId=` waits until an input ends, and @@ -36,7 +42,7 @@ import type { SessionView } from '@tanstack/ai-harness/view' * - `GET .../debug?runId=` returns what the test checks: the * `harness.handoff` events of the session stream, the transcript, the * messages of a session view, the open waits, the model calls, and the - * messages of the first request of `billing`. + * messages of each request of `billing`. * - `POST .../stop?runId=` closes the host. * * A custom `fetch` answers with canned Anthropic streams, so no aimock @@ -78,27 +84,31 @@ const ending = (stopReason: string): Array => [ { type: 'message_stop' }, ] -/** An answer that calls the `handoff` tool with `input`. */ -const callHandoff = (input: Record) => (id: string) => - sseOf([ - messageStart(id), - { - type: 'content_block_start', - index: 0, - content_block: { - type: 'tool_use', - id: `toolu_${id}`, - name: 'handoff', - input: {}, +/** An answer that calls the tool `name` with `input`. */ +const callTool = + (name: string, input: Record) => (id: string) => + sseOf([ + messageStart(id), + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: `toolu_${id}`, + name, + input: {}, + }, }, - }, - { - type: 'content_block_delta', - index: 0, - delta: { type: 'input_json_delta', partial_json: JSON.stringify(input) }, - }, - ...ending('tool_use'), - ]) + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'input_json_delta', + partial_json: JSON.stringify(input), + }, + }, + ...ending('tool_use'), + ]) /** An answer with the text `text`. */ const reply = (text: string) => (id: string) => @@ -138,7 +148,7 @@ function scriptedModel(prefix: string, answers: Array<(id: string) => string>) { return { adapter, counter, requests } } -type Scenario = 'agent' | 'human' | 'code' +type Scenario = 'agent' | 'human' | 'code' | 'history' | 'custom-tool' /** The answers of the main model and of `billing`, per scenario. */ const SCRIPTS: Record< @@ -149,12 +159,17 @@ const SCRIPTS: Record< } > = { agent: { - main: [callHandoff({ to: 'billing', note: 'Refund for order 42.' })], + main: [ + callTool('handoff', { to: 'billing', note: 'Refund for order 42.' }), + ], billing: [reply('Your refund is on its way.'), reply('Billing again.')], }, human: { main: [ - callHandoff({ to: { human: true }, note: 'The user asks for a person.' }), + callTool('handoff', { + to: { human: true }, + note: 'The user asks for a person.', + }), reply('The main model is back.'), ], billing: [], @@ -163,8 +178,48 @@ const SCRIPTS: Record< main: [], billing: [reply('Your refund is on its way.')], }, + history: { + main: [ + callTool('handoff', { to: 'billing', note: 'Refund for order 42.' }), + ], + billing: [ + callTool('refund', { orderId: '42' }), + reply('Refunded order 42.'), + reply('Anything else?'), + ], + }, + 'custom-tool': { + main: [ + callTool('handoff', { to: 'billing', note: 'Refund for order 42.' }), + ], + billing: [reply('Your refund is on its way.')], + }, } +/** The tool of `billing`. It always gives the same result. */ +const refund = toolDefinition({ + name: 'refund', + description: 'Refund an order', + inputSchema: z.object({ orderId: z.string() }), +}).server(async ({ orderId }) => `Refund R-${orderId} issued.`) + +/** + * The handoff tool of the `'custom-tool'` harness. It adds the speaker + * before the note, and gives a fixed result. + */ +const prefixedHandoff: NonNullable = ({ + from, + handoff, +}) => + toolDefinition({ + name: 'handoff', + description: 'Hand the user to another agent', + inputSchema: z.object({ to: z.string(), note: z.string() }), + }).server(async ({ to, note }) => { + await handoff({ to, note: `from ${from ?? 'main'}: ${note}` }) + return 'Passed to the next desk.' + }) + /** An agent with no model: its code hands the turn to `billing`. */ const triage = defineAgent({ name: 'triage', @@ -221,6 +276,7 @@ async function start(request: Request) { ctx.chat({ adapter: billing.adapter, systemPrompts: ['You are billing.'], + tools: [refund], }), }), ], @@ -229,6 +285,7 @@ async function start(request: Request) { routing: { router: () => (scenario === 'code' ? 'triage' : undefined), allowHandoffToHuman: scenario === 'human', + ...(scenario === 'custom-tool' ? { handoffTool: prefixedHandoff } : {}), }, }) const host: HarnessHost = createHarnessHost({ @@ -309,7 +366,9 @@ async function debug(run: Run) { waits: snapshot.pendingWaits, mainCalls: run.main.counter.calls, billingCalls: run.billing.counter.calls, - billingMessages: run.billing.requests[0]?.messages ?? [], + billingRequests: run.billing.requests.map( + (request) => request.messages ?? [], + ), }) } diff --git a/testing/e2e/tests/harness-handoff.spec.ts b/testing/e2e/tests/harness-handoff.spec.ts index 4f4971cd23..d570559f40 100644 --- a/testing/e2e/tests/harness-handoff.spec.ts +++ b/testing/e2e/tests/harness-handoff.spec.ts @@ -9,6 +9,18 @@ interface Started { threadId: string } +/** A content block of an Anthropic message. */ +interface Block { + type: string + id?: string + name?: string + input?: unknown + tool_use_id?: string +} + +/** The messages of one model request, as Anthropic gets them. */ +type RequestMessages = Array<{ role: string; content: string | Array }> + interface Debug { error?: string status: string @@ -22,15 +34,14 @@ interface Debug { waits: Array<{ waitId: string; kind: string; message?: string }> mainCalls: number billingCalls: number - /** The messages of the first request of `billing`, as Anthropic gets them. */ - billingMessages: Array<{ role: string; content: unknown }> + /** The messages of each request of `billing`, in order. */ + billingRequests: Array } +type Scenario = 'agent' | 'human' | 'code' | 'history' | 'custom-tool' + /** Open the thread of `scenario` for olga, shared with bea. */ -async function start( - request: APIRequestContext, - scenario: 'agent' | 'human' | 'code', -) { +async function start(request: APIRequestContext, scenario: Scenario) { const res = await request.post(`${ROUTE}/start`, { data: { scenario } }) expect(res.ok()).toBe(true) const started: Started = await res.json() @@ -110,6 +121,12 @@ const lines = (read: Debug) => by: harness?.agent ?? harness?.operator ?? null, })) +/** The content blocks of the messages of one request. */ +const blocksOf = (messages: RequestMessages) => + messages.flatMap(({ content }) => + typeof content === 'string' ? [] : content, + ) + const stop = (request: APIRequestContext, runId: string) => request.post(`${ROUTE}/stop?runId=${runId}`) @@ -214,23 +231,20 @@ test.describe('harness: handoffs inside one session', () => { ]) expect(read.mainCalls).toBe(0) expect(read.billingCalls).toBe(1) - // The model of billing gets the note of its handoff last: a `handoff` - // tool call with the note, and its result. - const [noteCall, noteResult] = read.billingMessages.slice(-2) - expect(noteCall).toMatchObject({ - role: 'assistant', - content: expect.arrayContaining([ - expect.objectContaining({ - type: 'tool_use', - name: 'handoff', - input: { to: 'billing', note: 'Refund for order 42.' }, - }), - ]), - }) - expect(noteResult).toMatchObject({ role: 'user' }) - expect(JSON.stringify(noteResult)).toContain( - 'Handed off to billing. billing answers next.', + // The model of billing gets the note of its handoff last, as text of + // another speaker (`routing.history: 'own'`): a user message, with no + // `handoff` call that billing could read as its own. + const sentToBilling = read.billingRequests[0]! + const noteMessage = sentToBilling.at(-1) + expect(noteMessage).toMatchObject({ role: 'user' }) + expect(JSON.stringify(noteMessage)).toContain( + 'triage handed off to billing. Note: Refund for order 42.', ) + expect( + blocksOf(sentToBilling).some( + (block) => block.type === 'tool_use' && block.name === 'handoff', + ), + ).toBe(false) expect(lines(read).at(-1)).toEqual({ role: 'assistant', text: 'billing:\nYour refund is on its way.', @@ -352,4 +366,107 @@ test.describe('harness: handoffs inside one session', () => { await stop(request, started.runId) } }) + + test('a routed agent sees its own tool calls of an earlier turn', async ({ + request, + }) => { + const started = await start(request, 'history') + try { + await control(request, started, { + op: 'prompt', + message: 'refund please', + inputId: 'in-1', + }) + expect(await outcomeOf(request, started.runId, 'in-1')).toBe('completed') + await control(request, started, { + op: 'prompt', + message: 'and now?', + inputId: 'in-2', + }) + expect(await outcomeOf(request, started.runId, 'in-2')).toBe('completed') + + const read = await readUntil( + request, + started.runId, + (done) => done.billingCalls === 3, + ) + expect(read.mainCalls).toBe(1) + expect(lines(read).slice(3)).toEqual([ + { + role: 'assistant', + text: 'billing:\nRefunded order 42.', + by: 'billing', + }, + { role: 'user', text: 'and now?', by: null }, + { role: 'assistant', text: 'billing:\nAnything else?', by: 'billing' }, + ]) + + // On turn 2, billing gets its turn 1 `refund` call and its result. + const later = read.billingRequests[2]! + const blocks = blocksOf(later) + const refundCall = blocks.find( + (block) => block.type === 'tool_use' && block.name === 'refund', + ) + expect(refundCall).toMatchObject({ input: { orderId: '42' } }) + const refundResult = blocks.find( + (block) => + block.type === 'tool_result' && block.tool_use_id === refundCall?.id, + ) + expect(JSON.stringify(refundResult)).toContain('Refund R-42 issued.') + // Its own run is not the `billing:` text of the host message. The + // handoff call of main is a text line with the note. JSON writes the + // newline as a backslash and `n`. + const sent = JSON.stringify(later) + expect(sent).not.toContain('billing:\\nRefunded order 42.') + expect(sent).toContain( + 'main handed off to billing. Note: Refund for order 42.', + ) + } finally { + await stop(request, started.runId) + } + }) + + test('a handoff tool from routing.handoffTool hands off with its own note', async ({ + request, + }) => { + const started = await start(request, 'custom-tool') + try { + await control(request, started, { + op: 'prompt', + message: 'refund please', + inputId: 'in-1', + }) + expect(await outcomeOf(request, started.runId, 'in-1')).toBe('completed') + + const read = await readUntil(request, started.runId, (done) => + done.view.some((message) => message.agent === 'billing'), + ) + expect(read.handoffs).toEqual([ + { + from: null, + to: 'billing', + note: 'from main: Refund for order 42.', + by: 'tool', + runId: expect.any(String), + }, + ]) + expect(lines(read)).toEqual([ + { role: 'user', text: 'refund please', by: null }, + { role: 'assistant', text: '', by: null }, + { role: 'tool', text: 'Passed to the next desk.', by: null }, + { + role: 'assistant', + text: 'billing:\nYour refund is on its way.', + by: 'billing', + }, + ]) + expect(read.mainCalls).toBe(1) + expect(read.billingCalls).toBe(1) + // billing gets the note once: from the tool call, with no extra note. + const sent = JSON.stringify(read.billingRequests[0]) + expect(sent.split('from main: Refund for order 42.')).toHaveLength(2) + } finally { + await stop(request, started.runId) + } + }) })