From 46cd641b89ba38609454aad26fca4c2c025927a2 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Thu, 12 Nov 2026 15:10:27 +0100 Subject: [PATCH 01/11] feat(ai-harness): keep routed agents' tool calls, root harnessAgent, routing.handoffTool - routing.history ('own' | 'all' | 'text', default 'own'): a routed agent gets its own earlier runs in full, with tool calls and results. The words of other speakers reach each model as labeled user text. - A harnessAgent in agents with routing keeps one child session across turns, gets what happened since its last turn, and can hand back. - routing.handoffTool makes the handoff tool of each speaker. An app tool with the handoff tool's name replaces the harness tool. - A handoff from a root agent's handoff tool adds the note messages to the shared thread, so the next speaker sees it. --- .agent/domain-map.json | 6 +- .changeset/routed-agent-history.md | 15 + docs/config.json | 4 +- docs/harness/handoffs.md | 70 +++- docs/harness/subagents.md | 66 +++- packages/ai-harness/src/define.ts | 55 +++ packages/ai-harness/src/handoff.ts | 185 ++++++--- packages/ai-harness/src/harness-agent.ts | 25 +- packages/ai-harness/src/index.ts | 1 + packages/ai-harness/src/routed-history.ts | 277 ++++++++++++++ packages/ai-harness/src/session/agents.ts | 111 +++++- packages/ai-harness/src/session/turn.ts | 111 ++++-- packages/ai-harness/src/session/types.ts | 19 + .../ai-harness/tests/handoff-tool.test.ts | 226 +++++++++++ packages/ai-harness/tests/handoff.test.ts | 76 +++- .../tests/harness-agent-root.test.ts | 260 +++++++++++++ .../ai-harness/tests/prompt-cache.test.ts | 4 +- .../ai-harness/tests/routed-history.test.ts | 356 ++++++++++++++++++ .../ai-harness/tests/routing-history.test.ts | 274 ++++++++++++++ .../e2e/src/routes/api.harness-handoff.$.ts | 115 ++++-- testing/e2e/tests/harness-handoff.spec.ts | 161 ++++++-- 21 files changed, 2245 insertions(+), 172 deletions(-) create mode 100644 .changeset/routed-agent-history.md create mode 100644 packages/ai-harness/src/routed-history.ts create mode 100644 packages/ai-harness/tests/handoff-tool.test.ts create mode 100644 packages/ai-harness/tests/harness-agent-root.test.ts create mode 100644 packages/ai-harness/tests/routed-history.test.ts create mode 100644 packages/ai-harness/tests/routing-history.test.ts diff --git a/.agent/domain-map.json b/.agent/domain-map.json index 9ba603e418..1da5026ded 100644 --- a/.agent/domain-map.json +++ b/.agent/domain-map.json @@ -44,9 +44,11 @@ "packages/ai-claude-code", "packages/ai-codex", "packages/ai-opencode", - "packages/ai-grok-build" + "packages/ai-grok-build", + "packages/ai-harness", + "packages/ai-harness-cli" ], - "talksTo": ["core", "sandbox"] + "talksTo": ["core", "sandbox", "persistence"] }, { "id": "client", diff --git a/.changeset/routed-agent-history.md b/.changeset/routed-agent-history.md new file mode 100644 index 0000000000..9d52032223 --- /dev/null +++ b/.changeset/routed-agent-history.md @@ -0,0 +1,15 @@ +--- +'@tanstack/ai-harness': minor +--- + +Routed agents now remember their own tool calls, you can pass your own handoff tool, and a `harnessAgent` can be a root agent. + +- `routing.history` sets what each model call gets of the routed runs of earlier turns. With `'own'` (the default), a routed agent gets its own earlier runs in full, with tool calls and results. It gets the other agents and the main model as labeled text in a user message, for example `main:\n...`. The main model gets the routed answers the same way, so no model reads the words of another speaker as its own. `'all'` gives every speaker every routed run in full. `'text'` gives routed runs as text only, as before. +- `routing.handoffTool` makes the handoff tool of each speaker. It gets `from`, `targets`, `allowHuman`, and `handoff`. The tool can have any name and its own fields. +- A `harnessAgent` in `agents` with `routing` keeps one child session across turns. Each turn sends it what happened in the thread since its last turn. Its model gets the `handoff` tool, so it can hand back. + +The default `history: 'own'` changes the messages that routed agents and handoff targets get. A target gets the turns of the main model as labeled text. To keep the earlier messages, and the prompt cache of the provider for a handoff target, set `routing: { router, history: 'text' }`. + +An app tool with the name of the handoff tool now replaces the harness tool. Before, the turn failed with a duplicate tool error. + +A handoff from the `handoff` tool of a root agent now adds the note messages to the thread, as a router or code handoff does. Before, the next speaker did not see that the agent handed off. On models that reject a context that ends on an assistant message (new Claude models), the main model failed after a hand-back. diff --git a/docs/config.json b/docs/config.json index 3cc74c896a..f2022b3e60 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1312,13 +1312,13 @@ "label": "Run agents from a harness", "to": "harness/subagents", "addedAt": "2026-10-13", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-12" }, { "label": "Hand off a conversation", "to": "harness/handoffs", "addedAt": "2026-10-18", - "updatedAt": "2026-10-20" + "updatedAt": "2026-11-12" }, { "label": "Add skills", diff --git a/docs/harness/handoffs.md b/docs/harness/handoffs.md index 0a4bd2b7d0..874a5c8373 100644 --- a/docs/harness/handoffs.md +++ b/docs/harness/handoffs.md @@ -58,10 +58,62 @@ Send `I want a refund for order 42.` The main model calls its `handoff` tool wit What the handoff does: - The log keeps the active agent. Later turns go to billing until it hands off again, or until the router picks another agent. -- The main model and each root agent get a `handoff` tool. Its `to` lists the root agents. A root agent can also hand back to `'main'`, the main model. +- The main model and each root agent get a `handoff` tool. Its `to` lists the root agents. A root agent can also hand back to `'main'`, the main model. A whole harness in `agents` can hand back too. See [Make a harness a root agent](./subagents#make-a-harness-a-root-agent). - One turn can hand off 10 times at most. - An agent that is only in `subagents` cannot be a target. A child that an agent starts cannot hand off either. -- If the app has its own tool named `handoff`, the turn fails with a duplicate tool error. Set `routing: { router, giveModelsHandoffTool: false }` to give no model the `handoff` tool. The router and your code can still hand off. +- An app tool named `handoff` replaces the harness tool. See [Use your own handoff tool](#use-your-own-handoff-tool). To give no model the `handoff` tool, set `routing: { router, giveModelsHandoffTool: false }`. The router and your code can still hand off. + +### Use your own handoff tool + +The `handoff` tool takes only `to` and a short `note`. The target then often asks the user again for facts that the first agent had. Pass `routing.handoffTool` to set the description, the fields, and the note of the tool: + +```ts +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { defineHarness } from '@tanstack/ai-harness' +import { openaiText } from '@tanstack/ai-openai' +import { z } from 'zod' + +const billing = defineAgent({ + name: 'billing', + description: 'Handles refunds, invoices, and payments', + run: (ctx) => + ctx.chat({ + adapter: openaiText('gpt-6.1-sol'), + systemPrompts: ['You handle refunds, invoices, and payments.'], + }), +}) + +export const support = defineHarness({ + name: 'acme/support', + adapter: openaiText('gpt-6.1-sol'), + agents: [billing], + routing: { + router: () => undefined, + handoffTool: ({ from, targets, handoff }) => + toolDefinition({ + name: 'handoff', + description: 'Hand the user to another agent. Say what you did and what the user still needs.', + inputSchema: z.object({ + to: z.enum(targets), + done: z.string().describe('What you did for the user'), + next: z.string().describe('What the user still needs'), + orderId: z.string().optional(), + }), + }).server(async ({ to, done, next, orderId }) => { + const order = orderId ? ` Order: ${orderId}.` : '' + await handoff({ to, note: `From ${from ?? 'main'}. Done: ${done}. Next: ${next}.${order}` }) + return `Handed off to ${to}.` + }), + }, +}) +``` + +- The harness calls `handoffTool` once per turn for each speaker: the main model (`from: null`) and each root agent. +- `targets` holds the names that this speaker can hand off to. `allowHuman` is the value of `routing.allowHandoffToHuman`. +- `handoff({ to, note })` hands off. The tool call is the record in the transcript, so the harness adds no note messages. +- The tool can have any name. With `giveModelsHandoffTool: false`, the harness does not call `handoffTool`. + +To change the tool of one speaker only, give that speaker an app tool named `handoff`. It replaces the harness tool. Its code hands off with `ctx.context.run.handoff` (main model) or `ctx.harness.handoff` (root agent), as in [Hand off from code](#2-hand-off-from-code). ## 2. Hand off from code @@ -105,16 +157,22 @@ The router can hand off too. Return `{ handoff: 'billing', note }` from `routing ## What the target gets - Its own system prompts and tools. The tools of the agent that handed off do not go with it. -- The saved transcript of the thread, the same messages that the members see. +- The saved transcript of the thread. `routing.history` sets how it sees the turns of the other speakers. See [Choose what later turns see](./subagents#choose-what-later-turns-see). - The note, in the transcript, in the log, and in the `harness.handoff` event. - The provider keys of the user who sent the turn. -The `handoff` tool call of a model is in the transcript already. A handoff from the router or from code adds the same two messages before the target answers: +The `handoff` tool call of the main model is in the transcript already. A root agent keeps its own tool calls in its own transcript. So a handoff from a root agent, from the router, or from code adds the same two messages before the target answers: - An assistant message with a `handoff` tool call. Its arguments are `{ to, note }`. - The tool result, for example `Handed off to billing. billing answers next.` -Both messages have `metadata.harness.handoff.by` (`'router'` or `'code'`). When an agent handed off, they also have its name in `metadata.harness.agent`. The messages go after the messages that the earlier model call got, so the prompt cache of the provider still holds. +Both messages have `metadata.harness.handoff.by` (`'tool'`, `'router'`, or `'code'`). When an agent handed off, they also have its name in `metadata.harness.agent`. + +What the model of the target gets depends on `routing.history`: + +- `'own'` (default): the turns of the other speakers are labeled text in a user message, for example `main:\n...`. The handoff is one text line, for example `main handed off to billing. Note: Refund for order 42`. When billing hands back, the main model gets `billing handed off to main.` the same way. +- `'text'`: the messages of the main model stay as they are, and the two handoff messages come after them. So the prompt cache of the provider still holds. +- `'all'`: the messages of the main model and of each agent stay as they are, with their tool calls. With `routing.giveModelsHandoffTool: false`, the models have no `handoff` tool. Then the harness adds one assistant message with the same text and the note. @@ -323,7 +381,7 @@ A [fork](./fork-and-reset) stays in its harness. `host.fork` with a thread of an ## What you have now -- Agents that hand the conversation to each other, from a tool, from code, or from the router. +- Agents that hand the conversation to each other, from a tool, from code, or from the router. The tool can be your own, with the fields that you need. - A log that keeps who answers, also after a restart. - A UI that shows the name of the agent on each message. - A pause for a person, operator replies, and a way to give the thread back. diff --git a/docs/harness/subagents.md b/docs/harness/subagents.md index b32ff72e94..9d659c9a8d 100644 --- a/docs/harness/subagents.md +++ b/docs/harness/subagents.md @@ -669,7 +669,7 @@ export const studio = defineHarness({ The router can also return a pick of its own: `'main'`, a name, a list of names, `{ names, order }`, or `{ steps }`. Anywhere a name goes, `{ name, input }` can go too, for example `{ name: 'pricer', input: { vendor: 'Acme' } }`. If a picked agent has an `inputSchema` and no input, the turn fails with `Agent "pricer" needs input. Return { name: 'pricer', input } from the router.` See [Route, or let the model pick](../chat/subagents#route-or-let-the-model-pick) for each pick. -The router also gets `session`, the `input` of the turn as the user sent it, `operationId`, `inputId`, `abortSignal`, `activeAgent`, and `model`. Next to `router`, `routing` takes `order`, `limits`, `sandbox`, `afterAgentsAnswer`, `allowHandoffToHuman`, and `giveModelsHandoffTool`. The first three work the same as in `subagents`. `giveModelsHandoffTool: false` gives no model the `handoff` tool. See [Hand off a conversation](./handoffs). +The router also gets `session`, the `input` of the turn as the user sent it, `operationId`, `inputId`, `abortSignal`, `activeAgent`, and `model`. Next to `router`, `routing` takes `order`, `limits`, `sandbox`, `afterAgentsAnswer`, `history`, `allowHandoffToHuman`, `giveModelsHandoffTool`, and `handoffTool`. The first three work the same as in `subagents`. For `history`, see [Choose what later turns see](#choose-what-later-turns-see). For the last three, see [Hand off a conversation](./handoffs). ### Count the model calls of the router @@ -751,6 +751,40 @@ How each pick changes the active agent: Agents can also hand the turn to each other, or to a person. See [Hand off a conversation](./handoffs). +### Choose what later turns see + +On turn 1, billing calls its `refund` tool and answers `I refunded order 42.` On turn 3, billing answers again. If billing reads only that text, with no tool call behind it, it can doubt the text and refund again. `routing.history` sets what each model call gets of the routed runs of earlier turns: + +```ts group=harness-routing +const billing = defineAgent({ + name: 'billing', + description: 'Handles refunds', + run: (ctx) => ctx.chat({ adapter: openaiText('gpt-6.1-sol'), stream: false }), +}) + +export const support = defineHarness({ + name: 'acme/support', + adapter: openaiText('gpt-6.1-sol'), + agents: [billing], + routing: { + router: ({ input }) => (typeof input === 'string' && input.includes('refund') ? 'billing' : undefined), + history: 'own', + }, +}) +``` + +| Value | A routed agent gets | The main model gets | +| --- | --- | --- | +| `'own'` (default) | Its own earlier runs in full, with tool calls and results. Other agents as `name:\ntext`, and the main model as `main:\ntext`. A handoff is one text line. | Its own tool calls in full. Routed runs as `billing:\n...` text, and a hand-back as one text line. | +| `'all'` | Every routed run in full. | Every routed run in full. | +| `'text'` | Every routed run as `name:\ntext`. | Every routed run as `name:\ntext`. | + +- With `'own'`, the text of another speaker reaches the model as a user message. So no model reads the words of another speaker as its own. +- The router gets the same messages with each value. +- The stored transcript does not change. Only the messages of each model call change. +- With `'text'`, a handoff target keeps the prompt cache of the provider. See [What the target gets](./handoffs#what-the-target-gets). +- The harness keeps the run records of the last 100 runs that ended. A routed run that is older reaches the model as text. + ## Call a harness as a child `harnessAgent` turns a harness into an agent. Put it in `subagents.agents`, and the main model calls it as a tool. The child harness keeps its own tools, plugins, and history. @@ -787,6 +821,32 @@ What you see in the parent session: Outside a session, for example in a plain `chat()`, `harnessAgent` runs the child harness on the host that you pass as `host`. Without `host`, it makes a host with a memory log the first time it runs. +### Make a harness a root agent + +The billing team has its own harness. Your front desk wants billing to answer whole turns and keep the conversation, not only answer one tool call. Put `harnessAgent` in `agents`, and add `routing`: + +```ts group=harness-subagents +const billingTeam = defineHarness({ + name: 'acme/billing', + description: 'Handles refunds, invoices, and payments', + adapter: openaiText('gpt-6.1-sol'), +}) + +export const frontDesk = defineHarness({ + name: 'acme/front-desk', + adapter: openaiText('gpt-6.1-sol'), + agents: [harnessAgent(billingTeam, { name: 'billing' })], + routing: { router: () => undefined }, +}) +``` + +- The agent keeps one child session for all turns, on the thread `/agent/billing`. So turn 2 sees turn 1 and its own tool calls. +- Each turn sends the child what happened in the thread since its last turn, as labeled text. Then it sends the latest user message. +- The model of the child gets the `handoff` tool of the outer turn, so it can hand back to `'main'`. A tool of the child can call `ctx.context.run.handoff`. See [Hand off a conversation](./handoffs). +- The child harness cannot have its own `routing`. If it has one, the turn fails. + +In `subagents.agents`, the same `harnessAgent` stays a tool of the main model, with a new child session for each call. + ## Stay within limits A harness always limits its children. Without `subagents.limits`, it uses a depth of 2, 3 children at once, and 12 children per tree. Agents started from code count against the same limits: @@ -894,5 +954,5 @@ const reader = defineAgent({ - Plugins, commands, and agents that start typed agents, in parallel or in series, and message them while they run. - Background agents that continue on the next host after a crash, with steps and children that do not run twice. - Runs with their own budget and time limit, in a tree that stays within limits. -- A router that sends each turn to the right agent, and a routed agent that takes steers like the main model. -- Harnesses that call other harnesses as child runs, and agents that share state by key. +- A router that sends each turn to the right agent, and a routed agent that takes steers like the main model and keeps its own tool calls. +- Harnesses that call other harnesses as child runs or as root agents, and agents that share state by key. diff --git a/packages/ai-harness/src/define.ts b/packages/ai-harness/src/define.ts index 197e688463..eaf5cbba52 100644 --- a/packages/ai-harness/src/define.ts +++ b/packages/ai-harness/src/define.ts @@ -16,6 +16,7 @@ import type { SystemPrompt, } from '@tanstack/ai' import type { AnyAgent } from './agents' +import type { HandoffOptions } from './handoff' import type { ProjectOptions, ReduceOptions } from './log' import type { MediaOptions } from './media' import type { HarnessPlugin, PluginMetadataCheck, PluginModel } from './plugins' @@ -126,6 +127,26 @@ export type HarnessRouterPick = | { handoff: string; note?: string } | undefined +/** + * What `routing.handoffTool` gets for one speaker of the turn. + */ +export interface HandoffToolContext { + /** The speaker: `null` for the main model, else the root agent name. */ + from: string | null + /** + * The names this speaker can hand off to: the root agents other than + * itself, plus `'main'` for an agent. + */ + targets: Array + /** The value of `routing.allowHandoffToHuman`. */ + allowHuman: boolean + /** + * Record a handoff with `by: 'tool'` and end the part. It adds no note + * messages: the tool call is in the transcript. + */ + handoff: (options: HandoffOptions) => Promise +} + /** * The `routing` option. The other fields mean the same as in `subagents`. * `limits` defaults to `subagents.limits`. The `strategy` of `subagents` is @@ -166,6 +187,40 @@ export interface HarnessRouting extends Pick< * hand off. Default `true`. */ giveModelsHandoffTool?: boolean + /** + * How later turns see the routed runs of earlier turns. The router gets + * the same messages in every mode. + * - `'own'`: each speaker sees only its own tool calls. A routed agent + * sees its own earlier routed runs in full. It sees the other speakers + * as labeled text (`name:\ntext`, `main:\ntext`), and a handoff call as + * a text line with the note. The main model sees routed runs as + * `name:\ntext`. + * - `'all'`: every speaker sees every routed run in full. + * - `'text'`: every speaker sees routed runs as `name:\ntext`. + * + * Default `'own'`. + */ + history?: 'own' | 'all' | 'text' + /** + * Make the handoff tool of each speaker. The harness calls it once per + * speaker (the main model and each root agent) per turn. The returned + * tool replaces the default `handoff` tool, and it can have any name. + * Not called when `giveModelsHandoffTool` is false. + * + * @example + * ```ts + * handoffTool: ({ from, targets, handoff }) => + * toolDefinition({ + * name: 'handoff', + * description: 'Hand the user to another agent', + * inputSchema: z.object({ to: z.enum(targets), note: z.string() }), + * }).server(async ({ to, note }) => { + * await handoff({ to, note: `${from ?? 'main'}: ${note}` }) + * return `Handed off to ${to}.` + * }) + * ``` + */ + handoffTool?: (ctx: HandoffToolContext) => AnyTool } /** diff --git a/packages/ai-harness/src/handoff.ts b/packages/ai-harness/src/handoff.ts index f7d795bac3..101e8c0aed 100644 --- a/packages/ai-harness/src/handoff.ts +++ b/packages/ai-harness/src/handoff.ts @@ -1,14 +1,20 @@ import { toolDefinition } from '@tanstack/ai' import { isHumanTarget } from './log-records' +import { routedRunId } from './routed-history' import { isRecord } from './utils' import type { + AfterToolCallInfo, AnyChatMiddleware, + AnyTool, + ChatMiddlewareConfig, + ChatMiddlewareContext, ModelMessage, SubagentRouterPick, + ToolCallHookContext, } from '@tanstack/ai' import type { MessageStore } from '@tanstack/ai-persistence' import type { AnyAgent } from './agents' -import type { HarnessRouterPick } from './define' +import type { HandoffToolContext, HarnessRouterPick } from './define' import type { HandoffRecord, HumanTarget } from './log-records' /** What `ctx.harness.handoff` takes. */ @@ -53,6 +59,53 @@ function handoffText(to: string | HumanTarget) { return `Handed off to ${to}. ${to} answers next.` } +/** The middleware context fields that tell which chat runs. */ +type ChatOf = Pick< + ChatMiddlewareContext, + 'subagentName' | 'parentSubagentRunId' +> + +/** + * The built-in `handoff` tool. Its `to` lists `targets`, and + * `{ human: true }` when `allowHuman` is set. + */ +function defaultHandoffTool({ + targets, + allowHuman, + handoff, +}: HandoffToolContext) { + const agents = { type: 'string', enum: targets } + const person = { + type: 'object', + properties: { human: { type: 'boolean', const: true } }, + required: ['human'], + } + return toolDefinition({ + name: HANDOFF_TOOL, + description: `Hand the conversation to another agent. That agent answers the user next, in this turn and in later turns.${allowHuman ? ' Use { human: true } only when the user needs a person.' : ''} Stop after this call.`, + inputSchema: { + type: 'object', + properties: { + to: allowHuman ? { anyOf: [agents, person] } : agents, + note: { + type: 'string', + description: 'What the next agent must know.', + }, + }, + required: ['to'], + }, + }).server(async (args: unknown) => { + const to = isRecord(args) ? args.to : undefined + const note = isRecord(args) ? args.note : undefined + const target = isHumanTarget(to) ? HUMAN : to + if (typeof target !== 'string' && !isHumanTarget(target)) { + throw new Error('A handoff needs `to`: the name of an agent.') + } + await handoff({ to: target, ...(typeof note === 'string' ? { note } : {}) }) + return handoffText(target) + }) +} + /** * The messages that show a router or code handoff in the transcript, so the * target knows that it got the conversation, and why. They look like a @@ -116,6 +169,10 @@ export class TurnHandoffs { /** The handoff that ends the current part. */ private pending: Handoff | undefined private count = 0 + /** The handoff tool of each speaker, made once per turn. */ + private readonly tools = new Map() + /** The ids of the handoff tool calls that run now. */ + private readonly activeCalls = new Set() constructor( private readonly options: { @@ -134,6 +191,8 @@ export class TurnHandoffs { humans: boolean /** `routing.giveModelsHandoffTool`: the root agents get the `handoff` tool. */ tool: boolean + /** `routing.handoffTool`: makes the handoff tool of each speaker. */ + handoffTool?: (ctx: HandoffToolContext) => AnyTool /** * Append the record. A handoff to a person opens its `human` wait in * the same append. Resolves with the record as the log keeps it. @@ -163,11 +222,13 @@ export class TurnHandoffs { } /** - * The messages that show a router or code handoff in the transcript. None - * for a `handoff` tool call: the call of the model is there already. + * The messages that show a handoff in the transcript. None for a tool + * call of the main model: its call is in the thread already. A root agent + * keeps its tool calls in its own transcript, so its handoff gets them, + * and the next speaker reads who handed off. */ messages(handoff: Handoff): Array { - if (handoff.by !== 'router' && handoff.by !== 'code') return [] + if (handoff.by === 'tool' && handoff.from === null) return [] // A turn that runs again after a crash makes the same ids, so the store // keeps each note once. const id = `handoff_${this.options.noteKey}_${this.count}` @@ -180,12 +241,18 @@ export class TurnHandoffs { * A handoff to a person ends the turn, and the session pauses. Throws for * a target that is not a root agent, for a person without * `routing.allowHandoffToHuman`, and for a second handoff in the same part. + * A code handoff inside a call of the handoff tool counts as a tool + * handoff: the tool call is in the transcript. */ async handoff( options: HandoffOptions, from: string | null, by: 'tool' | 'code', ): Promise { + // ponytail: any running handoff tool call of the turn counts. A parallel + // tool that calls `run.handoff` while a handoff call runs is also 'tool'. + // Keep the speaker of each call if that matters. + const recordedBy = this.activeCalls.size > 0 ? 'tool' : by const { to, note } = options const isHuman = isHumanTarget(to) if (isHuman && !this.options.humans) { @@ -227,7 +294,7 @@ export class TurnHandoffs { from, to: target, ...(typeof note === 'string' ? { note } : {}), - by, + by: recordedBy, // The agents an operator can hand back to: the ones this turn offers. ...(isHuman ? { agents: names } : {}), } @@ -252,82 +319,81 @@ export class TurnHandoffs { } /** - * The `handoff` tool of `from`: the main model (`null`) or a root agent. - * Its `to` lists the root agents, and `'main'` for an agent, but not - * `from`. With `routing.allowHandoffToHuman`, `{ human: true }` hands the conversation - * to a person. + * The handoff tool of `from`: the main model (`null`) or a root agent. + * `routing.handoffTool` makes it, else the built-in `handoff` tool. Its + * targets are the root agents, and `'main'` for an agent, but not `from`. + * With `routing.allowHandoffToHuman`, `{ human: true }` hands the + * conversation to a person. Each speaker gets one tool per turn. */ tool(from: string | null) { - const { humans } = this.options + const made = this.tools.get(from) + if (made) return made const targets = [ ...this.options.rootAgents.map((agent) => agent.name), ...(from === null ? [] : ['main']), ].filter((name) => name !== from) - const agents = { type: 'string', enum: targets } - const person = { - type: 'object', - properties: { human: { type: 'boolean', const: true } }, - required: ['human'], + const ctx: HandoffToolContext = { + from, + targets, + allowHuman: this.options.humans, + handoff: (options) => this.handoff(options, from, 'tool'), } - return toolDefinition({ - name: HANDOFF_TOOL, - description: `Hand the conversation to another agent. That agent answers the user next, in this turn and in later turns.${humans ? ' Use { human: true } only when the user needs a person.' : ''} Stop after this call.`, - inputSchema: { - type: 'object', - properties: { - to: humans ? { anyOf: [agents, person] } : agents, - note: { - type: 'string', - description: 'What the next agent must know.', - }, - }, - required: ['to'], - }, - }).server(async (args: unknown) => { - const to = isRecord(args) ? args.to : undefined - const note = isRecord(args) ? args.note : undefined - const target = isHumanTarget(to) ? HUMAN : to - if (typeof target !== 'string' && !isHumanTarget(target)) { - throw new Error('A handoff needs `to`: the name of an agent.') - } - await this.handoff( - { to: target, ...(typeof note === 'string' ? { note } : {}) }, - from, - 'tool', - ) - return handoffText(target) - }) + const tool = this.options.handoffTool?.(ctx) ?? defaultHandoffTool(ctx) + this.tools.set(from, tool) + return tool } /** * Middleware for the chat calls of a turn: a pending handoff stops the * tool loop of the chat that made it, so the target answers next. Other * chats of the part (a sibling root agent, a tool-mode child) go on. With - * `roots`, it also gives each routed root agent its `handoff` tool, unless - * `routing.giveModelsHandoffTool` is false. A nested child of a root agent gets no - * tool, so it cannot move the whole session. + * `roots`, it also gives each routed root agent its handoff tool, unless + * `routing.giveModelsHandoffTool` is false. A tool in `config.tools` with + * the same name wins. A nested child of a root agent gets no tool, so it + * cannot move the whole session. It also tracks the calls of the handoff + * tool, so a code handoff inside one records `by: 'tool'`. */ - middleware(options: { roots?: boolean } = {}): AnyChatMiddleware { + middleware(options: { roots?: boolean } = {}) { + /** The name of the handoff tool of a chat. Undefined when it has none. */ + const toolName = (ctx: ChatOf) => + this.options.tool && ctx.parentSubagentRunId === undefined + ? this.tool(ctx.subagentName ?? null).name + : undefined return { name: 'harness:handoff', - onConfig: (ctx, config) => { - const isRoot = - options.roots === true && - this.options.tool && - ctx.subagentName !== undefined && - ctx.parentSubagentRunId === undefined - if (!isRoot) return - if (config.tools.some((tool) => tool.name === HANDOFF_TOOL)) return + onConfig: (ctx: ChatOf, config: Pick) => { + const isRoot = options.roots === true && ctx.subagentName !== undefined + const name = isRoot ? toolName(ctx) : undefined + if (name === undefined) return undefined + if (config.tools.some((tool) => tool.name === name)) return undefined return { tools: [...config.tools, this.tool(ctx.subagentName ?? null)] } }, - onShouldContinue: (ctx) => { + // The guard tier, not onBeforeToolCall: the first onBeforeToolCall + // decision (such as changed args) stops the hooks after it. + onGuardToolCall: ( + ctx: ChatOf, + call: Pick, + ) => { + if (call.toolName === toolName(ctx)) { + this.activeCalls.add(call.toolCallId) + } + return undefined + }, + onAfterToolCall: ( + _ctx: ChatOf, + info: Pick, + ) => { + this.activeCalls.delete(info.toolCallId) + return undefined + }, + onShouldContinue: (ctx: ChatOf) => { const { pending } = this if (!pending || ctx.parentSubagentRunId !== undefined) return undefined // ponytail: a chat is known by its agent name ('main' has none). The // root agents of one pick have different names. return (ctx.subagentName ?? null) === pending.from ? false : undefined }, - } + } satisfies AnyChatMiddleware } private async write(handoff: Handoff) { @@ -397,10 +463,7 @@ export function answererOf(pick: SubagentRouterPick | undefined) { * root agents of run `runId` in one assistant message. */ function routedIndex(messages: ReadonlyArray, runId: string) { - return messages.findIndex((message) => { - const tanstack = message.metadata?.tanstack - return isRecord(tanstack) && tanstack.runId === runId - }) + return messages.findIndex((message) => routedRunId(message) === runId) } /** diff --git a/packages/ai-harness/src/harness-agent.ts b/packages/ai-harness/src/harness-agent.ts index 8f7d7ec532..9cec92c77e 100644 --- a/packages/ai-harness/src/harness-agent.ts +++ b/packages/ai-harness/src/harness-agent.ts @@ -6,11 +6,21 @@ import type { AnyHarness } from './define' import type { HarnessHost } from './host' /** - * Use a whole harness as a child agent: put it in `subagents.agents` and the - * main model can call it as a tool, with its own tools, plugins, and history. + * Use a whole harness as a child agent, with its own tools, plugins, and + * history. It works in two places: * - * In a harness session, the child is a session of `harness` on the thread - * `${rootThreadId}/agent/${subagentRunId}`, in the log of the root thread. + * - In `subagents.agents`: the main model calls it as a tool. + * - In `agents`, with `routing`: it is a root agent that answers turns of + * the thread. It keeps one child session per agent across turns, on the + * thread `${rootThreadId}/agent/${name}`. Each turn sends it the latest + * user message, after what happened in the thread since its last turn. + * Its model gets the `handoff` tool of the outer turn, so it can hand + * back. A tool of the child can call `ctx.context.run.handoff`. The child + * harness cannot have its own `routing`: that turn fails. + * + * In a harness session, the child is a session of `harness` in the log of + * the root thread. As a tool, its thread is + * `${rootThreadId}/agent/${subagentRunId}`, one per call. * It runs for the user of the run that starts it, with the keys of that * user. Its model calls count in the usage and the budget of that run, and * the snapshot of the session lists its waits. An answer there goes to the @@ -28,6 +38,13 @@ import type { HarnessHost } from './host' * adapter, * subagents: { agents: [harnessAgent(reviewer)] }, * }) + * + * defineHarness({ + * name: 'acme/front-desk', + * adapter, + * agents: [harnessAgent(billing, { name: 'billing' })], + * routing: { router: () => 'billing' }, + * }) * ``` */ export function harnessAgent( diff --git a/packages/ai-harness/src/index.ts b/packages/ai-harness/src/index.ts index a8f0029384..700f2d0df3 100644 --- a/packages/ai-harness/src/index.ts +++ b/packages/ai-harness/src/index.ts @@ -1,6 +1,7 @@ export { defineHarness, isHarnessDefinition } from './define' export type { AnyHarness, + HandoffToolContext, HarnessAgentsOf, HarnessConfig, HarnessDefinition, diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts new file mode 100644 index 0000000000..88bbae6dfb --- /dev/null +++ b/packages/ai-harness/src/routed-history.ts @@ -0,0 +1,277 @@ +import { textOf } from './first-party/session-tools' +import { isHumanTarget } from './log-records' +import { isRecord } from './utils' +import type { + AnyChatMiddleware, + ChatMiddlewareConfig, + ChatMiddlewareContext, + ModelMessage, +} from '@tanstack/ai' +import type { HarnessRouting } from './define' + +/** A routed child of a run: the agent that answered, and its transcript id. */ +type RoutedChild = { name: string; subagentRunId: string } + +/** Where the routed runs of a thread and their transcripts come from. */ +interface HistorySource { + mode: NonNullable + /** The name of the handoff tool. `'handoff'` when not set. */ + handoffToolName?: string + /** The routed children of run `runId`, in order. */ + childrenOf: (runId: string) => Array + /** The stored transcript of one routed child. */ + transcriptOf: (subagentRunId: string) => Promise> + /** + * True for a message of the shared thread. An assistant message that is + * not (the messages of the current chat, or of a resumed run) stays as it + * is. Every message counts when not set. + */ + isThreadMessage?: (message: ModelMessage) => boolean +} + +/** The run id of a routed host message: `metadata.tanstack.runId`. */ +export function routedRunId(message: ModelMessage) { + const tanstack = message.metadata?.tanstack + return isRecord(tanstack) && typeof tanstack.runId === 'string' + ? tanstack.runId + : undefined +} + +/** + * The run id of a routed pass. An earlier pass of a run with several passes + * keeps it in `metadata.harness.routedRunId`, because its `tanstack.runId` + * is gone. The last pass has `metadata.tanstack.runId`. + */ +function passRunId(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.routedRunId === 'string' + ? harness.routedRunId + : routedRunId(message) +} + +/** The root agent that wrote a message: `metadata.harness.agent`. */ +export function routedAgent(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.agent === 'string' + ? harness.agent + : undefined +} + +/** True for a router or code handoff note: `metadata.harness.handoff`. */ +export function isHandoffNote(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && harness.handoff !== undefined +} + +/** The line that tells a reader about one handoff call. */ +function handoffLine(from: string, args: string) { + let parsed: unknown + try { + parsed = JSON.parse(args) + } catch { + parsed = undefined + } + if (!isRecord(parsed)) return `${from} handed off.` + const { to, note } = parsed + const target = isHumanTarget(to) ? 'a person' : to + const head = + typeof target === 'string' + ? `${from} handed off to ${target}.` + : `${from} handed off.` + return typeof note === 'string' ? `${head} Note: ${note}` : head +} + +/** The joined text of the assistant messages of a transcript. */ +function assistantText(messages: ReadonlyArray) { + return messages + .filter((message) => message.role === 'assistant') + .map((message) => textOf(message).trim()) + .filter((text) => text !== '') + .join('\n\n') +} + +/** + * The messages of a routed child for the model: its transcript without the + * task it started from, and without an empty placeholder message. + */ +function childMessages(transcript: Array) { + const rest = transcript[0]?.role === 'user' ? transcript.slice(1) : transcript + return rest.filter( + (message) => + message.role !== 'assistant' || + textOf(message) !== '' || + (message.toolCalls?.length ?? 0) > 0, + ) +} + +/** + * The text of another speaker as one user message, with the id and the + * metadata of the message it replaces. A user message: the words are not + * the reader's own, and the context still ends on a user turn after a + * handoff (some models, new Claude models among them, reject a context that + * ends on an assistant message). The fields of an assistant message + * (thinking, tool calls) do not go with it. + */ +function otherSpeaker(message: ModelMessage | undefined, content: string) { + const user: ModelMessage = { + ...(message?.id !== undefined && { id: message.id }), + ...(message?.metadata !== undefined && { metadata: message.metadata }), + role: 'user', + content, + } + return user +} + +/** + * Replace a routed host message with the transcripts of its children. A + * child that `wanted` picks comes in full. Another child comes as its + * `name:\ntext`. Undefined when no picked child has a stored transcript. + */ +async function expandHost( + message: ModelMessage, + children: Array, + wanted: (name: string) => boolean, + transcriptOf: HistorySource['transcriptOf'], +) { + // A host message of one agent holds only that agent's pass of the run. + const agent = routedAgent(message) + const own = + agent === undefined + ? children + : children.filter((child) => child.name === agent) + if (!own.some((child) => wanted(child.name))) return undefined + const parts: Array = [] + let expanded = false + for (const child of own) { + const body = childMessages(await transcriptOf(child.subagentRunId)) + if (wanted(child.name) && body.length > 0) { + parts.push(...body) + expanded = true + continue + } + const text = assistantText(body) + if (text !== '') { + parts.push(otherSpeaker(undefined, `${child.name}:\n${text}`)) + } + } + return expanded ? parts : undefined +} + +/** + * Fold one message of the thread for `speaker` (`null` for the main model) + * in `'own'` mode. A message of the speaker stays. A message of another + * speaker becomes text in one user message, so the speaker does not read + * it as its own: the main model's text as `main:\ntext`, a routed answer + * as its `name:\ntext`, and each handoff call as one line. Other tool calls + * of another speaker go. `folded` collects the ids of the tool calls that + * this removes, so their results go too. + */ +function foldFor( + message: ModelMessage, + speaker: string | null, + handoffToolName: string, + folded: Set, +) { + if (message.role === 'tool') { + const answersFolded = + message.toolCallId !== undefined && folded.has(message.toolCallId) + return answersFolded ? [] : [message] + } + // A note names the agent that handed off. A router note of the main + // model names none. + const owner = routedAgent(message) ?? null + if (message.role !== 'assistant' || owner === speaker) return [message] + const isNote = isHandoffNote(message) + const from = owner ?? 'main' + const text = textOf(message).trim() + const lines: Array = [] + const calls = message.toolCalls ?? [] + if (isNote && calls.length === 0 && text !== '') { + // A plain note (no handoff tool): its text says who answers next. + lines.push(text) + } else if (!isNote && text !== '') { + // A routed answer has its `name:` label already. + lines.push(owner === null ? `main:\n${text}` : text) + } + for (const call of message.toolCalls ?? []) { + folded.add(call.id) + // A note pair always uses the built-in tool name. + if (isNote || call.function.name === handoffToolName) { + lines.push(handoffLine(from, call.function.arguments)) + } + } + return lines.length === 0 ? [] : [otherSpeaker(message, lines.join('\n\n'))] +} + +/** + * The history that `speaker` sees (`null` for the main model). + * + * - `'text'`: the messages as they are. + * - `'all'`: every routed host message becomes the transcripts of its run. + * - `'own'`: each speaker gets its own messages and its own routed runs in + * full, and the words of the other speakers as labeled text. + */ +export async function historyFor( + options: HistorySource & { + messages: Array + speaker: string | null + }, +) { + const { messages, speaker, mode, childrenOf, transcriptOf } = options + if (mode === 'text') return messages + const handoffToolName = options.handoffToolName ?? 'handoff' + const wanted = (name: string) => mode === 'all' || name === speaker + const folded = new Set() + const result: Array = [] + for (const message of messages) { + const isOutsideThread = + message.role === 'assistant' && + options.isThreadMessage?.(message) === false + if (isOutsideThread) { + result.push(message) + continue + } + const runId = passRunId(message) + const children = + message.role === 'assistant' && runId !== undefined + ? childrenOf(runId) + : [] + const expanded = + children.length > 0 + ? await expandHost(message, children, wanted, transcriptOf) + : undefined + if (expanded) result.push(...expanded) + else if (mode === 'all') result.push(message) + else result.push(...foldFor(message, speaker, handoffToolName, folded)) + } + return result +} + +/** + * Middleware that gives each routed root agent its history for + * `routing.history`. It sets `providerMessages`, so the stored thread does + * not change. A nested child of a root agent keeps its own messages. + */ +export function routedHistoryMiddleware(options: HistorySource) { + return { + name: 'harness:routed-history', + onConfig: async ( + ctx: Pick, + config: Pick, + ) => { + if (options.mode === 'text' || ctx.parentSubagentRunId !== undefined) { + return undefined + } + const source = config.providerMessages ?? config.messages + const providerMessages = await historyFor({ + ...options, + messages: source, + speaker: ctx.subagentName ?? null, + }) + const isSame = + providerMessages.length === source.length && + providerMessages.every((message, index) => message === source[index]) + return isSame ? undefined : { providerMessages } + }, + } satisfies AnyChatMiddleware +} diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index fe4859fbce..e475f2f975 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -55,6 +55,7 @@ import type { HeldPlugins } from './helpers' import { SessionTurnParts, stepWaitPrefix } from './turn-parts' import type { ChatMiddleware, + ContentPart, SubagentBudget, Interrupt, ModelMessage, @@ -69,6 +70,8 @@ import type { InnerTurnEnd } from '../harness-text' import type { AgentResultRecord, RunTreeNode } from '../log-records' import { NO_SESSION_CONTEXT, withHarnessChild } from '../agents' import { forwardTurn, lastUserInput } from '../harness-text' +import { textOf } from '../first-party/session-tools' +import { historyFor, isHandoffNote, routedAgent } from '../routed-history' import { wait as backoff, isTransientModelError } from '../turn' import { addCounts, emptyCounts } from '../usage' import { isRecord } from '../utils' @@ -537,15 +540,15 @@ export abstract class SessionAgents< const extend: SubagentBinding['extend'] = (turn || handoffs) && ((child: SubagentRunContext) => { - const handoff = + // A routed root agent: a `harnessAgent` there keeps its child session. + const root = handoffs && child.parentSubagentRunId === undefined - ? (handoffOptions: HandoffOptions) => - handoffs.handoff( - handoffOptions, - child.forward.subagentName, - 'code', - ) - : NO_SESSION_CONTEXT.handoff + ? { name: child.forward.subagentName, handoffs } + : undefined + const handoff = root + ? (handoffOptions: HandoffOptions) => + root.handoffs.handoff(handoffOptions, root.name, 'code') + : NO_SESSION_CONTEXT.handoff if (!turn) return { harness: { ...NO_SESSION_CONTEXT, handoff } } // The steps of the child are in the log under its run id: the same // id when it goes on after a wait of the turn. @@ -561,7 +564,12 @@ export abstract class SessionAgents< state: (stateOptions) => this.sharedState(stateOptions), handoff, }, - (inner, ctx) => this.runHarnessChild(inner, ctx, { operation, user }), + (inner, ctx) => + this.runHarnessChild(inner, ctx, { + operation, + user, + ...(root && { root }), + }), run, ) return { harness } @@ -829,6 +837,12 @@ export abstract class SessionAgents< * child turns with `parentRunId`, then a RUN_FINISHED with the usage of * the child. A stop of the parent run cancels the child, and a recoverable * close stops it as on a crash. The child closes when the run ends. + * + * With `parent.root` (a routed root agent), the thread is + * `${rootThreadId}/agent/${name}`: one child session per agent, kept + * across turns. Each turn sends what happened in the shared thread since + * the last answer of the agent. Its main model gets the `handoff` tool of + * the outer turn, unless `routing.giveModelsHandoffTool` is false. */ protected async *runHarnessChild( harness: AnyHarness, @@ -836,13 +850,23 @@ export abstract class SessionAgents< parent: { operation: OperationImpl | OperationImpl user: HarnessUser | undefined + root?: { name: string; handoffs: TurnHandoffs } }, ): AsyncGenerator { const open = this.openChild if (!open) throw new Error('This session cannot open a child session.') + const { root } = parent const parentRunId = parent.operation.id - const threadId = `${this.rootThreadId}/agent/${ctx.subagentRunId}` + const threadId = `${this.rootThreadId}/agent/${root?.name ?? ctx.subagentRunId}` const used = emptyCounts() + // With the handoff tool off, the child still hands off from code. + const handoff = root + ? childHandoff( + root.name, + root.handoffs, + this.harness.routing?.giveModelsHandoffTool !== false, + ) + : undefined const session = await open(harness, { threadId, user: parent.user, @@ -857,6 +881,7 @@ export abstract class SessionAgents< }) this.checkBudget(parent.operation) }, + ...(handoff && { handoff }), }, }) let operation: Operation | undefined @@ -899,8 +924,9 @@ export abstract class SessionAgents< }) signal.addEventListener('abort', onAbort, { once: true }) try { + const messages = convertMessagesToModelMessages(ctx.messages) operation = session.send( - lastUserInput(convertMessagesToModelMessages(ctx.messages)), + root ? await rootInput(messages, root.name) : lastUserInput(messages), ) if (signal.aborted) onAbort() let finished: InnerTurnEnd['finished'] = 'stop' @@ -2777,6 +2803,69 @@ function withParentRun(chunk: StreamChunk, parentRunId: string): StreamChunk { }) } +/** + * The link that lets the child session of the routed root agent `from` hand + * off in the outer turn: the outer `handoff` tool, `run.handoff`, and a check + * that stops the tool loop of the child after a handoff. + */ +function childHandoff(from: string, handoffs: TurnHandoffs, withTool: boolean) { + // The outer middleware stops the chat of `from` after its handoff. The + // child asks it the same question. + const { onShouldContinue } = handoffs.middleware() + return { + from, + ...(withTool ? { tool: handoffs.tool(from) } : {}), + call: (options: HandoffOptions) => handoffs.handoff(options, from, 'code'), + pending: () => onShouldContinue({ subagentName: from }) === false, + } +} + +/** + * The input of a turn of the routed root `harnessAgent` named `name`: the + * latest user message, after one text part with what happened in the shared + * thread since the last answer of `name`. Other speakers come with their + * labels, as `routing.history: 'own'` folds them. The child session keeps + * its own turns, so they are not sent again. On its first turn, the context + * is the conversation so far. + */ +async function rootInput(messages: Array, name: string) { + // A handoff note names its source agent too, but it is no answer. + const isAnswer = (message: ModelMessage) => + routedAgent(message) === name && !isHandoffNote(message) + const lastAnswer = messages.findLastIndex(isAnswer) + const latest = messages.findLastIndex((message) => message.role === 'user') + const since = messages.filter( + (_, index) => index > lastAnswer && index !== latest, + ) + // No transcripts: other runs stay as their `name:\ntext` host message. + const folded = await historyFor({ + messages: since, + speaker: name, + mode: 'own', + childrenOf: () => [], + transcriptOf: async () => [], + }) + // A message of the user comes back as the same object. A folded message + // of another speaker is a new user message that has its label already. + const fromUser = new Set(since.filter((message) => message.role === 'user')) + const lines = folded.flatMap((message) => { + const line = textOf(message).trim() + if (line === '' || message.role === 'tool') return [] + return [fromUser.has(message) ? `user:\n${line}` : line] + }) + const input = lastUserInput(messages) + if (lines.length === 0) return input + const head = + lastAnswer === -1 ? 'The conversation so far:' : 'Since your last turn:' + const sent: Array = + typeof input === 'string' ? [{ type: 'text', content: input }] : input + const parts: Array = [ + { type: 'text', content: [head, ...lines].join('\n\n') }, + ...sent, + ] + return parts +} + /** * The admission of agent runs: at most `max()` holders at once, no limit * when it is `undefined`. `busy` names more holders that the gate does not diff --git a/packages/ai-harness/src/session/turn.ts b/packages/ai-harness/src/session/turn.ts index 2aa178562b..9e51a7cf72 100644 --- a/packages/ai-harness/src/session/turn.ts +++ b/packages/ai-harness/src/session/turn.ts @@ -60,7 +60,8 @@ import { import { SessionAgents } from './agents' import { commandStepScope, stepWaitPrefix } from './turn-parts' import { ToolTurnAdapter } from '../tool-turn' -import { HANDOFF_TOOL, TurnHandoffs, answererOf, markRouted } from '../handoff' +import { TurnHandoffs, answererOf, markRouted } from '../handoff' +import { routedHistoryMiddleware } from '../routed-history' import type { AnyChatMiddleware, AnyTextAdapter, @@ -717,7 +718,15 @@ export abstract class SessionTurn< run, ...(attempt ? { attempt } : {}), } + /** The link of the child session of a routed root `harnessAgent`. */ + const childHandoff = this.childLink?.handoff try { + // Two handoff scopes in one session: a handoff of the child is unclear. + if (childHandoff && this.harness.routing) { + throw new Error( + `A harnessAgent that is a root agent cannot have its own routing. Remove routing from "${this.harness.name}".`, + ) + } await this.flushNotes() // The stored settings of the thread, as they are when the turn starts. const stored = this.currentSettings @@ -799,6 +808,9 @@ export abstract class SessionTurn< subagents: subagentList.map((agent) => agent.name), humans: this.harness.routing.allowHandoffToHuman === true, tool: this.harness.routing.giveModelsHandoffTool !== false, + ...(this.harness.routing.handoffTool + ? { handoffTool: this.harness.routing.handoffTool } + : {}), append: (record) => this.appendHandoff(record, storedUser(sender).user), publish: (record) => @@ -894,27 +906,19 @@ export abstract class SessionTurn< // here, so a middleware wrapper wraps the bound tool, // and again at each model call, for tools that a middleware returns. const bindTool = this.durableBinder(run) - // `routing.giveModelsHandoffTool: false` gives the models no `handoff` tool. + // `routing.giveModelsHandoffTool: false` gives the models no handoff tool. + // A child of a routed root agent gets the handoff tool of the outer turn. const handoffTool = handoffs && this.harness.routing?.giveModelsHandoffTool !== false ? handoffs.tool(null) - : undefined - const clash = handoffTool - ? prepared.find((tool) => tool.name === HANDOFF_TOOL) - : undefined - if (clash) { - const owner = - [ - ...(session?.owners.tools ?? []), - ...(runPlugins?.owners.tools ?? []), - ].find((entry) => entry.name === HANDOFF_TOOL)?.owner ?? 'the turn' - throw new Error( - `Duplicate tool "${HANDOFF_TOOL}": first owner routing (the handoff tool), second owner ${owner}. Rename the tool.`, - ) - } + : childHandoff?.tool + // An app tool with the same name wins, as it does for a root agent. + const isHandoffTaken = prepared.some( + (tool) => tool.name === handoffTool?.name, + ) const tools = [ ...prepared.map((tool) => bindDurable(tool, bindTool)), - ...(handoffTool ? [handoffTool] : []), + ...(handoffTool && !isHandoffTaken ? [handoffTool] : []), ] // The override tools as the turn has them: prepared and bound. const keptTools = tools.filter((tool) => turnToolNames.has(tool.name)) @@ -957,6 +961,47 @@ export abstract class SessionTurn< const durableTools = this.durableTools(bindTool) const signal = operation.abortController.signal const { routing } = this.harness + // How a model call sees the routed runs of the thread. The transcripts + // do not change in a turn, and onConfig runs before each model call, + // so the turn reads each one once. + const historyMode = routing?.history ?? 'own' + const transcripts = new Map>>() + const routedHistory = + handoffs && historyMode !== 'text' + ? routedHistoryMiddleware({ + mode: historyMode, + ...(handoffTool ? { handoffToolName: handoffTool.name } : {}), + // ponytail: the log fold (the run store too) keeps the last + // `KEPT_ENDED` ended runs. An older routed run has no children + // there, so the model gets it as text. Keep the routed children + // in the fold if that matters. + childrenOf: (runId) => + [...this.writer.state.thread.runs.values()].flatMap( + ({ parentRunId: parent, name, subagentRunId }) => + parent === runId && + name !== undefined && + subagentRunId !== undefined && + // Only routed root agents: a tool-mode child of the main + // model in the same run reports through its tool result. + rootAgents.some((agent) => agent.name === name) + ? [{ name, subagentRunId }] + : [], + ), + transcriptOf: (subagentRunId) => { + const transcript = + transcripts.get(subagentRunId) ?? + this.agentTranscript({ subagentRunId }) + transcripts.set(subagentRunId, transcript) + return transcript + }, + // A routed agent's own messages of its chat, and of a resumed + // run, are in its own thread. ponytail: a scan of the thread + // for each message. A set of ids if long threads get slow. + isThreadMessage: (entry) => + entry.id !== undefined && + this.writer.state.messages.some(({ id }) => id === entry.id), + }) + : undefined /** * The messages that the next chat() call adds after `message`: the * note of a handoff, or the partial answer and the note that a @@ -1038,6 +1083,8 @@ export abstract class SessionTurn< let answerer = isRootPart ? answererOf(pick) : null // A tool of the main model hands off as the main model. A tool of a // root agent hands off as that agent. + // The child of a routed root agent hands off in the outer turn. + if (childHandoff) turnHandoff = childHandoff.call if (handoffs) { turnHandoff = (options) => handoffs.handoff( @@ -1135,10 +1182,13 @@ export abstract class SessionTurn< // The root agents of a routed part take steers at each model // call, as the main model does, and get the ephemeral messages // of a turn hook. Those go after the reset cut, so it keeps them. + // The routed history goes after the reset cut too, so it + // changes only the context after a reset. const { chatMiddleware } = binding chatMiddleware.splice( chatMiddleware.indexOf(resetCut) + 1, 0, + ...(routedHistory ? [routedHistory] : []), ephemeralCall(ephemeral), ) if (!isToolTurn) { @@ -1190,10 +1240,23 @@ export abstract class SessionTurn< durableTools, this.detachTools(operation.id, sender), ...(handoffs ? [handoffs.middleware()] : []), + // After a handoff of the child, the outer turn runs the target. + ...(childHandoff + ? [ + { + name: 'harness:child-handoff', + onShouldContinue: () => + childHandoff.pending() ? false : undefined, + }, + ] + : []), ...(rootBag ? [] : [this.syncFold()]), // Again: a middleware that returns `messages` (as `syncFold` // does) resets what the model gets to the whole transcript. resetCut, + // After the reset cut. In `'own'` the main model gets the + // answers of the routed agents as labeled user text. + ...(routedHistory ? [routedHistory] : []), // After steering and the reset, so neither drops them. ...(start && turn.ephemeral ? [turnEphemeral(turn.ephemeral, start)] @@ -1914,18 +1977,24 @@ const attachmentId = (id: string) => `${id}:media` /** * The answer message of a routed run, kept under the id `${id}:${index}`. * It loses its run tag, so the store writes the next answer of the run as a - * new message. + * new message. The run id goes to `metadata.harness.routedRunId`, so the + * routed history of a later turn still finds the children of the pass. */ -function keptAnswer(message: ModelMessage, index: number): ModelMessage { +function keptAnswer(message: ModelMessage, index: number) { const { metadata } = message const tanstack = isRecord(metadata) ? metadata.tanstack : undefined if (!isRecord(metadata) || !isRecord(tanstack)) { return { ...message, id: `${message.id}:${index}` } } - const { runId: _runId, ...rest } = tanstack + const { runId, ...rest } = tanstack + const harness = isRecord(metadata.harness) ? metadata.harness : {} return { ...message, id: `${message.id}:${index}`, - metadata: { ...metadata, tanstack: rest }, + metadata: { + ...metadata, + tanstack: rest, + harness: { ...harness, routedRunId: runId }, + }, } } diff --git a/packages/ai-harness/src/session/types.ts b/packages/ai-harness/src/session/types.ts index 1d4a1c6ad9..b9cbf47364 100644 --- a/packages/ai-harness/src/session/types.ts +++ b/packages/ai-harness/src/session/types.ts @@ -1,5 +1,6 @@ import type { CredentialStore } from '@tanstack/ai/byok' import type { + AnyTool, SubagentBudget, Interrupt, Logger, @@ -29,6 +30,7 @@ import type { SessionImpl } from './harness-session' import type { AgentInputOf, AgentResultOf, AnyAgent } from '../agents' import type { PendingWait } from '../waits' import type { HandoffRecord } from '../log-records' +import type { HandoffOptions } from '../handoff' import type { AnyHarness, HarnessAgentsOf } from '../define' import type { HarnessPersistence, @@ -472,6 +474,23 @@ export interface ChildSessionLink { * the budget of that run. The child keeps no usage of its own. */ countUsage: (call: UsageCall) => Promise + /** + * Set only for the child session of a `harnessAgent` that is a routed + * root agent. It lets the child hand off in the outer turn. + */ + handoff?: { + /** The name of the agent in the outer turn. */ + from: string + /** + * The outer handoff tool, for the main model of the child. None when + * `routing.giveModelsHandoffTool` is false. + */ + tool?: AnyTool + /** Hand off in the outer turn. */ + call: (options: HandoffOptions) => Promise + /** True after a handoff. Then the child stops its tool loop. */ + pending: () => boolean + } } /** A child session of `harnessAgent` that a run of a session has open. */ diff --git a/packages/ai-harness/tests/handoff-tool.test.ts b/packages/ai-harness/tests/handoff-tool.test.ts new file mode 100644 index 0000000000..b2539135f8 --- /dev/null +++ b/packages/ai-harness/tests/handoff-tool.test.ts @@ -0,0 +1,226 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { TurnHandoffs } from '../src/handoff' +import type { AnyTool } from '@tanstack/ai' +import type { HandoffToolContext } from '../src' +import type { HandoffRecord } from '../src/log-records' + +function agent(name: string) { + return defineAgent({ + name, + description: `The ${name} desk`, + run: async () => name, + }) +} + +const ROOTS = [agent('billing'), agent('triage')] + +/** The middleware context of the main model's chat. */ +const MAIN = {} +/** The middleware context of the chat of root agent `name`. */ +const rootChat = (name: string) => ({ subagentName: name }) + +/** `TurnHandoffs` with a fake log: `records` gets each published record. */ +function turnHandoffs( + options: { + humans?: boolean + tool?: boolean + handoffTool?: (ctx: HandoffToolContext) => AnyTool + } = {}, +) { + const records: Array = [] + const handoffs = new TurnHandoffs({ + runId: 'run_1', + noteKey: 'input_1', + rootAgents: ROOTS, + subagents: [], + humans: options.humans ?? false, + tool: options.tool ?? true, + append: async (record) => record, + publish: (record) => { + records.push(record) + }, + ...(options.handoffTool ? { handoffTool: options.handoffTool } : {}), + }) + return { handoffs, records } +} + +/** A `handoffTool` factory that keeps each context it gets. */ +function transferFactory(name = 'transfer') { + const contexts: Array = [] + const factory = (ctx: HandoffToolContext) => { + contexts.push(ctx) + return toolDefinition({ + name, + description: 'Move the user to another desk.', + inputSchema: z.object({ to: z.string(), note: z.string() }), + }).server(async ({ to, note }) => { + await ctx.handoff({ to, note }) + return `Moved to ${to}.` + }) + } + return { contexts, factory } +} + +/** An app tool named `handoff`. */ +const appHandoff = toolDefinition({ + name: 'handoff', + description: 'The app hands off.', + inputSchema: z.object({ to: z.string() }), +}).server(async ({ to }) => `App moved to ${to}.`) + +/** The one record of `records`. */ +function onlyRecord(records: Array) { + expect(records).toHaveLength(1) + const [record] = records + if (!record) throw new Error('No handoff record.') + return record +} + +describe('routing.handoffTool factory', () => { + it.each([true, false])( + 'gets the speaker, its targets, and allowHuman %s', + (humans) => { + const { contexts, factory } = transferFactory() + const { handoffs } = turnHandoffs({ humans, handoffTool: factory }) + + handoffs.tool(null) + handoffs.tool('billing') + + expect( + contexts.map(({ from, targets, allowHuman }) => ({ + from, + targets, + allowHuman, + })), + ).toEqual([ + { from: null, targets: ['billing', 'triage'], allowHuman: humans }, + { from: 'billing', targets: ['triage', 'main'], allowHuman: humans }, + ]) + }, + ) + + it('records a handoff of the factory tool with by tool, and adds a note only for a root agent', async () => { + const { factory } = transferFactory() + const { handoffs, records } = turnHandoffs({ handoffTool: factory }) + + const result = await handoffs + .tool('billing') + .execute?.({ to: 'triage', note: 'Needs a ticket' }) + + expect(result).toBe('Moved to triage.') + const record = onlyRecord(records) + expect(record).toEqual({ + runId: 'run_1', + from: 'billing', + to: 'triage', + note: 'Needs a ticket', + by: 'tool', + }) + // A root agent keeps its tool calls in its own transcript, so the shared + // thread gets the note pair. The main model's call is in the thread. + expect(handoffs.messages(record)).toMatchObject([ + { + role: 'assistant', + toolCalls: [{ function: { name: 'handoff' } }], + metadata: { harness: { agent: 'billing', handoff: { by: 'tool' } } }, + }, + { role: 'tool', content: 'Handed off to triage. triage answers next.' }, + ]) + expect(handoffs.messages({ ...record, from: null })).toEqual([]) + }) + + it('keeps the default handoff tool when no factory is set', async () => { + const { handoffs, records } = turnHandoffs() + const tool = handoffs.tool(null) + + const result = await tool.execute?.({ to: 'billing' }) + + expect(tool.name).toBe('handoff') + expect(result).toBe('Handed off to billing. billing answers next.') + expect(onlyRecord(records)).toEqual({ + runId: 'run_1', + from: null, + to: 'billing', + by: 'tool', + }) + }) +}) + +describe('a code handoff inside a handoff tool call', () => { + it('records by tool while a handoff-named call runs', async () => { + const { handoffs, records } = turnHandoffs() + const middleware = handoffs.middleware() + + await middleware.onGuardToolCall(MAIN, { + toolName: 'handoff', + toolCallId: 'call_1', + }) + await handoffs.handoff({ to: 'billing', note: 'Refund' }, null, 'code') + + expect(onlyRecord(records).by).toBe('tool') + }) + + it('records by code after the handoff-named call ends', async () => { + const { handoffs, records } = turnHandoffs() + const middleware = handoffs.middleware() + + await middleware.onGuardToolCall(MAIN, { + toolName: 'handoff', + toolCallId: 'call_1', + }) + await middleware.onAfterToolCall(MAIN, { toolCallId: 'call_1' }) + await handoffs.handoff({ to: 'billing' }, null, 'code') + + expect(onlyRecord(records).by).toBe('code') + }) + + it('records by code inside a call of another tool', async () => { + const { handoffs, records } = turnHandoffs() + const middleware = handoffs.middleware() + + await middleware.onGuardToolCall(MAIN, { + toolName: 'search', + toolCallId: 'call_1', + }) + await handoffs.handoff({ to: 'billing' }, null, 'code') + + expect(onlyRecord(records).by).toBe('code') + }) +}) + +describe('the handoff tool of a root agent', () => { + it('adds no tool and never calls the factory with giveModelsHandoffTool false', async () => { + const { contexts, factory } = transferFactory() + const { handoffs } = turnHandoffs({ tool: false, handoffTool: factory }) + const middleware = handoffs.middleware({ roots: true }) + + const change = middleware.onConfig(rootChat('billing'), { tools: [] }) + await middleware.onGuardToolCall(rootChat('billing'), { + toolName: 'transfer', + toolCallId: 'call_1', + }) + + expect(change).toBeUndefined() + expect(contexts).toEqual([]) + }) + + it('keeps an app tool with the handoff name only for the agent that has it', () => { + const { factory } = transferFactory('handoff') + const { handoffs } = turnHandoffs({ handoffTool: factory }) + const middleware = handoffs.middleware({ roots: true }) + + const billing = middleware.onConfig(rootChat('billing'), { + tools: [appHandoff], + }) + const triage = middleware.onConfig(rootChat('triage'), { tools: [] }) + + expect(billing).toBeUndefined() + expect(triage).toEqual({ tools: [handoffs.tool('triage')] }) + // The next model call of triage has the tool already. + expect( + middleware.onConfig(rootChat('triage'), { tools: triage?.tools ?? [] }), + ).toBeUndefined() + }) +}) diff --git a/packages/ai-harness/tests/handoff.test.ts b/packages/ai-harness/tests/handoff.test.ts index 74d8d8b6c5..2f9fec43e9 100644 --- a/packages/ai-harness/tests/handoff.test.ts +++ b/packages/ai-harness/tests/handoff.test.ts @@ -55,6 +55,7 @@ async function openRouting(options: { humans?: boolean log?: LogStore ownership?: OwnershipOptions + handoffTool?: HarnessRouting['handoffTool'] }) { const { metadata } = memoryPersistence().stores const log = options.log ?? memoryLogStore() @@ -79,6 +80,7 @@ async function openRouting(options: { ? { giveModelsHandoffTool: options.handoffs } : {}), ...(options.humans ? { allowHandoffToHuman: true } : {}), + ...(options.handoffTool ? { handoffTool: options.handoffTool } : {}), }, }), { threadId: THREAD, user: USER }, @@ -381,17 +383,62 @@ describe('handoffs', () => { await host.close() }) - it('fails the turn when an app tool is named handoff', async () => { + it('an app tool named handoff replaces the harness tool for main', async () => { const billing = modelAgent('billing', []) - const { host, session } = await openRouting({ + const { host, session, main } = await openRouting({ agents: [billing.agent], tools: [plainTool('handoff')], - replies: [() => text('never')], + replies: [() => text('Main')], }) - await expect(session.send('hi')).rejects.toThrow( - 'Duplicate tool "handoff": first owner routing (the handoff tool), second owner the harness.', + expect((await session.send('hi')).text).toBe('Main') + // Only the app tool, once. + const tools = main.calls[0].tools + expect(toolNames(main.calls[0])).toEqual(['handoff']) + expect(tools[0].description).toBe('The handoff tool') + await host.close() + }) + + it('routing.handoffTool gives each speaker its own tool', async () => { + const billing = modelAgent('billing', [() => text('Refunded')]) + const { host, session, main, log } = await openRouting({ + agents: [billing.agent], + handoffTool: ({ from, handoff }) => + toolDefinition({ + name: 'transfer', + description: `Transfer from ${from ?? 'main'}`, + inputSchema: z.object({ to: z.string(), note: z.string() }), + }).server(async ({ to, note }) => { + await handoff({ to, note: `${from ?? 'main'}: ${note}` }) + return `Moved to ${to}.` + }), + replies: [ + () => + toolCall('transfer', { to: 'billing', note: 'order 42' }, 'call_t'), + ], + }) + + expect((await session.send('refund please')).text).toBe( + 'billing:\nRefunded', ) + const description = (call: { tools: Array }) => + call.tools.map((tool) => tool.description) + expect(toolNames(main.calls[0])).toEqual(['transfer']) + expect(description(main.calls[0])).toEqual(['Transfer from main']) + expect(toolNames(billing.model.calls[0])).toEqual(['transfer']) + expect(description(billing.model.calls[0])).toEqual([ + 'Transfer from billing', + ]) + expect( + (await handoffRecords(log)).map(({ from, to, note, by }) => ({ + from, + to, + note, + by, + })), + ).toEqual([ + { from: null, to: 'billing', note: 'main: order 42', by: 'tool' }, + ]) await host.close() }) @@ -559,11 +606,14 @@ describe('handoffs', () => { ...call(1, undefined, { to: 'triage' }), ...call(2, 'triage', { to: 'billing', note: 'A refund' }), ]) - // The target gets the note of its handoff last. - expect(billing.model.calls[0].messages.slice(-2)).toMatchObject([ - { role: 'assistant', toolCalls: [{ id: `handoff_${inputId}_2` }] }, - { role: 'tool', toolCallId: `handoff_${inputId}_2` }, - ]) + // The target gets the note of its handoff last, as a text line in a user + // message: with `routing.history: 'own'`, a call of another speaker is + // text, and the context ends on a user turn. + expect(billing.model.calls[0].messages.at(-1)).toMatchObject({ + id: `handoff_${inputId}_2`, + role: 'user', + content: 'triage handed off to billing. Note: A refund', + }) await host.close() }) @@ -605,7 +655,11 @@ describe('handoffs', () => { role: 'assistant', content: 'Handed off to billing. billing answers next.', } - expect(billing.model.calls[0].messages.at(-1)).toMatchObject(note) + // The target gets the plain note as text of another speaker. + expect(billing.model.calls[0].messages.at(-1)).toMatchObject({ + ...note, + role: 'user', + }) expect( (await session.transcript()).filter((message) => message.id?.startsWith('handoff_'), diff --git a/packages/ai-harness/tests/harness-agent-root.test.ts b/packages/ai-harness/tests/harness-agent-root.test.ts new file mode 100644 index 0000000000..a12aa5d8cf --- /dev/null +++ b/packages/ai-harness/tests/harness-agent-root.test.ts @@ -0,0 +1,260 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { memoryLogStore, memoryPersistence } from '@tanstack/ai-persistence' +import { createHarnessHost, defineHarness, harnessAgent } from '../src' +import { mockAdapter, storedThreadState, text, toolCall } from './helpers' +import type { ModelMessage } from '@tanstack/ai' +import type { AnyHarness, HarnessRouting } from '../src' +import type { Reply } from './helpers' + +const THREAD = 't1' + +/** A tool of the child harness, so its model has a tool call to keep. */ +const lookup = toolDefinition({ + name: 'lookup', + description: 'Looks a thing up', + inputSchema: z.object({}), +}).server(async () => 'found') + +/** The child harness of the `desk` root agent, and its model. */ +function deskHarness(replies: Array) { + const model = mockAdapter(replies) + const harness = defineHarness({ + name: 'test/desk', + adapter: model.adapter, + tools: [lookup], + }) + return { harness, model } +} + +/** The stores of a host. A second host on the same stores reads the same log. */ +function stores() { + return { + log: memoryLogStore(), + metadata: memoryPersistence().stores.metadata, + } +} + +/** + * Open the routed thread on a new host. Its only root agent is the + * harnessAgent `desk` of `child`. + */ +async function openRouted(options: { + child: AnyHarness + router: HarnessRouting['router'] + mainReplies?: Array + stores?: ReturnType +}) { + const { log, metadata } = options.stores ?? stores() + const main = mockAdapter(options.mainReplies ?? []) + const host = createHarnessHost({ persistence: { stores: { log, metadata } } }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: main.adapter, + agents: [harnessAgent(options.child, { name: 'desk' })], + routing: { router: options.router }, + }), + { threadId: THREAD }, + ) + return { host, session, main, log } +} + +/** + * The text that the user sent in each user message of one model call: the + * string, or the last text part. The parts before it are the context. + */ +const userTexts = (call: { messages: Array }) => + call.messages + .filter((message) => message.role === 'user') + .map((message) => + typeof message.content === 'string' + ? message.content + : (message.content ?? []).findLast((part) => part.type === 'text') + ?.content, + ) + +/** The names of the tool calls in the messages of one model call. */ +const calledTools = (call: { messages: Array }) => + call.messages.flatMap((message) => + (message.toolCalls ?? []).map((entry) => entry.function.name), + ) + +/** A router that picks `picks[n]` on turn `n`. */ +function picksInOrder(picks: Array) { + let turn = 0 + return () => picks[turn++] +} + +describe('harnessAgent as a routed root agent', () => { + it('keeps one child session, so turn 2 sees turn 1 and its own tool call', async () => { + const desk = deskHarness([ + () => toolCall('lookup', {}, 'call-lookup'), + () => text('Looked it up.'), + () => text('Second answer.'), + ]) + const { host, session } = await openRouted({ + child: desk.harness, + router: () => 'desk', + }) + + await session.send('first question') + await session.send('second question') + + const turn2 = desk.model.calls[2] + expect(userTexts(turn2)).toEqual(['first question', 'second question']) + expect(calledTools(turn2)).toEqual(['lookup']) + expect( + turn2.messages.some( + (message: ModelMessage) => + message.role === 'tool' && message.content === 'found', + ), + ).toBe(true) + await host.close() + }) + + it('sends what main said since its last turn, labeled main:', async () => { + const desk = deskHarness([ + () => text('Desk one.'), + () => text('Desk three.'), + ]) + const { host, session } = await openRouted({ + child: desk.harness, + router: picksInOrder(['desk', 'main', 'desk']), + mainReplies: [() => text('Main two.')], + }) + + await session.send('first') + await session.send('second') + await session.send('third') + + const turn3 = desk.model.calls[1] + const input = turn3.messages.at(-1) + expect(input.role).toBe('user') + expect(input.content).toEqual([ + { + type: 'text', + content: [ + 'Since your last turn:', + // Its own hand-back is in its own session, so it is not here. + 'user:\nsecond', + 'main:\nMain two.', + 'main handed off to desk.', + ].join('\n\n'), + }, + { type: 'text', content: 'third' }, + ]) + // The earlier turn of the desk stays in its own session once. + expect(userTexts(turn3).filter((entry) => entry === 'first')).toHaveLength( + 1, + ) + await host.close() + }) + + it('hands back to main with the handoff tool, and main stays active', async () => { + const desk = deskHarness([() => toolCall('handoff', { to: 'main' })]) + const { host, session, main, log } = await openRouted({ + child: desk.harness, + router: picksInOrder(['desk', undefined]), + mainReplies: [() => text('Main took over.'), () => text('Main again.')], + }) + + const first = await session.send('help') + + // The child model has the outer handoff tool, to main only. + const tool = desk.model.calls[0].tools.find( + (entry: { name: string }) => entry.name === 'handoff', + ) + expect(tool.inputSchema.properties.to).toEqual({ + type: 'string', + enum: ['main'], + }) + // The child stopped after the handoff, and main answered in the turn. + expect(desk.model.calls).toHaveLength(1) + expect(first.text).toContain('Main took over.') + // The call is in the child thread, so the shared thread gets the note. + // Main reads who handed the conversation back, as text of another + // speaker, not as a call of its own. + const transcript = await session.transcript() + expect( + transcript.find( + (message) => + message.role === 'tool' && + message.content === 'Handed off to main. main answers next.', + )?.metadata?.harness, + ).toEqual({ agent: 'desk', handoff: { by: 'tool' } }) + expect(main.calls[0].messages.at(-1)).toMatchObject({ + role: 'user', + content: 'desk handed off to main.', + }) + expect( + (await storedThreadState(log, { threadId: THREAD })).activeAgent, + ).toBeNull() + + expect((await session.send('more')).text).toBe('Main again.') + expect(desk.model.calls).toHaveLength(1) + expect(main.calls).toHaveLength(2) + await host.close() + }) + + it('fails the turn when the child harness has its own routing', async () => { + const model = mockAdapter([() => text('Never.')]) + const child = defineHarness({ + name: 'test/routed-child', + adapter: model.adapter, + agents: [ + defineAgent({ + name: 'inner', + description: 'An inner agent', + run: async () => 'Inner.', + }), + ], + routing: { router: () => undefined }, + }) + const { host, session } = await openRouted({ + child, + router: () => 'desk', + }) + + await expect(session.send('hi')).rejects.toThrow( + 'A harnessAgent that is a root agent cannot have its own routing. Remove routing from "test/routed-child".', + ) + expect(model.calls).toHaveLength(0) + await host.close() + }) + + it('opens the kept child thread again on a new host', async () => { + const desk = deskHarness([ + () => toolCall('lookup', {}, 'call-lookup'), + () => text('Looked it up.'), + () => text('Second answer.'), + () => text('Third answer.'), + ]) + const kept = stores() + const before = await openRouted({ + child: desk.harness, + router: () => 'desk', + stores: kept, + }) + await before.session.send('first question') + await before.session.send('second question') + await before.host.close() + + const after = await openRouted({ + child: desk.harness, + router: () => 'desk', + stores: kept, + }) + await after.session.send('third question') + + const turn3 = desk.model.calls[3] + expect(userTexts(turn3)).toEqual([ + 'first question', + 'second question', + 'third question', + ]) + expect(calledTools(turn3)).toEqual(['lookup']) + await after.host.close() + }) +}) diff --git a/packages/ai-harness/tests/prompt-cache.test.ts b/packages/ai-harness/tests/prompt-cache.test.ts index 07c6df4b46..12ee718b7a 100644 --- a/packages/ai-harness/tests/prompt-cache.test.ts +++ b/packages/ai-harness/tests/prompt-cache.test.ts @@ -117,7 +117,9 @@ describe('prompt cache', () => { adapter: main.adapter, tools: [escalate], agents: [billing], - routing: { router: () => undefined }, + // 'text' keeps the main model's messages as they are, so the target + // has them as its prefix. 'own' folds them to text for the target. + routing: { router: () => undefined, history: 'text' }, }), { threadId: 't' }, ) diff --git a/packages/ai-harness/tests/routed-history.test.ts b/packages/ai-harness/tests/routed-history.test.ts new file mode 100644 index 0000000000..7505701bdc --- /dev/null +++ b/packages/ai-harness/tests/routed-history.test.ts @@ -0,0 +1,356 @@ +import { describe, expect, it } from 'vitest' +import { historyFor, routedHistoryMiddleware } from '../src/routed-history' +import type { ModelMessage } from '@tanstack/ai' + +type Child = { name: string; subagentRunId: string } + +const CHILDREN: Record> = { + r1: [{ name: 'billing', subagentRunId: 's1' }], + r2: [{ name: 'research', subagentRunId: 's2' }], +} + +const BILLING_TRANSCRIPT: Array = [ + { id: 'task1', role: 'user', content: 'Refund order 42' }, + { + id: 'b1', + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'call_refund', + type: 'function', + function: { name: 'refund', arguments: '{"order":42}' }, + }, + ], + }, + { id: 'b2', role: 'tool', toolCallId: 'call_refund', content: 'refunded' }, + { id: 'b3', role: 'assistant', content: 'Order 42 is refunded.' }, +] + +const TRANSCRIPTS: Record> = { + s1: BILLING_TRANSCRIPT, + s2: [ + { id: 'task2', role: 'user', content: 'Find papers' }, + { id: 'c1', role: 'assistant', content: 'Found 3 papers.' }, + ], +} + +function host(runId: string, agent: string, text: string): ModelMessage { + return { + id: `assistant:${runId}`, + role: 'assistant', + content: `${agent}:\n${text}`, + metadata: { tanstack: { runId }, harness: { agent } }, + } +} + +const USER: ModelMessage = { id: 'u1', role: 'user', content: 'Hi' } +const BILLING_HOST = host('r1', 'billing', 'Order 42 is refunded.') +const RESEARCH_HOST = host('r2', 'research', 'Found 3 papers.') + +function handoffCall( + id: string, + name: string, + args: Record, +): ModelMessage { + return { + id: `m_${id}`, + role: 'assistant', + content: 'Let me pass you on.', + toolCalls: [ + { + id, + type: 'function', + function: { name, arguments: JSON.stringify(args) }, + }, + ], + } +} + +function toolResult(toolCallId: string): ModelMessage { + return { + id: `${toolCallId}_result`, + role: 'tool', + toolCallId, + content: 'Handed off to billing. billing answers next.', + } +} + +function history(options: { + messages: Array + speaker: string | null + mode: 'own' | 'all' | 'text' + handoffToolName?: string + transcripts?: Record> + children?: Record> + isThreadMessage?: (message: ModelMessage) => boolean +}) { + const transcripts = options.transcripts ?? TRANSCRIPTS + const children = options.children ?? CHILDREN + return historyFor({ + ...options, + childrenOf: (runId) => children[runId] ?? [], + transcriptOf: (subagentRunId) => + Promise.resolve(transcripts[subagentRunId] ?? []), + }) +} + +describe('historyFor', () => { + it("returns the messages unchanged in 'text' mode", async () => { + const messages = [USER, BILLING_HOST, RESEARCH_HOST] + const result = await history({ messages, speaker: 'billing', mode: 'text' }) + expect(result).toEqual([USER, BILLING_HOST, RESEARCH_HOST]) + }) + + it("gives a routed agent its own runs in full and others as text in 'own' mode", async () => { + const result = await history({ + messages: [USER, BILLING_HOST, RESEARCH_HOST], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + USER, + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + // Another agent's answer: a user message, so billing does not read + // it as its own. + { ...RESEARCH_HOST, role: 'user' }, + ]) + }) + + it("folds a main handoff call to text for a routed agent in 'own' mode", async () => { + const call = handoffCall('call_h', 'handoff', { + to: 'billing', + note: 'refund for order 42', + }) + const result = await history({ + messages: [USER, call, toolResult('call_h')], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + USER, + { + id: 'm_call_h', + role: 'user', + content: + 'main:\nLet me pass you on.\n\nmain handed off to billing. Note: refund for order 42', + }, + ]) + }) + + it('folds a handoff to a person and drops the other tool calls of main', async () => { + const call: ModelMessage = { + id: 'm2', + role: 'assistant', + content: '', + toolCalls: [ + { + id: 'call_w', + type: 'function', + function: { name: 'weather', arguments: '{}' }, + }, + { + id: 'call_p', + type: 'function', + function: { + name: 'handoff', + arguments: JSON.stringify({ to: { human: true } }), + }, + }, + ], + } + const result = await history({ + messages: [call, toolResult('call_w'), toolResult('call_p')], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + { id: 'm2', role: 'user', content: 'main handed off to a person.' }, + ]) + }) + + it('reads the handoff tool name from handoffToolName', async () => { + const custom = handoffCall('call_t', 'transfer', { to: 'billing' }) + const builtIn = handoffCall('call_b', 'handoff', { to: 'research' }) + const result = await history({ + messages: [custom, toolResult('call_t'), builtIn, toolResult('call_b')], + speaker: 'billing', + mode: 'own', + handoffToolName: 'transfer', + }) + expect(result).toEqual([ + { + id: 'm_call_t', + role: 'user', + content: 'main:\nLet me pass you on.\n\nmain handed off to billing.', + }, + { + id: 'm_call_b', + role: 'user', + content: 'main:\nLet me pass you on.', + }, + ]) + }) + + it("keeps the main model's own calls and gives it routed answers as user text in 'own' mode", async () => { + const call = handoffCall('call_h', 'handoff', { to: 'billing' }) + const messages = [USER, call, toolResult('call_h'), BILLING_HOST] + const result = await history({ messages, speaker: null, mode: 'own' }) + expect(result).toEqual([ + USER, + call, + toolResult('call_h'), + { ...BILLING_HOST, role: 'user' }, + ]) + }) + + it("expands every routed run for the main model in 'all' mode", async () => { + const call = handoffCall('call_h', 'handoff', { to: 'billing' }) + const result = await history({ + messages: [USER, call, toolResult('call_h'), BILLING_HOST, RESEARCH_HOST], + speaker: null, + mode: 'all', + }) + expect(result).toEqual([ + USER, + call, + toolResult('call_h'), + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + TRANSCRIPTS.s2?.[1], + ]) + }) + + it('expands a kept earlier pass of a run by its harness routedRunId', async () => { + // One run with two passes: billing's pass moved to its own id and lost + // its tanstack run id. research wrote the last pass. + const keptPass: ModelMessage = { + id: 'assistant:r3:1', + role: 'assistant', + content: 'billing:\nOrder 42 is refunded.', + metadata: { + tanstack: {}, + harness: { agent: 'billing', routedRunId: 'r3' }, + }, + } + const lastPass = host('r3', 'research', 'Found 3 papers.') + const result = await history({ + messages: [USER, keptPass, lastPass], + speaker: 'billing', + mode: 'own', + children: { + r3: [ + { name: 'billing', subagentRunId: 's1' }, + { name: 'research', subagentRunId: 's2' }, + ], + }, + }) + expect(result).toEqual([ + USER, + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + { ...lastPass, role: 'user' }, + ]) + }) + + it('keeps the messages that are not in the thread as they are', async () => { + const ownCall = handoffCall('call_own', 'lookup', {}) + const result = await history({ + messages: [USER, ownCall, toolResult('call_own')], + speaker: 'billing', + mode: 'own', + isThreadMessage: (message) => message.id === USER.id, + }) + expect(result).toEqual([USER, ownCall, toolResult('call_own')]) + }) + + it('keeps the host message when the run has no stored transcript', async () => { + const result = await history({ + messages: [USER, BILLING_HOST], + speaker: 'billing', + mode: 'own', + transcripts: {}, + }) + expect(result).toEqual([USER, BILLING_HOST]) + }) + + it('folds a router note pair of another agent to one text line', async () => { + const metadata = { + harness: { agent: 'billing', handoff: { by: 'router' } }, + } + const note: ModelMessage = { + id: 'n1', + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'n1', + type: 'function', + function: { + name: 'handoff', + arguments: JSON.stringify({ to: 'main', note: 'needs a human' }), + }, + }, + ], + metadata, + } + const result = await history({ + messages: [USER, note, { ...toolResult('n1'), metadata }], + speaker: 'research', + mode: 'own', + }) + expect(result).toEqual([ + USER, + { + id: 'n1', + role: 'user', + content: 'billing handed off to main. Note: needs a human', + metadata, + }, + ]) + }) +}) + +describe('routedHistoryMiddleware', () => { + const middleware = routedHistoryMiddleware({ + mode: 'own', + childrenOf: (runId) => CHILDREN[runId] ?? [], + transcriptOf: (subagentRunId) => + Promise.resolve(TRANSCRIPTS[subagentRunId] ?? []), + }) + const config = { messages: [USER, BILLING_HOST] } + + it('sets providerMessages for a routed agent', async () => { + const change = await middleware.onConfig( + { subagentName: 'billing' }, + config, + ) + expect(change).toEqual({ + providerMessages: [ + USER, + BILLING_TRANSCRIPT[1], + BILLING_TRANSCRIPT[2], + BILLING_TRANSCRIPT[3], + ], + }) + }) + + it("gives the main model a routed answer as user text in 'own' mode", async () => { + const change = await middleware.onConfig({}, config) + expect(change).toEqual({ + providerMessages: [USER, { ...BILLING_HOST, role: 'user' }], + }) + }) + + it('changes nothing for a nested child of a routed agent', async () => { + const change = await middleware.onConfig( + { subagentName: 'billing', parentSubagentRunId: 's1' }, + config, + ) + expect(change).toBeUndefined() + }) +}) diff --git a/packages/ai-harness/tests/routing-history.test.ts b/packages/ai-harness/tests/routing-history.test.ts new file mode 100644 index 0000000000..5b8c7025a7 --- /dev/null +++ b/packages/ai-harness/tests/routing-history.test.ts @@ -0,0 +1,274 @@ +import { memoryCredentialStore } from '@tanstack/ai/byok/server' +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' +import { memoryLogStore, memoryPersistence } from '@tanstack/ai-persistence' +import { createHarnessHost, defineHarness } from '../src' +import { messageTexts, mockAdapter, text, toolCall } from './helpers' +import type { AnyTool, ModelMessage } from '@tanstack/ai' +import type { AnyAgent, HarnessRouting } from '../src' +import type { Reply } from './helpers' + +/** A session whose harness routes, with `routing.history`. */ +async function openRouting(options: { + agents: ReadonlyArray + router: HarnessRouting['router'] + history?: HarnessRouting['history'] + replies?: Array +}) { + const { metadata } = memoryPersistence().stores + const main = mockAdapter(options.replies ?? []) + const host = createHarnessHost({ + persistence: { stores: { log: memoryLogStore(), metadata } }, + credentials: memoryCredentialStore(), + }) + const session = await host.open( + defineHarness({ + name: 'test/routing-history', + adapter: main.adapter, + agents: options.agents, + routing: { + router: options.router, + ...(options.history ? { history: options.history } : {}), + }, + }), + { threadId: 't1', user: { id: 'user-1' } }, + ) + return { host, session, main } +} + +/** An agent that answers with its own model and tools. */ +function modelAgent( + name: string, + replies: Array, + tools: ReadonlyArray = [], +) { + const model = mockAdapter(replies) + const agent = defineAgent({ + name, + description: `The ${name} desk`, + run: (ctx) => ctx.chat({ adapter: model.adapter, tools: [...tools] }), + }) + return { agent, model } +} + +/** A tool that answers `ok`. */ +function plainTool(name: string) { + return toolDefinition({ + name, + description: `The ${name} tool`, + inputSchema: z.object({}), + }).server(async () => 'ok') +} + +/** A router that picks an agent for some inputs, else the active agent. */ +const routeBy = + (picks: Record): HarnessRouting['router'] => + ({ input }) => + typeof input === 'string' ? picks[input] : undefined + +/** The names of the tool calls in the messages of one model call. */ +function calledTools(call: { messages: Array }) { + return call.messages.flatMap((message) => + (message.toolCalls ?? []).map((made) => made.function.name), + ) +} + +/** The tool call ids that have a tool result in one model call. */ +function answeredCalls(call: { messages: Array }) { + return call.messages.flatMap((message) => + message.role === 'tool' && message.toolCallId !== undefined + ? [message.toolCallId] + : [], + ) +} + +/** billing calls `refund`, then answers. A later turn answers `Welcome`. */ +function refundingBilling() { + return modelAgent( + 'billing', + [ + () => toolCall('refund', {}, 'call_r'), + () => text('Refunded'), + () => text('Welcome'), + ], + [plainTool('refund')], + ) +} + +describe('routing.history', () => { + it("'own' gives a routed agent its own earlier tool calls", async () => { + const billing = refundingBilling() + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing' }), + }) + + await session.send('refund please') + expect((await session.send('thanks')).text).toBe('billing:\nWelcome') + + const later = billing.model.calls[2] + expect(calledTools(later)).toContain('refund') + expect(answeredCalls(later)).toContain('call_r') + expect(messageTexts(later)).toContain('Refunded') + expect(messageTexts(later)).not.toContain('billing:\nRefunded') + await host.close() + }) + + it("'own' gives the main model a routed run as text only", async () => { + const billing = refundingBilling() + const { host, session, main } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing', 'ask main': 'main' }), + replies: [() => text('Main here')], + }) + + await session.send('refund please') + expect((await session.send('ask main')).text).toBe('Main here') + + const [call] = main.calls + expect(calledTools(call)).not.toContain('refund') + expect(messageTexts(call)).toContain('billing:\nRefunded') + await host.close() + }) + + it("'all' gives the main model the tool calls of a routed run", async () => { + const billing = refundingBilling() + const { host, session, main } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing', 'ask main': 'main' }), + history: 'all', + replies: [() => text('Main here')], + }) + + await session.send('refund please') + await session.send('ask main') + + const [call] = main.calls + expect(calledTools(call)).toContain('refund') + expect(answeredCalls(call)).toContain('call_r') + expect(messageTexts(call)).not.toContain('billing:\nRefunded') + await host.close() + }) + + it("'text' gives a routed agent its earlier run as text only", async () => { + const billing = refundingBilling() + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'refund please': 'billing' }), + history: 'text', + }) + + await session.send('refund please') + await session.send('thanks') + + const later = billing.model.calls[2] + expect(calledTools(later)).not.toContain('refund') + expect(messageTexts(later)).toContain('billing:\nRefunded') + await host.close() + }) + + it("'own' gives a routed agent the handoff of main as a text line", async () => { + const billing = modelAgent('billing', [ + () => text('Refunded'), + () => text('Welcome'), + ]) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: () => undefined, + replies: [ + () => + toolCall('handoff', { to: 'billing', note: 'order 42' }, 'call_h'), + ], + }) + + expect((await session.send('refund please')).text).toBe( + 'billing:\nRefunded', + ) + await session.send('thanks') + + const later = billing.model.calls[1] + expect(answeredCalls(later)).not.toContain('call_h') + expect(messageTexts(later)).toContain( + 'main handed off to billing. Note: order 42', + ) + await host.close() + }) + + it('gives each pass of a turn with two routed passes to its own agent', async () => { + const billing = modelAgent( + 'billing', + [ + () => toolCall('refund', {}, 'call_r'), + () => [ + ...text('Passing you to research.'), + ...toolCall('handoff', { to: 'research' }, 'call_hb'), + ], + () => text('Back in billing'), + ], + [plainTool('refund')], + ) + const research = modelAgent( + 'research', + [() => toolCall('search', {}, 'call_s'), () => text('Found 3 papers.')], + [plainTool('search')], + ) + const { host, session } = await openRouting({ + agents: [billing.agent, research.agent], + router: routeBy({ + 'refund please': 'billing', + 'back to billing': 'billing', + }), + }) + + expect((await session.send('refund please')).text).toBe( + 'research:\nFound 3 papers.', + ) + await session.send('back to billing') + + const later = billing.model.calls[2] + expect(calledTools(later)).toContain('refund') + expect(calledTools(later)).not.toContain('search') + expect(answeredCalls(later)).toEqual( + expect.arrayContaining(['call_r', 'call_hb']), + ) + expect(messageTexts(later)).toContain('research:\nFound 3 papers.') + await host.close() + }) + + it('gives a resumed routed agent the messages of its paused run', async () => { + // No server implementation: the turn waits for the result of the client. + const openScreen = toolDefinition({ + name: 'openScreen', + description: 'Open a screen', + inputSchema: z.object({}), + }) + const billing = modelAgent( + 'billing', + [() => toolCall('openScreen', {}, 'call_o'), () => text('Screen open')], + [openScreen], + ) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ 'open it': 'billing' }), + }) + + const [interrupt] = (await session.send('open it')).interrupts ?? [] + if (!interrupt) throw new Error('No client tool interrupt.') + await session.resolve({ + resume: [{ interruptId: interrupt.id, status: 'resolved', payload: {} }], + inputId: 'opened', + }) + expect(await session.settled({ inputId: 'opened' })).toMatchObject({ + outcome: 'completed', + }) + + const resumed = billing.model.calls[1] + const opened = calledTools(resumed).filter((name) => name === 'openScreen') + expect(opened).toEqual(['openScreen']) + const answered = answeredCalls(resumed).filter((id) => id === 'call_o') + expect(answered).toEqual(['call_o']) + expect(messageTexts(resumed)).toContain('open it') + await host.close() + }) +}) diff --git a/testing/e2e/src/routes/api.harness-handoff.$.ts b/testing/e2e/src/routes/api.harness-handoff.$.ts index b80998cc15..55b067f7ab 100644 --- a/testing/e2e/src/routes/api.harness-handoff.$.ts +++ b/testing/e2e/src/routes/api.harness-handoff.$.ts @@ -1,5 +1,6 @@ import { createFileRoute } from '@tanstack/react-router' -import { defineAgent } from '@tanstack/ai' +import { z } from 'zod' +import { defineAgent, toolDefinition } from '@tanstack/ai' import { createAnthropicChat } from '@tanstack/ai-anthropic' import { HARNESS_EVENTS, @@ -12,6 +13,7 @@ import { createSessionView } from '@tanstack/ai-harness/view' import { memoryPersistence } from '@tanstack/ai-persistence' import type { HarnessHost, + HarnessRouting, HarnessSession, HarnessUser, } from '@tanstack/ai-harness' @@ -19,8 +21,8 @@ import type { SessionView } from '@tanstack/ai-harness/view' /** * Handoffs inside one session, over the harness protocol. The harness has - * `routing` and one root agent, `billing`. The host has `membersOnly()` - * access: olga owns the thread, and bea is an operator. + * `routing` and one root agent, `billing`, with a `refund` tool. The host + * has `membersOnly()` access: olga owns the thread, and bea is an operator. * * - `POST .../start` with `{ scenario }` opens the thread for olga and * shares it with bea. Returns the run id and the thread id. @@ -29,6 +31,10 @@ import type { SessionView } from '@tanstack/ai-harness/view' * the `handoff` tool to a person. * - `'code'`: the router picks the `triage` agent, and its code hands off * to `billing` with `ctx.harness.handoff`. The main model never runs. + * - `'history'`: the main model hands off to `billing`. On turn 1, + * `billing` calls `refund`, then answers. On turn 2, `billing` answers. + * - `'custom-tool'`: the harness has `routing.handoffTool`. Its tool adds + * `from : ` before the note, and the main model calls it. * - `POST .../control?runId=` and `GET .../snapshot?runId=` go to the * harness handler. The `authorization` header names the user. * - `GET .../settled?runId=&inputId=` waits until an input ends, and @@ -36,7 +42,7 @@ import type { SessionView } from '@tanstack/ai-harness/view' * - `GET .../debug?runId=` returns what the test checks: the * `harness.handoff` events of the session stream, the transcript, the * messages of a session view, the open waits, the model calls, and the - * messages of the first request of `billing`. + * messages of each request of `billing`. * - `POST .../stop?runId=` closes the host. * * A custom `fetch` answers with canned Anthropic streams, so no aimock @@ -78,27 +84,31 @@ const ending = (stopReason: string): Array => [ { type: 'message_stop' }, ] -/** An answer that calls the `handoff` tool with `input`. */ -const callHandoff = (input: Record) => (id: string) => - sseOf([ - messageStart(id), - { - type: 'content_block_start', - index: 0, - content_block: { - type: 'tool_use', - id: `toolu_${id}`, - name: 'handoff', - input: {}, +/** An answer that calls the tool `name` with `input`. */ +const callTool = + (name: string, input: Record) => (id: string) => + sseOf([ + messageStart(id), + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: `toolu_${id}`, + name, + input: {}, + }, }, - }, - { - type: 'content_block_delta', - index: 0, - delta: { type: 'input_json_delta', partial_json: JSON.stringify(input) }, - }, - ...ending('tool_use'), - ]) + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'input_json_delta', + partial_json: JSON.stringify(input), + }, + }, + ...ending('tool_use'), + ]) /** An answer with the text `text`. */ const reply = (text: string) => (id: string) => @@ -138,7 +148,7 @@ function scriptedModel(prefix: string, answers: Array<(id: string) => string>) { return { adapter, counter, requests } } -type Scenario = 'agent' | 'human' | 'code' +type Scenario = 'agent' | 'human' | 'code' | 'history' | 'custom-tool' /** The answers of the main model and of `billing`, per scenario. */ const SCRIPTS: Record< @@ -149,12 +159,17 @@ const SCRIPTS: Record< } > = { agent: { - main: [callHandoff({ to: 'billing', note: 'Refund for order 42.' })], + main: [ + callTool('handoff', { to: 'billing', note: 'Refund for order 42.' }), + ], billing: [reply('Your refund is on its way.'), reply('Billing again.')], }, human: { main: [ - callHandoff({ to: { human: true }, note: 'The user asks for a person.' }), + callTool('handoff', { + to: { human: true }, + note: 'The user asks for a person.', + }), reply('The main model is back.'), ], billing: [], @@ -163,8 +178,48 @@ const SCRIPTS: Record< main: [], billing: [reply('Your refund is on its way.')], }, + history: { + main: [ + callTool('handoff', { to: 'billing', note: 'Refund for order 42.' }), + ], + billing: [ + callTool('refund', { orderId: '42' }), + reply('Refunded order 42.'), + reply('Anything else?'), + ], + }, + 'custom-tool': { + main: [ + callTool('handoff', { to: 'billing', note: 'Refund for order 42.' }), + ], + billing: [reply('Your refund is on its way.')], + }, } +/** The tool of `billing`. It always gives the same result. */ +const refund = toolDefinition({ + name: 'refund', + description: 'Refund an order', + inputSchema: z.object({ orderId: z.string() }), +}).server(async ({ orderId }) => `Refund R-${orderId} issued.`) + +/** + * The handoff tool of the `'custom-tool'` harness. It adds the speaker + * before the note, and gives a fixed result. + */ +const prefixedHandoff: NonNullable = ({ + from, + handoff, +}) => + toolDefinition({ + name: 'handoff', + description: 'Hand the user to another agent', + inputSchema: z.object({ to: z.string(), note: z.string() }), + }).server(async ({ to, note }) => { + await handoff({ to, note: `from ${from ?? 'main'}: ${note}` }) + return 'Passed to the next desk.' + }) + /** An agent with no model: its code hands the turn to `billing`. */ const triage = defineAgent({ name: 'triage', @@ -221,6 +276,7 @@ async function start(request: Request) { ctx.chat({ adapter: billing.adapter, systemPrompts: ['You are billing.'], + tools: [refund], }), }), ], @@ -229,6 +285,7 @@ async function start(request: Request) { routing: { router: () => (scenario === 'code' ? 'triage' : undefined), allowHandoffToHuman: scenario === 'human', + ...(scenario === 'custom-tool' ? { handoffTool: prefixedHandoff } : {}), }, }) const host: HarnessHost = createHarnessHost({ @@ -309,7 +366,9 @@ async function debug(run: Run) { waits: snapshot.pendingWaits, mainCalls: run.main.counter.calls, billingCalls: run.billing.counter.calls, - billingMessages: run.billing.requests[0]?.messages ?? [], + billingRequests: run.billing.requests.map( + (request) => request.messages ?? [], + ), }) } diff --git a/testing/e2e/tests/harness-handoff.spec.ts b/testing/e2e/tests/harness-handoff.spec.ts index 4f4971cd23..0c692aa247 100644 --- a/testing/e2e/tests/harness-handoff.spec.ts +++ b/testing/e2e/tests/harness-handoff.spec.ts @@ -9,6 +9,18 @@ interface Started { threadId: string } +/** A content block of an Anthropic message. */ +interface Block { + type: string + id?: string + name?: string + input?: unknown + tool_use_id?: string +} + +/** The messages of one model request, as Anthropic gets them. */ +type RequestMessages = Array<{ role: string; content: string | Array }> + interface Debug { error?: string status: string @@ -22,15 +34,14 @@ interface Debug { waits: Array<{ waitId: string; kind: string; message?: string }> mainCalls: number billingCalls: number - /** The messages of the first request of `billing`, as Anthropic gets them. */ - billingMessages: Array<{ role: string; content: unknown }> + /** The messages of each request of `billing`, in order. */ + billingRequests: Array } +type Scenario = 'agent' | 'human' | 'code' | 'history' | 'custom-tool' + /** Open the thread of `scenario` for olga, shared with bea. */ -async function start( - request: APIRequestContext, - scenario: 'agent' | 'human' | 'code', -) { +async function start(request: APIRequestContext, scenario: Scenario) { const res = await request.post(`${ROUTE}/start`, { data: { scenario } }) expect(res.ok()).toBe(true) const started: Started = await res.json() @@ -110,6 +121,12 @@ const lines = (read: Debug) => by: harness?.agent ?? harness?.operator ?? null, })) +/** The content blocks of the messages of one request. */ +const blocksOf = (messages: RequestMessages) => + messages.flatMap(({ content }) => + typeof content === 'string' ? [] : content, + ) + const stop = (request: APIRequestContext, runId: string) => request.post(`${ROUTE}/stop?runId=${runId}`) @@ -214,23 +231,20 @@ test.describe('harness: handoffs inside one session', () => { ]) expect(read.mainCalls).toBe(0) expect(read.billingCalls).toBe(1) - // The model of billing gets the note of its handoff last: a `handoff` - // tool call with the note, and its result. - const [noteCall, noteResult] = read.billingMessages.slice(-2) - expect(noteCall).toMatchObject({ - role: 'assistant', - content: expect.arrayContaining([ - expect.objectContaining({ - type: 'tool_use', - name: 'handoff', - input: { to: 'billing', note: 'Refund for order 42.' }, - }), - ]), - }) - expect(noteResult).toMatchObject({ role: 'user' }) - expect(JSON.stringify(noteResult)).toContain( - 'Handed off to billing. billing answers next.', + // The model of billing gets the note of its handoff last, as text of + // another speaker (`routing.history: 'own'`): a user message, with no + // `handoff` call that billing could read as its own. + const sentToBilling = read.billingRequests[0]! + const noteMessage = sentToBilling.at(-1) + expect(noteMessage).toMatchObject({ role: 'user' }) + expect(JSON.stringify(noteMessage)).toContain( + 'main handed off to billing. Note: Refund for order 42.', ) + expect( + blocksOf(sentToBilling).some( + (block) => block.type === 'tool_use' && block.name === 'handoff', + ), + ).toBe(false) expect(lines(read).at(-1)).toEqual({ role: 'assistant', text: 'billing:\nYour refund is on its way.', @@ -352,4 +366,107 @@ test.describe('harness: handoffs inside one session', () => { await stop(request, started.runId) } }) + + test('a routed agent sees its own tool calls of an earlier turn', async ({ + request, + }) => { + const started = await start(request, 'history') + try { + await control(request, started, { + op: 'prompt', + message: 'refund please', + inputId: 'in-1', + }) + expect(await outcomeOf(request, started.runId, 'in-1')).toBe('completed') + await control(request, started, { + op: 'prompt', + message: 'and now?', + inputId: 'in-2', + }) + expect(await outcomeOf(request, started.runId, 'in-2')).toBe('completed') + + const read = await readUntil( + request, + started.runId, + (done) => done.billingCalls === 3, + ) + expect(read.mainCalls).toBe(1) + expect(lines(read).slice(3)).toEqual([ + { + role: 'assistant', + text: 'billing:\nRefunded order 42.', + by: 'billing', + }, + { role: 'user', text: 'and now?', by: null }, + { role: 'assistant', text: 'billing:\nAnything else?', by: 'billing' }, + ]) + + // On turn 2, billing gets its turn 1 `refund` call and its result. + const later = read.billingRequests[2]! + const blocks = blocksOf(later) + const refundCall = blocks.find( + (block) => block.type === 'tool_use' && block.name === 'refund', + ) + expect(refundCall).toMatchObject({ input: { orderId: '42' } }) + const refundResult = blocks.find( + (block) => + block.type === 'tool_result' && block.tool_use_id === refundCall?.id, + ) + expect(JSON.stringify(refundResult)).toContain('Refund R-42 issued.') + // Its own run is not the `billing:` text of the host message. The + // handoff call of main is a text line with the note. JSON writes the + // newline as a backslash and `n`. + const sent = JSON.stringify(later) + expect(sent).not.toContain('billing:\\nRefunded order 42.') + expect(sent).toContain( + 'main handed off to billing. Note: Refund for order 42.', + ) + } finally { + await stop(request, started.runId) + } + }) + + test('a handoff tool from routing.handoffTool hands off with its own note', async ({ + request, + }) => { + const started = await start(request, 'custom-tool') + try { + await control(request, started, { + op: 'prompt', + message: 'refund please', + inputId: 'in-1', + }) + expect(await outcomeOf(request, started.runId, 'in-1')).toBe('completed') + + const read = await readUntil(request, started.runId, (done) => + done.view.some((message) => message.agent === 'billing'), + ) + expect(read.handoffs).toEqual([ + { + from: null, + to: 'billing', + note: 'from main: Refund for order 42.', + by: 'tool', + runId: expect.any(String), + }, + ]) + expect(lines(read)).toEqual([ + { role: 'user', text: 'refund please', by: null }, + { role: 'assistant', text: '', by: null }, + { role: 'tool', text: 'Passed to the next desk.', by: null }, + { + role: 'assistant', + text: 'billing:\nYour refund is on its way.', + by: 'billing', + }, + ]) + expect(read.mainCalls).toBe(1) + expect(read.billingCalls).toBe(1) + // billing gets the note once: from the tool call, with no extra note. + const sent = JSON.stringify(read.billingRequests[0]) + expect(sent.split('from main: Refund for order 42.')).toHaveLength(2) + } finally { + await stop(request, started.runId) + } + }) }) From d07a002791ad51e609383becebf7b36131fadf82 Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Thu, 12 Nov 2026 14:45:45 +0000 Subject: [PATCH 02/11] fix(ai-harness): pass custom handoff notes to the target and correct code-handoff speaker E2E Shard 1/4 failed two harness-handoff cases. A custom routing.handoffTool can rewrite the note it records, but the target still saw the model's original tool-call args. After a main-model tool handoff, stamp that recorded note on the call and fold it in 'own' history so the target gets it once, with no extra note messages. The code-handoff E2E assertion expected "main handed off to billing"; the last handoff is from triage, matching the unit tests. TanStack-Source-Factory-Run: factory-2145-d42d7302-70f6-4e2a-a9e9-248917f13c6c TanStack-Source-Agent: ci-heal --- packages/ai-harness/src/handoff.ts | 40 ++++++++++++++++++- packages/ai-harness/src/routed-history.ts | 32 ++++++++++++++- packages/ai-harness/src/session/turn.ts | 23 ++++++++++- packages/ai-harness/tests/handoff.test.ts | 5 +++ .../ai-harness/tests/routed-history.test.ts | 29 ++++++++++++++ testing/e2e/tests/harness-handoff.spec.ts | 2 +- 6 files changed, 127 insertions(+), 4 deletions(-) diff --git a/packages/ai-harness/src/handoff.ts b/packages/ai-harness/src/handoff.ts index 101e8c0aed..ca8fcbb3a7 100644 --- a/packages/ai-harness/src/handoff.ts +++ b/packages/ai-harness/src/handoff.ts @@ -1,6 +1,6 @@ import { toolDefinition } from '@tanstack/ai' import { isHumanTarget } from './log-records' -import { routedRunId } from './routed-history' +import { isHandoffNote, routedRunId } from './routed-history' import { isRecord } from './utils' import type { AfterToolCallInfo, @@ -466,6 +466,44 @@ function routedIndex(messages: ReadonlyArray, runId: string) { return messages.findIndex((message) => routedRunId(message) === runId) } +/** + * Put the recorded note of a main-model tool handoff on that tool call. + * A custom `handoffTool` can pass a different note than the model put in + * the call. `'own'` then folds that note for the target. The stored call + * arguments stay as the model wrote them. No extra note messages. + */ +export async function stampHandoffNote( + store: MessageStore, + threadId: string, + toolName: string, + note: string, +) { + const history = await store.loadThread(threadId) + for (let index = history.length - 1; index >= 0; index -= 1) { + const message = history[index] + if ( + message?.role !== 'assistant' || + isHandoffNote(message) || + !message.toolCalls?.some((call) => call.function.name === toolName) + ) { + continue + } + const harness = isRecord(message.metadata?.harness) + ? message.metadata.harness + : {} + if (harness.handoffNote === note) return + const updated: ModelMessage = { + ...message, + metadata: { + ...message.metadata, + harness: { ...harness, handoffNote: note }, + }, + } + await store.saveThread(threadId, history.with(index, updated)) + return + } +} + /** * Mark the routed message of run `runId` with the agent that wrote it, in * `metadata.harness.agent`. A routed pass that runs again on the same run id diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts index 88bbae6dfb..b18010112f 100644 --- a/packages/ai-harness/src/routed-history.ts +++ b/packages/ai-harness/src/routed-history.ts @@ -81,6 +81,26 @@ function handoffLine(from: string, args: string) { return typeof note === 'string' ? `${head} Note: ${note}` : head } +/** The note a custom handoff tool recorded on this call, when it set one. */ +function recordedHandoffNote(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.handoffNote === 'string' + ? harness.handoffNote + : undefined +} + +/** `args` with `note` set, so a fold reads the recorded note. */ +function argumentsWithNote(args: string, note: string) { + try { + const parsed = JSON.parse(args) + if (!isRecord(parsed)) return JSON.stringify({ note }) + if (parsed.note === note) return args + return JSON.stringify({ ...parsed, note }) + } catch { + return JSON.stringify({ note }) + } +} + /** The joined text of the assistant messages of a transcript. */ function assistantText(messages: ReadonlyArray) { return messages @@ -197,7 +217,17 @@ function foldFor( folded.add(call.id) // A note pair always uses the built-in tool name. if (isNote || call.function.name === handoffToolName) { - lines.push(handoffLine(from, call.function.arguments)) + // A custom handoff tool can record a different note than the model + // put in the call. The target reads that note; the stored call stays. + const stamped = isNote ? undefined : recordedHandoffNote(message) + lines.push( + handoffLine( + from, + stamped === undefined + ? call.function.arguments + : argumentsWithNote(call.function.arguments, stamped), + ), + ) } } return lines.length === 0 ? [] : [otherSpeaker(message, lines.join('\n\n'))] diff --git a/packages/ai-harness/src/session/turn.ts b/packages/ai-harness/src/session/turn.ts index 9e51a7cf72..dc13a9345f 100644 --- a/packages/ai-harness/src/session/turn.ts +++ b/packages/ai-harness/src/session/turn.ts @@ -60,7 +60,12 @@ import { import { SessionAgents } from './agents' import { commandStepScope, stepWaitPrefix } from './turn-parts' import { ToolTurnAdapter } from '../tool-turn' -import { TurnHandoffs, answererOf, markRouted } from '../handoff' +import { + TurnHandoffs, + answererOf, + markRouted, + stampHandoffNote, +} from '../handoff' import { routedHistoryMiddleware } from '../routed-history' import type { AnyChatMiddleware, @@ -1577,6 +1582,22 @@ export abstract class SessionTurn< if (isFinished) break const handoff = handoffs?.take() const target = handoff?.to + // A custom handoff tool can rewrite the note. Stamp it on the main + // model's call so the target reads it. No extra note messages: the + // call is already in the thread. + if ( + handoff?.by === 'tool' && + handoff.from === null && + typeof handoff.note === 'string' && + handoffs + ) { + await stampHandoffNote( + this.messages, + this.threadId, + handoffs.tool(null).name, + handoff.note, + ) + } // The note of a code handoff, for the transcript. const note = handoff ? (handoffs?.messages(handoff) ?? []) : [] if (isHumanTarget(target)) { diff --git a/packages/ai-harness/tests/handoff.test.ts b/packages/ai-harness/tests/handoff.test.ts index 2f9fec43e9..fa815d367c 100644 --- a/packages/ai-harness/tests/handoff.test.ts +++ b/packages/ai-harness/tests/handoff.test.ts @@ -439,6 +439,11 @@ describe('handoffs', () => { ).toEqual([ { from: null, to: 'billing', note: 'main: order 42', by: 'tool' }, ]) + // billing gets the rewritten note from the tool call, once. + expect(billing.model.calls[0].messages.at(-1)).toMatchObject({ + role: 'user', + content: 'main handed off to billing. Note: main: order 42', + }) await host.close() }) diff --git a/packages/ai-harness/tests/routed-history.test.ts b/packages/ai-harness/tests/routed-history.test.ts index 7505701bdc..e98fb87277 100644 --- a/packages/ai-harness/tests/routed-history.test.ts +++ b/packages/ai-harness/tests/routed-history.test.ts @@ -171,6 +171,35 @@ describe('historyFor', () => { ]) }) + it('folds a stamped recorded note instead of the model call args', async () => { + const call = { + ...handoffCall('call_h', 'handoff', { + to: 'billing', + note: 'Refund for order 42.', + }), + metadata: { + harness: { handoffNote: 'from main: Refund for order 42.' }, + }, + } + const result = await history({ + messages: [USER, call, toolResult('call_h')], + speaker: 'billing', + mode: 'own', + }) + expect(result).toEqual([ + USER, + { + id: 'm_call_h', + role: 'user', + content: + 'main:\nLet me pass you on.\n\nmain handed off to billing. Note: from main: Refund for order 42.', + metadata: { + harness: { handoffNote: 'from main: Refund for order 42.' }, + }, + }, + ]) + }) + it('reads the handoff tool name from handoffToolName', async () => { const custom = handoffCall('call_t', 'transfer', { to: 'billing' }) const builtIn = handoffCall('call_b', 'handoff', { to: 'research' }) diff --git a/testing/e2e/tests/harness-handoff.spec.ts b/testing/e2e/tests/harness-handoff.spec.ts index 0c692aa247..d570559f40 100644 --- a/testing/e2e/tests/harness-handoff.spec.ts +++ b/testing/e2e/tests/harness-handoff.spec.ts @@ -238,7 +238,7 @@ test.describe('harness: handoffs inside one session', () => { const noteMessage = sentToBilling.at(-1) expect(noteMessage).toMatchObject({ role: 'user' }) expect(JSON.stringify(noteMessage)).toContain( - 'main handed off to billing. Note: Refund for order 42.', + 'triage handed off to billing. Note: Refund for order 42.', ) expect( blocksOf(sentToBilling).some( From 8548c7ba284c25788a25e68a2f63ea8f9a7f9e88 Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Thu, 12 Nov 2026 15:33:58 +0000 Subject: [PATCH 03/11] fix(ai-harness): stop same-turn root harnessAgent resends and duplicate routed transcripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A root harnessAgent invoked again in one outer turn (desk → main → desk) no longer re-appends the already-answered user message to its kept child session. Routed history now links each host message to that pass's child run so two billing passes in one turn expand each tool call once. TanStack-Source-Factory-Run: factory-2145-5df4a69d-4bdc-401a-9713-51d1c7ebbb4a TanStack-Source-Agent: fixer --- packages/ai-harness/src/handoff.ts | 17 +++++--- packages/ai-harness/src/routed-history.ts | 25 ++++++++--- packages/ai-harness/src/session/agents.ts | 35 ++++++++++------ packages/ai-harness/src/session/turn.ts | 23 ++++++++--- .../tests/harness-agent-root.test.ts | 22 ++++++++++ .../ai-harness/tests/routing-history.test.ts | 41 +++++++++++++++++++ 6 files changed, 133 insertions(+), 30 deletions(-) diff --git a/packages/ai-harness/src/handoff.ts b/packages/ai-harness/src/handoff.ts index ca8fcbb3a7..463e8c3e18 100644 --- a/packages/ai-harness/src/handoff.ts +++ b/packages/ai-harness/src/handoff.ts @@ -506,17 +506,18 @@ export async function stampHandoffNote( /** * Mark the routed message of run `runId` with the agent that wrote it, in - * `metadata.harness.agent`. A routed pass that runs again on the same run id - * (a handoff to another root agent, a late join, a hook) first keeps this - * message under its own id, so each pass keeps its own message. + * `metadata.harness.agent`, and the child run of that pass, in + * `metadata.harness.subagentRunId`. A routed pass that runs again on the + * same run id (a handoff to another root agent, a late join, a hook) first + * keeps this message under its own id, so each pass keeps its own message. */ export async function markRouted( store: MessageStore, threadId: string, runId: string, - mark: { agent?: string }, + mark: { agent?: string; subagentRunId?: string }, ) { - const { agent } = mark + const { agent, subagentRunId } = mark if (agent === undefined) return const history = await store.loadThread(threadId) const index = routedIndex(history, runId) @@ -529,7 +530,11 @@ export async function markRouted( ...message, metadata: { ...message.metadata, - harness: { ...harness, agent }, + harness: { + ...harness, + agent, + ...(subagentRunId !== undefined ? { subagentRunId } : {}), + }, }, } await store.saveThread(threadId, history.with(index, updated)) diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts index b18010112f..b50bff4651 100644 --- a/packages/ai-harness/src/routed-history.ts +++ b/packages/ai-harness/src/routed-history.ts @@ -17,8 +17,8 @@ interface HistorySource { mode: NonNullable /** The name of the handoff tool. `'handoff'` when not set. */ handoffToolName?: string - /** The routed children of run `runId`, in order. */ - childrenOf: (runId: string) => Array + /** The routed children of run `runId` for this host message, in order. */ + childrenOf: (runId: string, message: ModelMessage) => Array /** The stored transcript of one routed child. */ transcriptOf: (subagentRunId: string) => Promise> /** @@ -57,6 +57,14 @@ export function routedAgent(message: ModelMessage) { : undefined } +/** The child run of a routed pass: `metadata.harness.subagentRunId`. */ +export function routedChildId(message: ModelMessage) { + const harness = message.metadata?.harness + return isRecord(harness) && typeof harness.subagentRunId === 'string' + ? harness.subagentRunId + : undefined +} + /** True for a router or code handoff note: `metadata.harness.handoff`. */ export function isHandoffNote(message: ModelMessage) { const harness = message.metadata?.harness @@ -154,11 +162,16 @@ async function expandHost( transcriptOf: HistorySource['transcriptOf'], ) { // A host message of one agent holds only that agent's pass of the run. + // A pass-specific child id wins over agent name: two passes of the same + // agent in one turn share the outer run id and the agent name. + const childId = routedChildId(message) const agent = routedAgent(message) const own = - agent === undefined - ? children - : children.filter((child) => child.name === agent) + childId !== undefined + ? children.filter((child) => child.subagentRunId === childId) + : agent === undefined + ? children + : children.filter((child) => child.name === agent) if (!own.some((child) => wanted(child.name))) return undefined const parts: Array = [] let expanded = false @@ -264,7 +277,7 @@ export async function historyFor( const runId = passRunId(message) const children = message.role === 'assistant' && runId !== undefined - ? childrenOf(runId) + ? childrenOf(runId, message) : [] const expanded = children.length > 0 diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index e475f2f975..455cdddd4e 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -2821,12 +2821,15 @@ function childHandoff(from: string, handoffs: TurnHandoffs, withTool: boolean) { } /** - * The input of a turn of the routed root `harnessAgent` named `name`: the - * latest user message, after one text part with what happened in the shared - * thread since the last answer of `name`. Other speakers come with their - * labels, as `routing.history: 'own'` folds them. The child session keeps - * its own turns, so they are not sent again. On its first turn, the context - * is the conversation so far. + * The input of a turn of the routed root `harnessAgent` named `name`: + * what happened in the shared thread since the last answer of `name`, then + * the latest user message when that message is new (it comes after the last + * answer). Other speakers come with their labels, as `routing.history: 'own'` + * folds them. The child session keeps its own turns, so they are + * not sent again. A second pass in the same outer turn does not resend the + * already-answered request: only the context since the last answer, or a + * short continue prompt when that context is empty. On its first turn, the + * context is the conversation so far. */ async function rootInput(messages: Array, name: string) { // A handoff note names its source agent too, but it is no answer. @@ -2834,6 +2837,11 @@ async function rootInput(messages: Array, name: string) { routedAgent(message) === name && !isHandoffNote(message) const lastAnswer = messages.findLastIndex(isAnswer) const latest = messages.findLastIndex((message) => message.role === 'user') + // The latest user message is new input only when it comes after this + // agent last answered. A second pass in the same outer turn (desk → + // main → desk) has lastAnswer after latest, so the kept child session + // must not get that request again. + const hasNewUser = latest > lastAnswer const since = messages.filter( (_, index) => index > lastAnswer && index !== latest, ) @@ -2853,17 +2861,18 @@ async function rootInput(messages: Array, name: string) { if (line === '' || message.role === 'tool') return [] return [fromUser.has(message) ? `user:\n${line}` : line] }) - const input = lastUserInput(messages) - if (lines.length === 0) return input + const input = hasNewUser ? lastUserInput(messages) : undefined + if (lines.length === 0) return input ?? 'Continue.' const head = lastAnswer === -1 ? 'The conversation so far:' : 'Since your last turn:' + const context: ContentPart = { + type: 'text', + content: [head, ...lines].join('\n\n'), + } + if (input === undefined) return [context] const sent: Array = typeof input === 'string' ? [{ type: 'text', content: input }] : input - const parts: Array = [ - { type: 'text', content: [head, ...lines].join('\n\n') }, - ...sent, - ] - return parts + return [context, ...sent] } /** diff --git a/packages/ai-harness/src/session/turn.ts b/packages/ai-harness/src/session/turn.ts index dc13a9345f..d41563e45e 100644 --- a/packages/ai-harness/src/session/turn.ts +++ b/packages/ai-harness/src/session/turn.ts @@ -66,7 +66,7 @@ import { markRouted, stampHandoffNote, } from '../handoff' -import { routedHistoryMiddleware } from '../routed-history' +import { routedChildId, routedHistoryMiddleware } from '../routed-history' import type { AnyChatMiddleware, AnyTextAdapter, @@ -980,18 +980,21 @@ export abstract class SessionTurn< // `KEPT_ENDED` ended runs. An older routed run has no children // there, so the model gets it as text. Keep the routed children // in the fold if that matters. - childrenOf: (runId) => - [...this.writer.state.thread.runs.values()].flatMap( + childrenOf: (runId, message) => { + const childId = routedChildId(message) + return [...this.writer.state.thread.runs.values()].flatMap( ({ parentRunId: parent, name, subagentRunId }) => parent === runId && name !== undefined && subagentRunId !== undefined && // Only routed root agents: a tool-mode child of the main // model in the same run reports through its tool result. - rootAgents.some((agent) => agent.name === name) + rootAgents.some((agent) => agent.name === name) && + (childId === undefined || subagentRunId === childId) ? [{ name, subagentRunId }] : [], - ), + ) + }, transcriptOf: (subagentRunId) => { const transcript = transcripts.get(subagentRunId) ?? @@ -1571,8 +1574,18 @@ export abstract class SessionTurn< // The routed message of this part gets the name of its agent. A part // that waits for an answer gets it too. if (routed && failure === undefined && !signal.aborted) { + const childId = + typeof answerer === 'string' + ? [...this.writer.state.thread.runs.values()].findLast( + (run) => + run.parentRunId === operation.id && + run.name === answerer && + run.subagentRunId !== undefined, + )?.subagentRunId + : undefined await markRouted(this.messages, this.threadId, operation.id, { ...(answerer ? { agent: answerer } : {}), + ...(childId !== undefined ? { subagentRunId: childId } : {}), }) } const isFinished = diff --git a/packages/ai-harness/tests/harness-agent-root.test.ts b/packages/ai-harness/tests/harness-agent-root.test.ts index a12aa5d8cf..af7511bbcb 100644 --- a/packages/ai-harness/tests/harness-agent-root.test.ts +++ b/packages/ai-harness/tests/harness-agent-root.test.ts @@ -198,6 +198,28 @@ describe('harnessAgent as a routed root agent', () => { await host.close() }) + it('does not resend the original user message when invoked again in the same turn', async () => { + const desk = deskHarness([ + () => toolCall('handoff', { to: 'main' }), + () => text('Desk again.'), + ]) + const { host, session } = await openRouted({ + child: desk.harness, + router: () => 'desk', + mainReplies: [() => toolCall('handoff', { to: 'desk' })], + }) + + await session.send('refund please') + + expect(desk.model.calls).toHaveLength(2) + const second = desk.model.calls[1] + expect( + userTexts(second).filter((entry) => entry === 'refund please'), + ).toHaveLength(1) + expect(userTexts(second).at(-1)).not.toBe('refund please') + await host.close() + }) + it('fails the turn when the child harness has its own routing', async () => { const model = mockAdapter([() => text('Never.')]) const child = defineHarness({ diff --git a/packages/ai-harness/tests/routing-history.test.ts b/packages/ai-harness/tests/routing-history.test.ts index 5b8c7025a7..14b79c2073 100644 --- a/packages/ai-harness/tests/routing-history.test.ts +++ b/packages/ai-harness/tests/routing-history.test.ts @@ -236,6 +236,47 @@ describe('routing.history', () => { await host.close() }) + it('gives a later billing turn each tool call once after billing → main → billing in one turn', async () => { + const billing = modelAgent( + 'billing', + [ + () => toolCall('refund', {}, 'call_r'), + () => [ + ...text('Refunded'), + ...toolCall('handoff', { to: 'main' }, 'call_hb'), + ], + () => toolCall('lookup', {}, 'call_l'), + () => text('Looked up.'), + () => text('Welcome'), + ], + [plainTool('refund'), plainTool('lookup')], + ) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ + 'refund please': 'billing', + thanks: 'billing', + }), + replies: [() => toolCall('handoff', { to: 'billing' }, 'call_hm')], + }) + + expect((await session.send('refund please')).text).toBe( + 'billing:\nLooked up.', + ) + await session.send('thanks') + + const later = billing.model.calls[4] + expect(calledTools(later).filter((name) => name === 'refund')).toEqual([ + 'refund', + ]) + expect(calledTools(later).filter((name) => name === 'lookup')).toEqual([ + 'lookup', + ]) + expect(answeredCalls(later).filter((id) => id === 'call_r')).toHaveLength(1) + expect(answeredCalls(later).filter((id) => id === 'call_l')).toHaveLength(1) + await host.close() + }) + it('gives a resumed routed agent the messages of its paused run', async () => { // No server implementation: the turn waits for the result of the client. const openScreen = toolDefinition({ From 727c6c8e35f3abf95157c57918eda647b7fc832d Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Thu, 12 Nov 2026 16:04:38 +0000 Subject: [PATCH 04/11] fix(ai-harness): fold unstamped parallel routed answers as user text and copy parent media into routed root harnessAgent sessions In history: 'own', a list / { names } / { steps } pick never stamps metadata.harness.agent, so fold that host row to a user message when this speaker owns none of its children. Copy parent-thread harness-media parts into the child thread before session.send, and keep media from earlier user messages in the conversation-so-far context. TanStack-Source-Factory-Run: factory-2145-a3bc551e-20b7-4086-9bb4-3238d1566af2 TanStack-Source-Agent: fixer --- packages/ai-harness/src/routed-history.ts | 12 ++- packages/ai-harness/src/session/agents.ts | 102 +++++++++++++++--- .../tests/harness-agent-root.test.ts | 44 ++++++++ .../ai-harness/tests/routed-history.test.ts | 21 ++++ .../ai-harness/tests/routing-history.test.ts | 38 +++++++ 5 files changed, 202 insertions(+), 15 deletions(-) diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts index b50bff4651..4ecfdf51b0 100644 --- a/packages/ai-harness/src/routed-history.ts +++ b/packages/ai-harness/src/routed-history.ts @@ -285,7 +285,17 @@ export async function historyFor( : undefined if (expanded) result.push(...expanded) else if (mode === 'all') result.push(message) - else result.push(...foldFor(message, speaker, handoffToolName, folded)) + else if ( + children.length > 0 && + !children.some((child) => wanted(child.name)) + ) { + // Unstamped multi-agent host (a list / { names } / { steps } pick + // never sets metadata.harness.agent). Fold as user text and keep + // the existing name: labels, so the main model does not read them + // as its own. + const text = textOf(message).trim() + if (text !== '') result.push(otherSpeaker(message, text)) + } else result.push(...foldFor(message, speaker, handoffToolName, folded)) } return result } diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index 455cdddd4e..5051a7617c 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -6,7 +6,7 @@ import { runAgentStream, validateWithStandardSchema, } from '@tanstack/ai' -import { withPersistence } from '@tanstack/ai-persistence' +import { withPersistence, type BlobBody } from '@tanstack/ai-persistence' import { redactMiddleware } from '../auth' import { BOUNDARY_INTERRUPTS, withoutBoundaryWaits } from '../boundary-waits' import { errorsOnlyDebug } from '../logger' @@ -71,6 +71,7 @@ import type { AgentResultRecord, RunTreeNode } from '../log-records' import { NO_SESSION_CONTEXT, withHarnessChild } from '../agents' import { forwardTurn, lastUserInput } from '../harness-text' import { textOf } from '../first-party/session-tools' +import { mediaIdOf, mediaPart } from '../media-ref' import { historyFor, isHandoffNote, routedAgent } from '../routed-history' import { wait as backoff, isTransientModelError } from '../turn' import { addCounts, emptyCounts } from '../usage' @@ -926,7 +927,16 @@ export abstract class SessionAgents< try { const messages = convertMessagesToModelMessages(ctx.messages) operation = session.send( - root ? await rootInput(messages, root.name) : lastUserInput(messages), + root + ? await copyThreadMedia( + { + getMedia: (id) => this.getMedia({ id }), + open: (id) => this.mediaStore.open(id), + }, + session, + await rootInput(messages, root.name), + ) + : lastUserInput(messages), ) if (signal.aborted) onAbort() let finished: InnerTurnEnd['finished'] = 'stop' @@ -2856,23 +2866,87 @@ async function rootInput(messages: Array, name: string) { // A message of the user comes back as the same object. A folded message // of another speaker is a new user message that has its label already. const fromUser = new Set(since.filter((message) => message.role === 'user')) - const lines = folded.flatMap((message) => { + const lines: Array = [] + const earlierMedia: Array = [] + for (const message of folded) { + if (message.role === 'tool') continue const line = textOf(message).trim() - if (line === '' || message.role === 'tool') return [] - return [fromUser.has(message) ? `user:\n${line}` : line] - }) + if (fromUser.has(message)) { + if (line !== '') lines.push(`user:\n${line}`) + if (Array.isArray(message.content)) { + for (const part of message.content) { + if (mediaIdOf(part) !== undefined) earlierMedia.push(part) + } + } + continue + } + if (line !== '') lines.push(line) + } const input = hasNewUser ? lastUserInput(messages) : undefined - if (lines.length === 0) return input ?? 'Continue.' + const sent: Array = + input === undefined + ? [] + : typeof input === 'string' + ? [{ type: 'text', content: input }] + : input + if (lines.length === 0 && earlierMedia.length === 0) { + return input ?? 'Continue.' + } const head = lastAnswer === -1 ? 'The conversation so far:' : 'Since your last turn:' - const context: ContentPart = { - type: 'text', - content: [head, ...lines].join('\n\n'), + const context: Array = [ + { + type: 'text', + content: lines.length === 0 ? head : [head, ...lines].join('\n\n'), + }, + ...earlierMedia, + ] + return sent.length === 0 ? context : [...context, ...sent] +} + +/** + * Copy parent-thread `harness-media:` parts in `input` into `dest`, so the + * child session can read them. Same copy as `adoptMessages`: each file gets + * a new id on the child thread. + */ +async function copyThreadMedia( + source: { + getMedia: (id: string) => Promise + open: (id: string) => Promise + }, + dest: { + putMedia: (upload: { + body: BlobBody + mimeType: string + name: string + }) => Promise + }, + input: UserInput, +): Promise { + if (typeof input === 'string') return input + const copies = new Map() + const copy = async (id: string) => { + if (!copies.has(id)) { + const record = await source.getMedia(id) + copies.set( + id, + record && + (await dest.putMedia({ + body: await source.open(id), + mimeType: record.mimeType, + name: record.name, + })), + ) + } + return copies.get(id) ?? undefined } - if (input === undefined) return [context] - const sent: Array = - typeof input === 'string' ? [{ type: 'text', content: input }] : input - return [context, ...sent] + return Promise.all( + input.map(async (part) => { + const id = mediaIdOf(part) + const made = id === undefined ? undefined : await copy(id) + return made ? mediaPart(made) : part + }), + ) } /** diff --git a/packages/ai-harness/tests/harness-agent-root.test.ts b/packages/ai-harness/tests/harness-agent-root.test.ts index af7511bbcb..350ed5c051 100644 --- a/packages/ai-harness/tests/harness-agent-root.test.ts +++ b/packages/ai-harness/tests/harness-agent-root.test.ts @@ -3,6 +3,7 @@ import { z } from 'zod' import { defineAgent, toolDefinition } from '@tanstack/ai' import { memoryLogStore, memoryPersistence } from '@tanstack/ai-persistence' import { createHarnessHost, defineHarness, harnessAgent } from '../src' +import { mediaPart } from '../src/media-entry' import { mockAdapter, storedThreadState, text, toolCall } from './helpers' import type { ModelMessage } from '@tanstack/ai' import type { AnyHarness, HarnessRouting } from '../src' @@ -198,6 +199,49 @@ describe('harnessAgent as a routed root agent', () => { await host.close() }) + it('gives a routed root harnessAgent the bytes of parent-thread media', async () => { + const encoder = new TextEncoder() + const model = mockAdapter([() => text('A cat.')]) + const child = defineHarness({ + name: 'test/desk', + adapter: { ...model.adapter, inputModalities: ['text', 'image'] }, + }) + const main = mockAdapter([]) + const host = createHarnessHost({ persistence: memoryPersistence() }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: { ...main.adapter, inputModalities: ['text', 'image'] }, + agents: [harnessAgent(child, { name: 'desk' })], + routing: { router: () => ({ name: 'desk' }) }, + }), + { threadId: THREAD }, + ) + const record = await session.putMedia({ + body: encoder.encode('hello'), + mimeType: 'image/png', + name: 'cat.png', + }) + await session.send([ + { type: 'text', content: 'What is this?' }, + mediaPart(record), + ]) + + const last = model.calls[0].messages.at(-1) + expect(last.role).toBe('user') + const parts = Array.isArray(last.content) ? last.content : [] + expect(parts).toEqual( + expect.arrayContaining([ + { + type: 'image', + source: { type: 'data', value: 'aGVsbG8=', mimeType: 'image/png' }, + }, + ]), + ) + expect(JSON.stringify(last.content)).not.toContain('not found') + await host.close() + }) + it('does not resend the original user message when invoked again in the same turn', async () => { const desk = deskHarness([ () => toolCall('handoff', { to: 'main' }), diff --git a/packages/ai-harness/tests/routed-history.test.ts b/packages/ai-harness/tests/routed-history.test.ts index e98fb87277..43fee9c7e5 100644 --- a/packages/ai-harness/tests/routed-history.test.ts +++ b/packages/ai-harness/tests/routed-history.test.ts @@ -223,6 +223,27 @@ describe('historyFor', () => { ]) }) + it("folds an unstamped multi-agent host as user text for the main model in 'own' mode", async () => { + const parallel: ModelMessage = { + id: 'assistant:r3', + role: 'assistant', + content: 'billing:\nRefunded\n\nshipping:\nShipped', + metadata: { tanstack: { runId: 'r3' } }, + } + const result = await history({ + messages: [USER, parallel], + speaker: null, + mode: 'own', + children: { + r3: [ + { name: 'billing', subagentRunId: 's1' }, + { name: 'shipping', subagentRunId: 's2' }, + ], + }, + }) + expect(result).toEqual([USER, { ...parallel, role: 'user' }]) + }) + it("keeps the main model's own calls and gives it routed answers as user text in 'own' mode", async () => { const call = handoffCall('call_h', 'handoff', { to: 'billing' }) const messages = [USER, call, toolResult('call_h'), BILLING_HOST] diff --git a/packages/ai-harness/tests/routing-history.test.ts b/packages/ai-harness/tests/routing-history.test.ts index 14b79c2073..6d1d398b7e 100644 --- a/packages/ai-harness/tests/routing-history.test.ts +++ b/packages/ai-harness/tests/routing-history.test.ts @@ -168,6 +168,44 @@ describe('routing.history', () => { await host.close() }) + it("'own' gives the main model a parallel run as user text, not assistant", async () => { + const billing = modelAgent('billing', [() => text('Refunded')]) + const shipping = modelAgent('shipping', [() => text('Shipped')]) + const { host, session, main } = await openRouting({ + agents: [billing.agent, shipping.agent], + router: ({ input }) => + input === 'both' + ? { names: ['billing', 'shipping'], order: 'parallel' } + : 'main', + replies: [() => text('Main here')], + }) + + await session.send('both') + expect((await session.send('ask main')).text).toBe('Main here') + + const [call] = main.calls + const labeled = call.messages.filter( + (message: ModelMessage) => + typeof message.content === 'string' && + message.content.includes('billing:') && + message.content.includes('shipping:'), + ) + expect(labeled.length).toBeGreaterThan(0) + expect( + labeled.every((message: ModelMessage) => message.role === 'user'), + ).toBe(true) + expect( + call.messages.some( + (message: ModelMessage) => + message.role === 'assistant' && + typeof message.content === 'string' && + (message.content.includes('billing:') || + message.content.includes('Refunded')), + ), + ).toBe(false) + await host.close() + }) + it("'own' gives a routed agent the handoff of main as a text line", async () => { const billing = modelAgent('billing', [ () => text('Refunded'), From f8c9ad156a9e660b4733e3cb311c12f6693f60b3 Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Thu, 12 Nov 2026 16:32:50 +0000 Subject: [PATCH 05/11] fix(ai-harness): split BlobBody type import for oxlint @tanstack/ai-harness:test:oxlint failed on import/consistent-type-specifier-style (prefer-top-level) for `import { withPersistence, type BlobBody }`. Use a value import plus a separate import type, matching the rest of the package. TanStack-Source-Factory-Run: factory-2145-5c450be5-f48f-4da6-ab9c-60ed56e651f2 TanStack-Source-Agent: ci-heal --- packages/ai-harness/src/session/agents.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index 5051a7617c..a535d07630 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -6,7 +6,8 @@ import { runAgentStream, validateWithStandardSchema, } from '@tanstack/ai' -import { withPersistence, type BlobBody } from '@tanstack/ai-persistence' +import { withPersistence } from '@tanstack/ai-persistence' +import type { BlobBody } from '@tanstack/ai-persistence' import { redactMiddleware } from '../auth' import { BOUNDARY_INTERRUPTS, withoutBoundaryWaits } from '../boundary-waits' import { errorsOnlyDebug } from '../logger' From 97cc34a5483af30c56f651cac1c616b9a1facf74 Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Thu, 12 Nov 2026 16:57:03 +0000 Subject: [PATCH 06/11] fix(ai-harness): fold fabricated handoff notes for every speaker in own history foldFor kept a message when owner === speaker before isHandoffNote ran, so a later billing turn (and main, for unstamped router notes) saw a second harness-fabricated handoff__ assistant tool pair it never produced. Fold notes first, then keep the speaker's real messages. Add a routing-history test that billing's later turn expands its real handoff once. TanStack-Source-Factory-Run: factory-2145-161b8c26-242b-4da1-a451-3578cae110a8 TanStack-Source-Agent: fixer --- packages/ai-harness/src/routed-history.ts | 21 ++++++---- .../ai-harness/tests/routing-history.test.ts | 40 +++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts index 4ecfdf51b0..95234ad64d 100644 --- a/packages/ai-harness/src/routed-history.ts +++ b/packages/ai-harness/src/routed-history.ts @@ -192,12 +192,14 @@ async function expandHost( /** * Fold one message of the thread for `speaker` (`null` for the main model) - * in `'own'` mode. A message of the speaker stays. A message of another - * speaker becomes text in one user message, so the speaker does not read - * it as its own: the main model's text as `main:\ntext`, a routed answer - * as its `name:\ntext`, and each handoff call as one line. Other tool calls - * of another speaker go. `folded` collects the ids of the tool calls that - * this removes, so their results go too. + * in `'own'` mode. A message of the speaker stays, except a fabricated + * handoff note (`metadata.harness.handoff`): those fold for every speaker, + * including the owner, because they are not a model call. A message of + * another speaker becomes text in one user message, so the speaker does + * not read it as its own: the main model's text as `main:\ntext`, a routed + * answer as its `name:\ntext`, and each handoff call as one line. Other + * tool calls of another speaker go. `folded` collects the ids of the tool + * calls that this removes, so their results go too. */ function foldFor( message: ModelMessage, @@ -211,10 +213,13 @@ function foldFor( return answersFolded ? [] : [message] } // A note names the agent that handed off. A router note of the main - // model names none. + // model names none. Notes are fabricated: fold them for every speaker + // so a later turn does not keep a second handoff pair it never produced. const owner = routedAgent(message) ?? null - if (message.role !== 'assistant' || owner === speaker) return [message] const isNote = isHandoffNote(message) + if (message.role !== 'assistant' || (owner === speaker && !isNote)) { + return [message] + } const from = owner ?? 'main' const text = textOf(message).trim() const lines: Array = [] diff --git a/packages/ai-harness/tests/routing-history.test.ts b/packages/ai-harness/tests/routing-history.test.ts index 6d1d398b7e..270e8844d3 100644 --- a/packages/ai-harness/tests/routing-history.test.ts +++ b/packages/ai-harness/tests/routing-history.test.ts @@ -233,6 +233,46 @@ describe('routing.history', () => { await host.close() }) + it("'own' folds a root agent's handoff note so a later turn keeps only its real handoff", async () => { + const billing = modelAgent('billing', [ + () => [ + ...text('Refunded'), + ...toolCall('handoff', { to: 'main' }, 'call_hb'), + ], + () => text('Welcome'), + ]) + const { host, session } = await openRouting({ + agents: [billing.agent], + router: routeBy({ + 'refund please': 'billing', + thanks: 'billing', + }), + replies: [() => text('Main here')], + }) + + expect((await session.send('refund please')).text).toBe('Main here') + await session.send('thanks') + + const later = billing.model.calls[1] + expect(calledTools(later).filter((name) => name === 'handoff')).toEqual([ + 'handoff', + ]) + expect(answeredCalls(later).filter((id) => id === 'call_hb')).toEqual([ + 'call_hb', + ]) + expect( + later.messages.some( + (message: ModelMessage) => + message.role === 'assistant' && + (message.id?.startsWith('handoff_') || + (message.toolCalls ?? []).some((call) => + call.id.startsWith('handoff_'), + )), + ), + ).toBe(false) + await host.close() + }) + it('gives each pass of a turn with two routed passes to its own agent', async () => { const billing = modelAgent( 'billing', From 16f14384a454bccb69d034f87ccf71eaf8cde608 Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Thu, 12 Nov 2026 17:24:33 +0000 Subject: [PATCH 07/11] fix(ai-harness): keep a root agent's own handoff out of child turn context Coverage failed two ai-harness tests after fabricated handoff notes were folded for every speaker. rootInput was sending the agent's own hand-back (for example "desk handed off to main.") into the kept child session; skip those notes so "Since your last turn" only has other speakers. The afterAgentsAnswer main-model input now matches history: 'own' (a folded "main handed off to writer." line instead of the stored assistant+tool pair). TanStack-Source-Factory-Run: factory-2145-607045fd-83c3-44b2-91c4-c3d1a3ae8103 TanStack-Source-Agent: ci-heal --- packages/ai-harness/src/session/agents.ts | 6 +++++- packages/ai-harness/tests/routing.test.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index a535d07630..bafd352d4d 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -2854,7 +2854,11 @@ async function rootInput(messages: Array, name: string) { // must not get that request again. const hasNewUser = latest > lastAnswer const since = messages.filter( - (_, index) => index > lastAnswer && index !== latest, + (message, index) => + index > lastAnswer && + index !== latest && + // This agent's own handoff is in its child session, not this context. + !(isHandoffNote(message) && routedAgent(message) === name), ) // No transcripts: other runs stay as their `name:\ntext` host message. const folded = await historyFor({ diff --git a/packages/ai-harness/tests/routing.test.ts b/packages/ai-harness/tests/routing.test.ts index b9f9eeb54b..2cb146c1c9 100644 --- a/packages/ai-harness/tests/routing.test.ts +++ b/packages/ai-harness/tests/routing.test.ts @@ -355,7 +355,7 @@ describe('routing.router', () => { expect(main.calls).toHaveLength(1) expect(messageTexts(main.calls[0])).toEqual([ 'write it', - ...routedTo('writer'), + 'main handed off to writer.', 'writer:\nDraft', ]) expect( From 43e6445890a7c1a314f3d5de8dc6ad958af3a946 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:09:34 +1100 Subject: [PATCH 08/11] Fix: fold unstamped hosts, pass the handoff tool name, and correct the handoff docs - Fold an unstamped multi-agent host as other-speaker text when the pass has no children. - Pass the main model's handoff tool name into rootInput so a renamed tool still folds. - Say a root-agent tool handoff still adds the two note messages. - Say an app tool replaces the harness tool only when the names match. - Say 'all' expands host messages into child transcripts, and a root harnessAgent always gets the 'own' fold. - Throw when stampHandoffNote cannot find the tool call. --- docs/harness/handoffs.md | 7 ++++--- docs/harness/subagents.md | 3 ++- packages/ai-harness/src/define.ts | 8 ++++--- packages/ai-harness/src/handoff.ts | 3 +++ packages/ai-harness/src/routed-history.ts | 18 +++++++++------- packages/ai-harness/src/session/agents.ts | 21 ++++++++++++------- .../ai-harness/tests/handoff-tool.test.ts | 16 +++++++++++++- .../ai-harness/tests/routed-history.test.ts | 19 +++++++++++++++++ 8 files changed, 72 insertions(+), 23 deletions(-) diff --git a/docs/harness/handoffs.md b/docs/harness/handoffs.md index 874a5c8373..0bcaacb937 100644 --- a/docs/harness/handoffs.md +++ b/docs/harness/handoffs.md @@ -110,10 +110,11 @@ export const support = defineHarness({ - The harness calls `handoffTool` once per turn for each speaker: the main model (`from: null`) and each root agent. - `targets` holds the names that this speaker can hand off to. `allowHuman` is the value of `routing.allowHandoffToHuman`. -- `handoff({ to, note })` hands off. The tool call is the record in the transcript, so the harness adds no note messages. +- `handoff({ to, note })` hands off. For the main model, the tool call is already in the transcript. The harness adds no note messages. +- A tool handoff from a root agent still adds the two note messages. - The tool can have any name. With `giveModelsHandoffTool: false`, the harness does not call `handoffTool`. -To change the tool of one speaker only, give that speaker an app tool named `handoff`. It replaces the harness tool. Its code hands off with `ctx.context.run.handoff` (main model) or `ctx.harness.handoff` (root agent), as in [Hand off from code](#2-hand-off-from-code). +Give one speaker an app tool with the same name as the tool that `handoffTool` returns. It replaces the harness tool. Its code hands off with `ctx.context.run.handoff` (main model) or `ctx.harness.handoff` (root agent), as in [Hand off from code](#2-hand-off-from-code). ## 2. Hand off from code @@ -172,7 +173,7 @@ What the model of the target gets depends on `routing.history`: - `'own'` (default): the turns of the other speakers are labeled text in a user message, for example `main:\n...`. The handoff is one text line, for example `main handed off to billing. Note: Refund for order 42`. When billing hands back, the main model gets `billing handed off to main.` the same way. - `'text'`: the messages of the main model stay as they are, and the two handoff messages come after them. So the prompt cache of the provider still holds. -- `'all'`: the messages of the main model and of each agent stay as they are, with their tool calls. +- `'all'`: each routed host message becomes the child transcripts of that run, with their tool calls. With `routing.giveModelsHandoffTool: false`, the models have no `handoff` tool. Then the harness adds one assistant message with the same text and the note. diff --git a/docs/harness/subagents.md b/docs/harness/subagents.md index 9d659c9a8d..13ef18dbdb 100644 --- a/docs/harness/subagents.md +++ b/docs/harness/subagents.md @@ -776,7 +776,7 @@ export const support = defineHarness({ | Value | A routed agent gets | The main model gets | | --- | --- | --- | | `'own'` (default) | Its own earlier runs in full, with tool calls and results. Other agents as `name:\ntext`, and the main model as `main:\ntext`. A handoff is one text line. | Its own tool calls in full. Routed runs as `billing:\n...` text, and a hand-back as one text line. | -| `'all'` | Every routed run in full. | Every routed run in full. | +| `'all'` | Each routed host message becomes the child transcripts of that run. | Each routed host message becomes the child transcripts of that run. | | `'text'` | Every routed run as `name:\ntext`. | Every routed run as `name:\ntext`. | - With `'own'`, the text of another speaker reaches the model as a user message. So no model reads the words of another speaker as its own. @@ -844,6 +844,7 @@ export const frontDesk = defineHarness({ - Each turn sends the child what happened in the thread since its last turn, as labeled text. Then it sends the latest user message. - The model of the child gets the `handoff` tool of the outer turn, so it can hand back to `'main'`. A tool of the child can call `ctx.context.run.handoff`. See [Hand off a conversation](./handoffs). - The child harness cannot have its own `routing`. If it has one, the turn fails. +- `routing.history` does not apply to this child. It always gets the `'own'` fold of labeled text. In `subagents.agents`, the same `harnessAgent` stays a tool of the main model, with a new child session for each call. diff --git a/packages/ai-harness/src/define.ts b/packages/ai-harness/src/define.ts index eaf5cbba52..fd33a0f5af 100644 --- a/packages/ai-harness/src/define.ts +++ b/packages/ai-harness/src/define.ts @@ -141,8 +141,9 @@ export interface HandoffToolContext { /** The value of `routing.allowHandoffToHuman`. */ allowHuman: boolean /** - * Record a handoff with `by: 'tool'` and end the part. It adds no note - * messages: the tool call is in the transcript. + * Record a handoff with `by: 'tool'` and end the part. For the main + * model, the tool call is the record, so this adds no note messages. + * A root-agent tool handoff still adds the two note messages. */ handoff: (options: HandoffOptions) => Promise } @@ -195,7 +196,8 @@ export interface HarnessRouting extends Pick< * as labeled text (`name:\ntext`, `main:\ntext`), and a handoff call as * a text line with the note. The main model sees routed runs as * `name:\ntext`. - * - `'all'`: every speaker sees every routed run in full. + * - `'all'`: each routed host message becomes the child transcripts of + * that run. A root harnessAgent always gets the `'own'` fold. * - `'text'`: every speaker sees routed runs as `name:\ntext`. * * Default `'own'`. diff --git a/packages/ai-harness/src/handoff.ts b/packages/ai-harness/src/handoff.ts index 463e8c3e18..f0139eb503 100644 --- a/packages/ai-harness/src/handoff.ts +++ b/packages/ai-harness/src/handoff.ts @@ -502,6 +502,9 @@ export async function stampHandoffNote( await store.saveThread(threadId, history.with(index, updated)) return } + throw new Error( + `No ${toolName} call in the thread to stamp with the handoff note.`, + ) } /** diff --git a/packages/ai-harness/src/routed-history.ts b/packages/ai-harness/src/routed-history.ts index 95234ad64d..04f412e638 100644 --- a/packages/ai-harness/src/routed-history.ts +++ b/packages/ai-harness/src/routed-history.ts @@ -288,16 +288,18 @@ export async function historyFor( children.length > 0 ? await expandHost(message, children, wanted, transcriptOf) : undefined + // Unstamped multi-agent host (a list / { names } / { steps } pick + // never sets metadata.harness.agent). Fold as user text, even when + // this pass has no children left, and keep the existing name: labels. + // A stamped host still folds: its owner is the agent name. + const unstamped = + message.role === 'assistant' && + runId !== undefined && + routedAgent(message) === undefined && + !children.some((child) => wanted(child.name)) if (expanded) result.push(...expanded) else if (mode === 'all') result.push(message) - else if ( - children.length > 0 && - !children.some((child) => wanted(child.name)) - ) { - // Unstamped multi-agent host (a list / { names } / { steps } pick - // never sets metadata.harness.agent). Fold as user text and keep - // the existing name: labels, so the main model does not read them - // as its own. + else if (unstamped) { const text = textOf(message).trim() if (text !== '') result.push(otherSpeaker(message, text)) } else result.push(...foldFor(message, speaker, handoffToolName, folded)) diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index bafd352d4d..57dae69502 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -862,12 +862,10 @@ export abstract class SessionAgents< const threadId = `${this.rootThreadId}/agent/${root?.name ?? ctx.subagentRunId}` const used = emptyCounts() // With the handoff tool off, the child still hands off from code. + const withHandoffTool = + this.harness.routing?.giveModelsHandoffTool !== false const handoff = root - ? childHandoff( - root.name, - root.handoffs, - this.harness.routing?.giveModelsHandoffTool !== false, - ) + ? childHandoff(root.name, root.handoffs, withHandoffTool) : undefined const session = await open(harness, { threadId, @@ -935,7 +933,11 @@ export abstract class SessionAgents< open: (id) => this.mediaStore.open(id), }, session, - await rootInput(messages, root.name), + await rootInput( + messages, + root.name, + withHandoffTool ? root.handoffs.tool(null).name : undefined, + ), ) : lastUserInput(messages), ) @@ -2842,7 +2844,11 @@ function childHandoff(from: string, handoffs: TurnHandoffs, withTool: boolean) { * short continue prompt when that context is empty. On its first turn, the * context is the conversation so far. */ -async function rootInput(messages: Array, name: string) { +async function rootInput( + messages: Array, + name: string, + handoffToolName?: string, +) { // A handoff note names its source agent too, but it is no answer. const isAnswer = (message: ModelMessage) => routedAgent(message) === name && !isHandoffNote(message) @@ -2867,6 +2873,7 @@ async function rootInput(messages: Array, name: string) { mode: 'own', childrenOf: () => [], transcriptOf: async () => [], + ...(handoffToolName !== undefined && { handoffToolName }), }) // A message of the user comes back as the same object. A folded message // of another speaker is a new user message that has its label already. diff --git a/packages/ai-harness/tests/handoff-tool.test.ts b/packages/ai-harness/tests/handoff-tool.test.ts index b2539135f8..38ba4b13a0 100644 --- a/packages/ai-harness/tests/handoff-tool.test.ts +++ b/packages/ai-harness/tests/handoff-tool.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' import { z } from 'zod' import { defineAgent, toolDefinition } from '@tanstack/ai' -import { TurnHandoffs } from '../src/handoff' +import { TurnHandoffs, stampHandoffNote } from '../src/handoff' import type { AnyTool } from '@tanstack/ai' import type { HandoffToolContext } from '../src' import type { HandoffRecord } from '../src/log-records' @@ -224,3 +224,17 @@ describe('the handoff tool of a root agent', () => { ).toBeUndefined() }) }) + +describe('stampHandoffNote', () => { + it('throws when the thread has no call of that tool', async () => { + const store = { + loadThread: async () => [], + saveThread: async () => {}, + } + await expect( + stampHandoffNote(store, 'thread_1', 'handoff', 'Refund for order 42'), + ).rejects.toThrow( + 'No handoff call in the thread to stamp with the handoff note.', + ) + }) +}) diff --git a/packages/ai-harness/tests/routed-history.test.ts b/packages/ai-harness/tests/routed-history.test.ts index 43fee9c7e5..187662b864 100644 --- a/packages/ai-harness/tests/routed-history.test.ts +++ b/packages/ai-harness/tests/routed-history.test.ts @@ -244,6 +244,25 @@ describe('historyFor', () => { expect(result).toEqual([USER, { ...parallel, role: 'user' }]) }) + it('folds an unstamped host as user text when the run has no children', async () => { + const parallel: ModelMessage = { + id: 'assistant:r3', + role: 'assistant', + content: 'billing:\nRefunded\n\nshipping:\nShipped', + metadata: { tanstack: { runId: 'r3' } }, + } + const asUser = { ...parallel, role: 'user' as const } + const seen = async (speaker: string | null) => + history({ + messages: [USER, parallel], + speaker, + mode: 'own', + children: {}, + }) + expect(await seen(null)).toEqual([USER, asUser]) + expect(await seen('billing')).toEqual([USER, asUser]) + }) + it("keeps the main model's own calls and gives it routed answers as user text in 'own' mode", async () => { const call = handoffCall('call_h', 'handoff', { to: 'billing' }) const messages = [USER, call, toolResult('call_h'), BILLING_HOST] From 1ff246173560c98e20f6131b2693a2e2e01a75d7 Mon Sep 17 00:00:00 2001 From: TanStack Source Date: Sat, 21 Nov 2026 04:15:54 +0000 Subject: [PATCH 09/11] fix(ai-harness): keep routed child turns alive when parent media is missing Catch MediaError 404 in copyThreadMedia and keep the original part. Collect every non-text part from earlier user messages so HTTP images still reach a routed root harnessAgent. TanStack-Source-Factory-Run: factory-2145-b85bb259-f667-4a38-a7ac-994195e5ff70 TanStack-Source-Agent: fixer --- packages/ai-harness/src/session/agents.ts | 24 ++++--- .../tests/harness-agent-root.test.ts | 69 +++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/packages/ai-harness/src/session/agents.ts b/packages/ai-harness/src/session/agents.ts index 57dae69502..4cb0f4a2d6 100644 --- a/packages/ai-harness/src/session/agents.ts +++ b/packages/ai-harness/src/session/agents.ts @@ -11,7 +11,7 @@ import type { BlobBody } from '@tanstack/ai-persistence' import { redactMiddleware } from '../auth' import { BOUNDARY_INTERRUPTS, withoutBoundaryWaits } from '../boundary-waits' import { errorsOnlyDebug } from '../logger' -import { mediaCapture, mediaMiddleware } from '../media' +import { MediaError, mediaCapture, mediaMiddleware } from '../media' import { OperationImpl } from '../operation' import { checkpointMiddleware, @@ -2887,7 +2887,7 @@ async function rootInput( if (line !== '') lines.push(`user:\n${line}`) if (Array.isArray(message.content)) { for (const part of message.content) { - if (mediaIdOf(part) !== undefined) earlierMedia.push(part) + if (part.type !== 'text') earlierMedia.push(part) } } continue @@ -2940,15 +2940,23 @@ async function copyThreadMedia( const copy = async (id: string) => { if (!copies.has(id)) { const record = await source.getMedia(id) - copies.set( - id, - record && - (await dest.putMedia({ + let copied: MediaRecord | null = null + if (record) { + try { + copied = await dest.putMedia({ body: await source.open(id), mimeType: record.mimeType, name: record.name, - })), - ) + }) + } catch (error) { + // A missing blob must not abort the child turn: keep the original + // part so the child's media middleware can emit a not-found note. + if (!(error instanceof MediaError && error.status === 404)) { + throw error + } + } + } + copies.set(id, copied) } return copies.get(id) ?? undefined } diff --git a/packages/ai-harness/tests/harness-agent-root.test.ts b/packages/ai-harness/tests/harness-agent-root.test.ts index 350ed5c051..a0c91c54aa 100644 --- a/packages/ai-harness/tests/harness-agent-root.test.ts +++ b/packages/ai-harness/tests/harness-agent-root.test.ts @@ -242,6 +242,75 @@ describe('harnessAgent as a routed root agent', () => { await host.close() }) + it('completes a routed root harnessAgent turn when the parent blob is missing', async () => { + const encoder = new TextEncoder() + const model = mockAdapter([() => text('No image.')]) + const child = defineHarness({ + name: 'test/desk', + adapter: { ...model.adapter, inputModalities: ['text', 'image'] }, + }) + const main = mockAdapter([]) + const persistence = memoryPersistence() + const host = createHarnessHost({ persistence }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: { ...main.adapter, inputModalities: ['text', 'image'] }, + agents: [harnessAgent(child, { name: 'desk' })], + routing: { router: () => ({ name: 'desk' }) }, + }), + { threadId: THREAD }, + ) + const record = await session.putMedia({ + body: encoder.encode('hello'), + mimeType: 'image/png', + name: 'cat.png', + }) + const blobKey = (await persistence.stores.artifacts.get(record.id))?.blobKey + await persistence.stores.blobs.delete(blobKey ?? '') + await session.send([ + { type: 'text', content: 'What is this?' }, + mediaPart(record), + ]) + + expect(model.calls).toHaveLength(1) + const last = model.calls[0].messages.at(-1) + expect(last.role).toBe('user') + expect(JSON.stringify(last.content)).toContain('not found') + await host.close() + }) + + it('keeps an earlier user image URL in the conversation so far for a routed root harnessAgent', async () => { + const model = mockAdapter([() => text('A cat.')]) + const child = defineHarness({ + name: 'test/desk', + adapter: { ...model.adapter, inputModalities: ['text', 'image'] }, + }) + const main = mockAdapter([() => text('Main.')]) + const host = createHarnessHost({ persistence: memoryPersistence() }) + const session = await host.open( + defineHarness({ + name: 'test/front', + adapter: { ...main.adapter, inputModalities: ['text', 'image'] }, + agents: [harnessAgent(child, { name: 'desk' })], + routing: { router: picksInOrder([undefined, 'desk']) }, + }), + { threadId: THREAD }, + ) + const image = { + type: 'image', + source: { type: 'url', value: 'https://example.com/cat.png' }, + } + await session.send([{ type: 'text', content: 'Look at this.' }, image]) + await session.send('What is in the image?') + + const last = model.calls[0].messages.at(-1) + expect(last.role).toBe('user') + const parts = Array.isArray(last.content) ? last.content : [] + expect(parts).toEqual(expect.arrayContaining([image])) + await host.close() + }) + it('does not resend the original user message when invoked again in the same turn', async () => { const desk = deskHarness([ () => toolCall('handoff', { to: 'main' }), From d63c20f7d39d1c4314f699f02e8c48d30da4e7c6 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sat, 21 Nov 2026 16:22:58 +1100 Subject: [PATCH 10/11] test(ai-harness): keep the root image fixture as a content part The image object widened to string, so session.send had no matching overload. --- packages/ai-harness/tests/harness-agent-root.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ai-harness/tests/harness-agent-root.test.ts b/packages/ai-harness/tests/harness-agent-root.test.ts index a0c91c54aa..425d68a4a3 100644 --- a/packages/ai-harness/tests/harness-agent-root.test.ts +++ b/packages/ai-harness/tests/harness-agent-root.test.ts @@ -299,8 +299,8 @@ describe('harnessAgent as a routed root agent', () => { ) const image = { type: 'image', - source: { type: 'url', value: 'https://example.com/cat.png' }, - } + source: { type: 'url', value: 'https://tanstack.com/cat.png' }, + } as const await session.send([{ type: 'text', content: 'Look at this.' }, image]) await session.send('What is in the image?') From 2ef4af91bcbae6f876e4b9ef008f3bf53ac7b658 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sat, 21 Nov 2026 17:25:35 +1100 Subject: [PATCH 11/11] test(ai-harness): treat an unreaped Linux zombie as stopped Source CI's PID 1 never reaps a killed child, so kill(pid, 0) still succeeds. Read /proc to see the zombie, matching main. --- packages/ai-harness/tests/workspace-tools.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/ai-harness/tests/workspace-tools.test.ts b/packages/ai-harness/tests/workspace-tools.test.ts index 31d2017a1a..2198915b5b 100644 --- a/packages/ai-harness/tests/workspace-tools.test.ts +++ b/packages/ai-harness/tests/workspace-tools.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs' import { mkdtemp, readFile, realpath, rm, symlink } from 'node:fs/promises' import { tmpdir } from 'node:os' import * as nodePath from 'node:path' @@ -173,10 +174,18 @@ const toolNames = (call: { tools: Array<{ name: string }> }) => function isRunning(pid: number) { try { process.kill(pid, 0) - return true } catch { return false } + // A zombie still answers `kill(pid, 0)`. In a CI container whose PID 1 + // never reaps it (TanStack Source), a killed child stays a zombie, so read + // its state on Linux. With no /proc, the process counts as running. + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8') + return !stat.slice(stat.lastIndexOf(')') + 2).startsWith('Z') + } catch { + return true + } } describe.each(fixtures)('workspace tools on the $name backend', (fixture) => {