From 7f38b550341986daa3cb3bcf5d7b3f3a9bb0a0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Pe=CC=81rard?= Date: Tue, 17 Nov 2026 13:44:10 +0100 Subject: [PATCH 1/2] fix(ai): lazy discovery returns tools already offered instead of "Unknown tool" The discovery tool only knew lazy tools. When a model asked it for an eager tool, or for a tool a middleware added, it answered "Unknown tool" and some models then stopped using that tool. LazyToolManager now keeps the tools offered this step (the post-middleware set the engine passes to getExecutableTools) and discovery returns their definition, like it does for a re-requested lazy tool. An eager tool that a middleware removed still gets "Unknown tool". Co-Authored-By: Claude Opus 5.5 --- .changeset/lazy-discovery-offered-tools.md | 5 ++ docs/config.json | 2 +- docs/tools/lazy-tool-discovery.md | 2 + .../chat/tools/lazy-tool-manager.ts | 16 +++++- packages/ai/tests/lazy-tool-manager.test.ts | 56 +++++++++++++++++++ .../lazy-tool-discovery-offered.json | 41 ++++++++++++++ testing/e2e/src/lib/tools-test-tools.ts | 6 ++ testing/e2e/tests/lazy-tools.spec.ts | 32 +++++++++++ 8 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 .changeset/lazy-discovery-offered-tools.md create mode 100644 testing/e2e/fixtures/tools-test/lazy-tool-discovery-offered.json diff --git a/.changeset/lazy-discovery-offered-tools.md b/.changeset/lazy-discovery-offered-tools.md new file mode 100644 index 0000000000..1d3a35e7a2 --- /dev/null +++ b/.changeset/lazy-discovery-offered-tools.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai': patch +--- + +The lazy tool discovery tool no longer says `Unknown tool` for a tool that the model can already call. Before, it knew only the lazy tools. When a model asked it for an eager tool, or for a tool that a middleware added, it returned `Unknown tool`, and some models then stopped using that tool. Discovery now returns the definition of each tool offered to the model in that step. An eager tool that a middleware removed still gets `Unknown tool`. diff --git a/docs/config.json b/docs/config.json index bc2f2896e9..a266ace95e 100644 --- a/docs/config.json +++ b/docs/config.json @@ -402,7 +402,7 @@ "label": "Lazy Tool Discovery", "to": "tools/lazy-tool-discovery", "addedAt": "2026-04-15", - "updatedAt": "2026-10-22" + "updatedAt": "2026-11-17" }, { "label": "Tool Architecture", diff --git a/docs/tools/lazy-tool-discovery.md b/docs/tools/lazy-tool-discovery.md index c05f08972d..76cfce205d 100644 --- a/docs/tools/lazy-tool-discovery.md +++ b/docs/tools/lazy-tool-discovery.md @@ -143,6 +143,8 @@ The model can ask for one tool or for many in one call: __lazy__tool__discovery__({ toolNames: ["compareProducts", "calculateFinancing"] }) ``` +The model can also ask for a tool that it can already call: an eager tool, a tool it found before, or a tool that a middleware added. The discovery tool returns the definition of that tool too, so the model calls it. Any other name, such as an eager tool that a middleware removed, gets an `Unknown tool` error. + ## When to make a tool lazy Make a tool lazy when: diff --git a/packages/ai/src/activities/chat/tools/lazy-tool-manager.ts b/packages/ai/src/activities/chat/tools/lazy-tool-manager.ts index d4c2668b49..a1cbce1f86 100644 --- a/packages/ai/src/activities/chat/tools/lazy-tool-manager.ts +++ b/packages/ai/src/activities/chat/tools/lazy-tool-manager.ts @@ -25,6 +25,8 @@ export class LazyToolManager { private hasNewDiscoveries: boolean private readonly discoveryTool: Tool | null private readonly lazyToolsConfig: LazyToolsConfig + /** Tools offered to the model this turn, set by `getExecutableTools()`. */ + private offeredTools = new Map() constructor( tools: ReadonlyArray, @@ -117,6 +119,13 @@ export class LazyToolManager { activeTools: ReadonlyArray, pendingToolCallNames: ReadonlyArray, ): ReadonlyArray { + // `activeTools` is the set after middleware. The discovery tool reads it + // so it does not call an offered tool "Unknown". + this.offeredTools = new Map( + activeTools + .filter((t) => t.name !== DISCOVERY_TOOL_NAME) + .map((t) => [t.name, t]), + ) if ( this.discoveryTool && pendingToolCallNames.includes(DISCOVERY_TOOL_NAME) && @@ -263,13 +272,16 @@ export class LazyToolManager { const errors: Array = [] for (const name of args.toolNames) { - const tool = lazyToolMap.get(name) + const lazyTool = lazyToolMap.get(name) + // A tool already offered this turn (eager, discovered, or added by + // middleware) returns its schema like a re-discovered lazy tool. + const tool = lazyTool ?? manager.offeredTools.get(name) if (tool) { // Only flag a refresh for genuinely new discoveries. Re-requesting // an already-discovered tool still returns its schema below (the // model asked for it), but must not trigger a redundant tool-list // refresh + continue in the agent loop. - if (!manager.discoveredTools.has(name)) { + if (lazyTool && !manager.discoveredTools.has(name)) { manager.discoveredTools.add(name) manager.hasNewDiscoveries = true } diff --git a/packages/ai/tests/lazy-tool-manager.test.ts b/packages/ai/tests/lazy-tool-manager.test.ts index 38c99847df..2adb17f615 100644 --- a/packages/ai/tests/lazy-tool-manager.test.ts +++ b/packages/ai/tests/lazy-tool-manager.test.ts @@ -289,6 +289,62 @@ describe('LazyToolManager', () => { }) }) + describe('discovery of tools that are already offered', () => { + it('returns an eager tool instead of "Unknown tool"', async () => { + const manager = new LazyToolManager( + [makeTool('eagerA'), makeTool('lazyA', { lazy: true })], + [], + ) + const active = manager.getActiveTools() + const exec = manager.getExecutableTools(active, [DISCOVERY_TOOL_NAME]) + const discovery = exec.find((t) => t.name === DISCOVERY_TOOL_NAME) + + const result = await discovery!.execute!({ toolNames: ['eagerA'] }) + + expect(result.tools).toEqual([ + { name: 'eagerA', description: 'Tool: eagerA' }, + ]) + expect(result.errors).toBeUndefined() + expect(manager.hasNewlyDiscoveredTools()).toBe(false) + }) + + it('returns a tool that a middleware added to the offered set', async () => { + const manager = new LazyToolManager( + [makeTool('lazyA', { lazy: true })], + [], + ) + const offered = [...manager.getActiveTools(), makeTool('fromMiddleware')] + const exec = manager.getExecutableTools(offered, [DISCOVERY_TOOL_NAME]) + const discovery = exec.find((t) => t.name === DISCOVERY_TOOL_NAME) + + const result = await discovery!.execute!({ + toolNames: ['fromMiddleware'], + }) + + expect(result.tools.map((t: Tool) => t.name)).toEqual(['fromMiddleware']) + expect(result.errors).toBeUndefined() + }) + + it('still rejects an eager tool that a middleware filtered out', async () => { + const manager = new LazyToolManager( + [makeTool('eagerA'), makeTool('lazyA', { lazy: true })], + [], + ) + const offered = manager + .getActiveTools() + .filter((t) => t.name !== 'eagerA') + const exec = manager.getExecutableTools(offered, [DISCOVERY_TOOL_NAME]) + const discovery = exec.find((t) => t.name === DISCOVERY_TOOL_NAME) + + const result = await discovery!.execute!({ toolNames: ['eagerA'] }) + + expect(result.tools).toEqual([]) + expect(result.errors).toEqual([ + "Unknown tool: 'eagerA'. Available tools: [lazyA]", + ]) + }) + }) + describe('idempotent discovery', () => { it('re-discovering an already-discovered tool returns its schema but does not flag a new discovery', async () => { const tools = [ diff --git a/testing/e2e/fixtures/tools-test/lazy-tool-discovery-offered.json b/testing/e2e/fixtures/tools-test/lazy-tool-discovery-offered.json new file mode 100644 index 0000000000..394380364d --- /dev/null +++ b/testing/e2e/fixtures/tools-test/lazy-tool-discovery-offered.json @@ -0,0 +1,41 @@ +{ + "fixtures": [ + { + "match": { + "userMessage": "[lazy-tool-discovery-offered] run test", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "__lazy__tool__discovery__", + "arguments": "{\"toolNames\":[\"get_weather\"]}" + } + ] + } + }, + { + "match": { + "userMessage": "[lazy-tool-discovery-offered] run test", + "sequenceIndex": 1 + }, + "response": { + "toolCalls": [ + { + "name": "get_weather", + "arguments": "{\"city\":\"Paris\"}" + } + ] + } + }, + { + "match": { + "userMessage": "[lazy-tool-discovery-offered] run test", + "sequenceIndex": 2 + }, + "response": { + "content": "It is sunny in Paris." + } + } + ] +} diff --git a/testing/e2e/src/lib/tools-test-tools.ts b/testing/e2e/src/lib/tools-test-tools.ts index 2d9b3bde21..bf9e88501e 100644 --- a/testing/e2e/src/lib/tools-test-tools.ts +++ b/testing/e2e/src/lib/tools-test-tools.ts @@ -275,6 +275,11 @@ export const SCENARIO_LIST = [ label: 'Lazy Tool Discovery', category: 'basic', }, + { + id: 'lazy-tool-discovery-offered', + label: 'Lazy Discovery of an Offered Tool', + category: 'basic', + }, { id: 'custom-events', label: 'Custom Event Emitting', category: 'basic' }, { id: 'server-context', @@ -460,6 +465,7 @@ export function getToolsForScenario(scenario: string) { return [serverTools.get_weather] case 'lazy-tool-discovery': + case 'lazy-tool-discovery-offered': return [serverTools.get_weather, searchInventory] case 'custom-events': diff --git a/testing/e2e/tests/lazy-tools.spec.ts b/testing/e2e/tests/lazy-tools.spec.ts index 6f31228f2f..cc0d6699a5 100644 --- a/testing/e2e/tests/lazy-tools.spec.ts +++ b/testing/e2e/tests/lazy-tools.spec.ts @@ -5,6 +5,7 @@ import { waitForTestComplete, getMetadata, getToolCalls, + getMessages, } from './tools-test/helpers' test.describe('Lazy Tool Discovery', () => { @@ -21,4 +22,35 @@ test.describe('Lazy Tool Discovery', () => { expect(toolNames).toContain('__lazy__tool__discovery__') expect(toolNames).toContain('search_inventory') }) + + test('discovery of an already offered tool returns it, not "Unknown tool"', async ({ + page, + testId, + aimockPort, + }) => { + await selectScenario( + page, + 'lazy-tool-discovery-offered', + testId, + aimockPort, + ) + await runTest(page) + await waitForTestComplete(page, 20000, 2) + + const toolCalls = await getToolCalls(page) + expect(toolCalls.map((tc: any) => tc.name)).toEqual([ + '__lazy__tool__discovery__', + 'get_weather', + ]) + + const messages = await getMessages(page) + const discoveryResult = messages + .flatMap((m: any) => m.parts) + .find( + (p: any) => + p.type === 'tool-result' && p.toolCallId === toolCalls[0].id, + ) + expect(discoveryResult?.content).toContain('get_weather') + expect(discoveryResult?.content).not.toContain('Unknown tool') + }) }) From 04235bbac43c9e2e4f5a0c03946af81f887218c4 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Sat, 21 Nov 2026 13:56:54 +1100 Subject: [PATCH 2/2] Fix: correct the discovery docs for removed lazy tools, drop a no-signal e2e assertion docs and changeset: a lazy tool that a middleware removed still returns its definition e2e: remove toContain('get_weather'), which passes on unfixed code because the old error text contains the name --- .changeset/lazy-discovery-offered-tools.md | 2 +- docs/tools/lazy-tool-discovery.md | 2 +- testing/e2e/tests/lazy-tools.spec.ts | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.changeset/lazy-discovery-offered-tools.md b/.changeset/lazy-discovery-offered-tools.md index 1d3a35e7a2..da9152841b 100644 --- a/.changeset/lazy-discovery-offered-tools.md +++ b/.changeset/lazy-discovery-offered-tools.md @@ -2,4 +2,4 @@ '@tanstack/ai': patch --- -The lazy tool discovery tool no longer says `Unknown tool` for a tool that the model can already call. Before, it knew only the lazy tools. When a model asked it for an eager tool, or for a tool that a middleware added, it returned `Unknown tool`, and some models then stopped using that tool. Discovery now returns the definition of each tool offered to the model in that step. An eager tool that a middleware removed still gets `Unknown tool`. +The lazy tool discovery tool no longer says `Unknown tool` for a tool that the model can already call. Before, it knew only the lazy tools. When a model asked it for an eager tool, or for a tool that a middleware added, it returned `Unknown tool`, and some models then stopped using that tool. Discovery now returns the definition of each tool offered to the model in that step. An eager tool that a middleware removed still gets `Unknown tool`. A lazy tool that a middleware removed still returns its definition. diff --git a/docs/tools/lazy-tool-discovery.md b/docs/tools/lazy-tool-discovery.md index 76cfce205d..2276e76883 100644 --- a/docs/tools/lazy-tool-discovery.md +++ b/docs/tools/lazy-tool-discovery.md @@ -143,7 +143,7 @@ The model can ask for one tool or for many in one call: __lazy__tool__discovery__({ toolNames: ["compareProducts", "calculateFinancing"] }) ``` -The model can also ask for a tool that it can already call: an eager tool, a tool it found before, or a tool that a middleware added. The discovery tool returns the definition of that tool too, so the model calls it. Any other name, such as an eager tool that a middleware removed, gets an `Unknown tool` error. +The model can also ask for a tool that it can already call: an eager tool, a tool it found before, or a tool that a middleware added. The discovery tool returns the definition of that tool too, so the model can call it. A name that is not a lazy tool and that the model cannot call, such as an eager tool that a middleware removed, gets an `Unknown tool` error. A lazy tool always returns its definition, even when a middleware removed it. ## When to make a tool lazy diff --git a/testing/e2e/tests/lazy-tools.spec.ts b/testing/e2e/tests/lazy-tools.spec.ts index cc0d6699a5..f7b15a8c56 100644 --- a/testing/e2e/tests/lazy-tools.spec.ts +++ b/testing/e2e/tests/lazy-tools.spec.ts @@ -50,7 +50,6 @@ test.describe('Lazy Tool Discovery', () => { (p: any) => p.type === 'tool-result' && p.toolCallId === toolCalls[0].id, ) - expect(discoveryResult?.content).toContain('get_weather') expect(discoveryResult?.content).not.toContain('Unknown tool') }) })