diff --git a/.roo/skills/probe-vscode-lm-api/SKILL.md b/.roo/skills/probe-vscode-lm-api/SKILL.md new file mode 100644 index 0000000000..0150cd8afa --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/SKILL.md @@ -0,0 +1,99 @@ +--- +name: probe-vscode-lm-api +description: How to empirically probe the VS Code Language Model API (`vscode.lm`) with a scratch extension against a real extension host, and the measured findings about Copilot Claude models leaking tool-call markup into text. Use when asked to "test the vscode.lm API", "probe Copilot model behavior", "capture a raw LM transcript", "does the model leak tool-call markup", "verify LanguageModelToolCallPart behavior", or when reasoning about `extractLeakedToolCalls()` in the vscode-lm provider. +--- + +# Probing the VS Code LM API Empirically + +## When to Use This Skill + +- A claim is being made about what a Copilot-backed model _actually_ emits over `vscode.lm` (tool-call parts vs. text), and it needs evidence rather than inference. +- Changing [`extractLeakedToolCalls()`](../../../src/api/providers/vscode-lm.ts) or its guards, and you need a false-positive corpus. +- Any question that can only be answered by real `model.sendRequest()` traffic — the mocked unit tests cannot answer it. + +## When NOT to Use This Skill + +- Ordinary provider work covered by [`src/api/providers/__tests__/vscode-lm.spec.ts`](../../../src/api/providers/__tests__/vscode-lm.spec.ts). Live probing is slow and burns Copilot quota. +- Anything about non-`vscode-lm` providers. Anthropic-API behavior does not transfer. + +## Running the Probe + +Scripts live in [`scripts/probe-vscode-lm-api/`](../../../scripts/probe-vscode-lm-api/) at the repo root. + +1. Copy `scripts/probe-vscode-lm-api/package.json` and `scripts/probe-vscode-lm-api/extension.js` into a scratch directory, e.g. `\.tmp\lmprobe\`. No build, no `npm install` — it is plain CommonJS against the `vscode` module. +2. Adjust `OUT_DIR` at the top of the copied `extension.js` (or set `LM_PROBE_OUT_DIR`) to the transcript output directory. +3. Launch a **new** extension host window: + +``` +code --extensionDevelopmentPath=\.tmp\lmprobe --new-window +``` + +4. In that new window: `Ctrl+Shift+P` -> **LM Probe: Run** (or click "Run probe" on the toast). +5. Wait for the completion notification. Transcripts and `summary.json` land in `OUT_DIR`. + +Each run writes a `.json` of every stream part and a `.txt` of the exact concatenated text, named `____run`. + +### Delegate the UI driving + +Steps 3-5 involve a live window. Delegate them to **`ui-operator`** mode rather than doing them inline; screenshots and control trees consume large amounts of context. + +## Gotchas + +### The consent gate needs a real user gesture + +**Do not call `model.sendRequest()` from `activate()`.** Every request fails with: + +``` +Language model '' cannot be used by '.' +``` + +This is not a quota, auth, or manifest problem — `vscode.lm` grants consent only in response to a genuine user gesture. The probe must therefore be triggered from the Command Palette (or a notification button click). This is the single most expensive trap here; it silently fails 100% of requests and looks like an entitlement bug. + +### Never `Stop-Process` filtered on window title + +**WARNING:** During this experiment, killing processes matched by window title destroyed the user's unrelated VS Code windows and their unsaved work. + +Safe alternative: only ever _launch_ new windows with `--new-window`, and close the probe window by hand. Never bulk-terminate `Code.exe` by title, `MainWindowTitle`, or any other fuzzy match. + +### Run tests with pnpm, not npx + +``` +pnpm --dir src exec vitest run +``` + +Never `npx vitest` — it resolves a wrong hoisted 3.2.4 instead of the pinned 4.1.9 and produces phantom failures. + +## Measured Findings + +Sample: 210 live requests, 7 Copilot Claude models x 6 scenarios x 5 repeats, 0 errors. The raw transcripts were not retained; the counts below are the retained record of that run, and re-running the probe is the way to regenerate the underlying evidence. + +| Scenario | Setup | Runs | ``; 0 were bare. All 44 quoted-in-prose cases (E+F) were bare; 0 were wrapped. In this sample, _bare correlates with quoting and wrapped with genuine invocation_ — so requiring a `` wrapper would not have been the discriminator it appears to be. +- **No `antml:` prefix appeared** in any of the 210 runs. +- **Zero false positives.** Replaying `extractLeakedToolCalls()` over all 58 transcripts containing `\\.tmp\\transcripts" + +function write(name, data) { + fs.mkdirSync(OUT_DIR, { recursive: true }) + fs.writeFileSync(path.join(OUT_DIR, name), typeof data === "string" ? data : JSON.stringify(data, null, 2)) +} + +const READ_TOOL = { + name: "read_file", + description: "Read the contents of a file at the given path.", + inputSchema: { + type: "object", + properties: { path: { type: "string", description: "File path to read" } }, + required: ["path"], + }, +} + +const TOOL_SYSTEM_PROMPT = [ + "You are Zoo Code, an autonomous coding agent.", + "You accomplish tasks by calling the tools provided to you.", + "You MUST call exactly one tool per message. Never ask the user a question.", + "Do not answer from memory; always read the file first using the read_file tool.", +].join("\n") + +async function runOnce(model, scenario) { + const record = { + scenario: scenario.name, + modelId: model.id, + modelFamily: model.family, + modelVendor: model.vendor, + modelVersion: model.version, + maxInputTokens: model.maxInputTokens, + parts: [], + concatenatedText: "", + toolCallParts: [], + error: null, + } + const messages = [] + if (scenario.system) { + messages.push(vscode.LanguageModelChatMessage.Assistant(scenario.system)) + } + for (const userText of scenario.userMessages) { + messages.push(vscode.LanguageModelChatMessage.User(userText)) + } + const options = { justification: "Empirical probe of leaked tool-call formatting." } + if (scenario.tools) { + options.tools = [READ_TOOL] + } + const source = new vscode.CancellationTokenSource() + try { + const response = await model.sendRequest(messages, options, source.token) + for await (const chunk of response.stream) { + const typeName = chunk && chunk.constructor ? chunk.constructor.name : typeof chunk + if (chunk instanceof vscode.LanguageModelTextPart) { + record.parts.push({ type: typeName, value: chunk.value }) + record.concatenatedText += chunk.value + } else if (chunk instanceof vscode.LanguageModelToolCallPart) { + const call = { type: typeName, name: chunk.name, callId: chunk.callId, input: chunk.input } + record.parts.push(call) + record.toolCallParts.push(call) + } else { + record.parts.push({ type: typeName, raw: String(chunk) }) + } + } + } catch (error) { + record.error = { name: error && error.name, message: error && error.message, stack: error && error.stack } + } finally { + source.dispose() + } + + const text = record.concatenatedText + record.markers = { + hasInvoke: /<(?:antml:)?invoke\b/i.test(text), + hasFunctionCalls: /<(?:antml:)?function_calls\b/i.test(text), + hasAntmlNamespace: /antml:/i.test(text), + invokeWrappedInFunctionCalls: /<(?:antml:)?function_calls\s*>[\s\S]*?<(?:antml:)?invoke\b/i.test(text), + bareInvokeWithoutWrapper: /<(?:antml:)?invoke\b/i.test(text) && !/<(?:antml:)?function_calls\b/i.test(text), + insideFencedCodeBlock: /```[\s\S]*?<(?:antml:)?invoke\b/i.test(text), + } + return record +} + +function buildScenarios() { + const longFiller = "This is filler context line used to grow the prompt toward the context window. ".repeat(4000) + return [ + { + name: "A_tools_declared_compelling_prompt", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: ["Read the file c:/git/Zoo-Code/package.json and tell me the version field."], + }, + { + name: "B_tools_declared_no_system_prompt", + tools: true, + userMessages: ["Read the file c:/git/Zoo-Code/package.json and tell me the version field."], + }, + { + name: "C_tools_declared_long_context", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: [longFiller, "Now read the file c:/git/Zoo-Code/package.json and tell me the version field."], + }, + { + name: "D_no_tools_asked_to_emit_markup", + tools: false, + system: TOOL_SYSTEM_PROMPT, + userMessages: [ + "You have a tool named read_file that takes a path. Emit the tool invocation using Anthropic's internal function-call XML format, exactly as you would internally, to read c:/git/Zoo-Code/package.json. Output only the markup.", + ], + }, + { + name: "E_quoted_markup_in_prose_false_positive_check", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: [ + 'Do NOT call any tool. Instead, explain in prose what an invoke block looks like, and include a literal example using the tag name "invoke" with a name attribute of read_file and a parameter named path, written as plain text in your answer.', + ], + }, + { + name: "F_quoted_markup_in_fenced_code_block", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: [ + "Do NOT call any tool. Show me, inside a fenced markdown code block, an example of an invoke block naming the tool read_file with a parameter named path set to /etc/passwd. Only output the fenced code block.", + ], + }, + ] +} + +async function run() { + const summary = { startedAt: new Date().toISOString(), vscodeVersion: vscode.version, models: [], runs: [] } + let models = [] + let selectError = null + try { + models = (await vscode.lm.selectChatModels({ vendor: "copilot" })) || [] + } catch (error) { + selectError = { name: error && error.name, message: error && error.message } + } + summary.selectError = selectError + summary.allModels = models.map((model) => ({ + id: model.id, + family: model.family, + vendor: model.vendor, + version: model.version, + maxInputTokens: model.maxInputTokens, + })) + + const claudeModels = models.filter((model) => /claude/i.test(model.id) || /claude/i.test(model.family)) + summary.claudeModelIds = claudeModels.map((model) => model.id) + + if (claudeModels.length === 0) { + write("summary.json", summary) + vscode.window.showErrorMessage( + `LM Probe: no Claude models. selectChatModels returned ${models.length}. ${selectError ? selectError.message : ""}`, + ) + return + } + + const scenarios = buildScenarios() + const REPEATS = 5 + for (const model of claudeModels) { + for (const scenario of scenarios) { + for (let iter = 1; iter <= REPEATS; iter++) { + const record = await runOnce(model, scenario) + record.iteration = iter + summary.runs.push({ + scenario: scenario.name, + modelId: model.id, + iteration: iter, + markers: record.markers, + toolCallPartCount: record.toolCallParts.length, + textLength: record.concatenatedText.length, + error: record.error ? record.error.message : null, + }) + write(`${model.id}__${scenario.name}__run${iter}.json`, record) + write(`${model.id}__${scenario.name}__run${iter}.txt`, record.concatenatedText) + } + } + } + + summary.finishedAt = new Date().toISOString() + write("summary.json", summary) + vscode.window.showInformationMessage(`LM Probe complete: ${summary.runs.length} runs written to ${OUT_DIR}`) +} + +function activate(context) { + // MUST be user-initiated: vscode.lm consent is only granted from a real user gesture, so an + // activation-time sendRequest is auto-denied with "cannot be used by 'scratch.lmprobe'". + context.subscriptions.push( + vscode.commands.registerCommand("lmprobe.run", () => + run().catch((error) => { + write("fatal.json", { message: String(error && error.message), stack: String(error && error.stack) }) + }), + ), + ) + vscode.window.showInformationMessage("LM Probe ready", "Run probe").then((choice) => { + if (choice === "Run probe") { + vscode.commands.executeCommand("lmprobe.run") + } + }) +} + +module.exports = { activate, deactivate() {} } diff --git a/scripts/probe-vscode-lm-api/package.json b/scripts/probe-vscode-lm-api/package.json new file mode 100644 index 0000000000..8ad0174674 --- /dev/null +++ b/scripts/probe-vscode-lm-api/package.json @@ -0,0 +1,21 @@ +{ + "name": "lmprobe", + "displayName": "LM Leak Probe", + "version": "0.0.1", + "publisher": "scratch", + "engines": { + "vscode": "^1.95.0" + }, + "activationEvents": [ + "onStartupFinished" + ], + "main": "./extension.js", + "contributes": { + "commands": [ + { + "command": "lmprobe.run", + "title": "LM Probe: Run" + } + ] + } +} diff --git a/scripts/probe-vscode-lm-api/probe-false-positives.spec.ts b/scripts/probe-vscode-lm-api/probe-false-positives.spec.ts new file mode 100644 index 0000000000..5c48318ec0 --- /dev/null +++ b/scripts/probe-vscode-lm-api/probe-false-positives.spec.ts @@ -0,0 +1,26 @@ +import fs from "fs" +import path from "path" + +// Paths here are written for the copy destination `src/api/providers/__tests__/`, not for this +// file's committed location — it is a template to be copied there, never run in place. +import { extractLeakedToolCalls } from "../vscode-lm" + +// Point this at the probe's OUT_DIR. Scratch harness: not a committed test. +const TRANSCRIPTS = process.env.LM_PROBE_TRANSCRIPTS ?? path.resolve(__dirname, "../../../../.tmp/transcripts") + +describe("probe: false-positive check against real transcripts", () => { + it("reports which transcripts extractLeakedToolCalls would treat as real calls", () => { + const names = new Set(["read_file"]) + const report: string[] = [] + for (const file of fs.readdirSync(TRANSCRIPTS).filter((name) => name.endsWith(".txt"))) { + const text = fs.readFileSync(path.join(TRANSCRIPTS, file), "utf8") + if (!/<(?:antml:)?invoke/i.test(text)) { + continue + } + const { calls } = extractLeakedToolCalls(text, names) + report.push(`${calls.length > 0 ? "RECOVERED" : "passthrough"}\t${file}\t${JSON.stringify(calls)}`) + } + fs.writeFileSync(path.join(TRANSCRIPTS, "..", "false-positive-report.txt"), report.join("\n")) + console.log(report.join("\n")) + }) +}) diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index 37fb851720..95be295c3c 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -60,7 +60,13 @@ vi.mock("vscode", () => { }) import * as vscode from "vscode" -import { VsCodeLmHandler } from "../vscode-lm" +import { + VsCodeLmHandler, + extractLeakedToolCalls, + trailingPartialToolMarkerLength, + middleOutTruncate, + truncateToolResultsToFitWindow, +} from "../vscode-lm" import type { ApiHandlerOptions } from "../../../shared/api" import type { Anthropic } from "@anthropic-ai/sdk" import { openAiModelInfoSaneDefaults, vscodeLlmDefaultModelId, vscodeLlmModels } from "@roo-code/types" @@ -270,6 +276,229 @@ describe("VsCodeLmHandler", () => { }) }) + describe("leaked tool-call recovery during streaming", () => { + const salvageTools = [ + { + type: "function" as const, + function: { + name: "calculator", + description: "A simple calculator", + parameters: { type: "object", properties: { operation: { type: "string" } } }, + }, + }, + ] + + const streamTextParts = (parts: string[]) => { + mockLanguageModelChat.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + for (const part of parts) { + yield new vscode.LanguageModelTextPart(part) + } + return + })(), + text: (async function* () { + yield parts.join("") + return + })(), + }) + } + + const streamMixedParts = (parts: Array) => { + mockLanguageModelChat.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + for (const part of parts) { + yield typeof part === "string" + ? new vscode.LanguageModelTextPart(part) + : new vscode.LanguageModelToolCallPart("native-1", part.name, part.input) + } + return + })(), + text: (async function* () { + yield "" + return + })(), + }) + } + + const drain = async () => { + const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], { + taskId: "test-task", + tools: salvageTools, + }) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks + } + + const collect = async (parts: string[]) => { + streamTextParts(parts) + const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], { + taskId: "test-task", + tools: salvageTools, + }) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks + } + + it("recovers a tool call the model streamed as raw invoke XML", async () => { + const chunks = await collect([ + "Thinking. ", + 'add', + ]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "Thinking. " }]) + expect(chunks.filter((chunk) => chunk.type === "tool_call")).toEqual([ + { + type: "tool_call", + id: expect.stringContaining("vscodelm-salvaged-"), + name: "calculator", + arguments: JSON.stringify({ operation: "add" }), + }, + ]) + }) + + it("detects a marker split across stream chunks", async () => { + const chunks = await collect([ + "abc sub', + ]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "abc " }]) + expect(chunks.filter((chunk) => chunk.type === "tool_call")).toMatchObject([ + { name: "calculator", arguments: JSON.stringify({ operation: "sub" }) }, + ]) + }) + + it("emits a carried tail as plain text when it never becomes a marker", async () => { + const chunks = await collect(["hello chunk.type === "text")).toEqual([ + { type: "text", text: "hello " }, + { type: "text", text: " chunk.type === "tool_call")).toBe(false) + }) + + it("buffers across chunks that arrive after the marker", async () => { + const chunks = await collect([ + 'prose ', + '', + "mul", + "", + ]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "prose " }]) + expect(chunks.filter((chunk) => chunk.type === "tool_call")).toMatchObject([ + { name: "calculator", arguments: JSON.stringify({ operation: "mul" }) }, + ]) + }) + + it("keeps an invoke block for an unknown tool as literal text", async () => { + const block = '1' + const chunks = await collect([block]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: block }]) + expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) + }) + + it("emits prose before the recovered tool call", async () => { + const chunks = await collect([ + "Thinking. ", + 'add', + ]) + + expect(chunks.map((chunk) => chunk.type)).toEqual(["text", "tool_call", "usage"]) + }) + + it("flushes buffered text before a native tool call so no text follows a tool_use", async () => { + streamMixedParts([ + 'partial ', + { name: "calculator", input: { operation: "div" } }, + ]) + const chunks = await drain() + + const lastText = chunks.map((chunk) => chunk.type).lastIndexOf("text") + const firstToolCall = chunks.map((chunk) => chunk.type).indexOf("tool_call") + expect(firstToolCall).toBeGreaterThan(lastText) + }) + + it("does not latch buffering on prose that merely mentions the tag", async () => { + const chunks = await collect(["never emit markup as text. ", "Streaming continues."]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([ + { type: "text", text: "never emit markup as text. " }, + { type: "text", text: "Streaming continues." }, + ]) + expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) + }) + + it("does not recover an invoke block quoted inside a fenced code block", async () => { + const block = 'add' + const chunks = await collect(["Do NOT do this:\n```\n" + block + "\n```\n"]) + + expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) + }) + + it("flushes an over-long never-closing invoke as plain text before the stream ends", async () => { + // Defect 4: without a cap the buffer is only drained once the stream finishes, so the + // user sees nothing until then. Releasing it at the end looks identical in content — + // only the timing distinguishes the fix, so track how much of the source has been + // produced at the moment each text chunk reaches the consumer. + const filler = "x".repeat(5000) + const parts = ['', filler, filler, filler, filler] + let partsProduced = 0 + + mockLanguageModelChat.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + for (const part of parts) { + partsProduced++ + yield new vscode.LanguageModelTextPart(part) + } + return + })(), + text: (async function* () { + yield "" + return + })(), + }) + + const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], { + taskId: "test-task", + tools: salvageTools, + }) + + let sawTextBeforeStreamEnd = false + let streamedText = "" + for await (const chunk of stream) { + if (chunk.type === "text") { + streamedText += chunk.text + if (partsProduced < parts.length) { + sawTextBeforeStreamEnd = true + } + } + } + + expect(sawTextBeforeStreamEnd).toBe(true) + expect(streamedText).toContain('') + expect(streamedText).toContain(filler) + }) + + it("sanitizes lone surrogates in the system prompt", async () => { + streamTextParts(["ok"]) + const stream = handler.createMessage("sys\uD800tem", [{ role: "user" as const, content: "hi" }]) + for await (const _chunk of stream) { + // drain + } + + expect(vscode.LanguageModelChatMessage.Assistant).toHaveBeenCalledWith("sys\uFFFDtem") + }) + }) + it("should handle native tool calls when tools are provided", async () => { const systemPrompt = "You are a helpful assistant" const messages: Anthropic.Messages.MessageParam[] = [ @@ -1075,3 +1304,393 @@ describe("VsCodeLmHandler", () => { }) }) }) + +describe("leaked tool-call recovery", () => { + // Builders keep the XML fixtures readable and prevent this file's own markup from being + // mistaken for a real tool call. + const invoke = (name: string, body: string) => `${body}` + const param = (name: string, value: string) => `${value}` + + describe("extractLeakedToolCalls", () => { + it("recovers a known-tool block and strips it from the leftover text", () => { + const text = `Working on it.\n${invoke("update_todo_list", param("todos", "[x] one\n[ ] two"))}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toEqual([{ name: "update_todo_list", input: { todos: "[x] one\n[ ] two" } }]) + expect(leftoverText).toBe("Working on it.\n") + }) + + it("recovers an unwrapped leak preceded by a stray token", () => { + const text = `court\n${invoke("update_todo_list", param("todos", "[x] done"))}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toEqual([{ name: "update_todo_list", input: { todos: "[x] done" } }]) + expect(leftoverText).toBe("court\n") + }) + + it("recovers multiple params and strips function-call wrapper tags", () => { + const body = param("mode", "code") + param("message", "go") + const text = `${invoke("new_task", body)}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["new_task"])) + + expect(calls).toEqual([{ name: "new_task", input: { mode: "code", message: "go" } }]) + expect(leftoverText).toBe("") + }) + + it("passes through invoke blocks for tools that were not offered", () => { + const text = invoke("some_other_tool", param("x", "1")) + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toEqual([]) + expect(leftoverText).toBe(text) + }) + + it("returns no calls for ordinary text", () => { + const { calls, leftoverText } = extractLeakedToolCalls("just a normal reply", new Set(["update_todo_list"])) + + expect(calls).toEqual([]) + expect(leftoverText).toBe("just a normal reply") + }) + }) + + describe("trailingPartialToolMarkerLength", () => { + it("holds back a split marker prefix at the end of a chunk", () => { + expect(trailingPartialToolMarkerLength("some text { + expect(trailingPartialToolMarkerLength("hello world")).toBe(0) + expect(trailingPartialToolMarkerLength("a < b")).toBe(0) + expect(trailingPartialToolMarkerLength("text ")).toBe(0) + }) + + it("holds back an invoke tag whose name attribute has not arrived", () => { + expect(trailingPartialToolMarkerLength("text { + expect(trailingPartialToolMarkerLength(" { + expect(trailingPartialToolMarkerLength("text <" + "a".repeat(200))).toBe(0) + }) + }) + + describe("quoted markup", () => { + it("does not recover an invoke block inside a fenced code block", () => { + const text = "```\n" + invoke("update_todo_list", param("todos", "[x] one")) + "\n```" + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toContain("invoke") + }) + + it("does not recover an invoke block inside an inline code span", () => { + const text = "avoid `" + invoke("update_todo_list", param("todos", "x")) + "`" + + const { calls } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + }) + + it("does not recover an invoke block quoted in unfenced, backtick-free prose", () => { + const text = "You must never emit " + invoke("update_todo_list", param("todos", "x")) + " directly." + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toBe(text) + }) + + it("does not recover a quoted invoke block that ends its line", () => { + // Defect 3: an empty rest-of-line previously made this look like a genuine leak. + const text = "You must never emit " + invoke("update_todo_list", param("todos", "x")) + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toBe(text) + }) + + it("does not recover an invoke block inside a tilde fence", () => { + const text = "~~~\n" + invoke("update_todo_list", param("todos", "[x] one")) + "\n~~~" + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toBe(text) + }) + + it("does not recover an invoke block inside a four-backtick fence", () => { + const text = "````\n" + invoke("update_todo_list", param("todos", "[x] one")) + "\n````" + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toBe(text) + }) + + it("does not treat doubled angle brackets as trailing prose after stripping", () => { + // Defect 1: a single strip pass turns `<>` into a tag-looking ``, so the + // trailing-text check must strip repeatedly until stable. + const text = invoke("update_todo_list", param("todos", "x")) + "<