From b4e17274ed657046838fcb569768612a5632be96 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 7 Aug 2026 15:02:26 -0700 Subject: [PATCH 1/9] fix(vscode-lm): sanitize surrogates, recover leaked tool calls, and window-safe tool_result truncation Hardens the VS Code Language Model provider (notably GitHub Copilot serving Anthropic Claude) against three failure modes: - Surrogate sanitization: a lone UTF-16 surrogate cannot be encoded as UTF-8, so the backend rejects the entire request with a 400. sanitizeSurrogates() replaces unpaired surrogates with U+FFFD while preserving valid pairs (emoji, CJK ext.), applied to string messages, tool results, and text parts. - Leaked tool-call recovery: some backends stream a tool call as raw XML instead of a structured LanguageModelToolCallPart, leaving the turn with no tool_use block and stalling the task in a "no tools used" retry loop. extractLeakedToolCalls() and trailingPartialToolMarkerLength() detect the markup mid-stream (including markers split across chunk boundaries) and replay it as a real tool call, conservatively: only for names matching a tool actually offered that turn, and only when tools were offered. - Window-safe tool_result truncation: Copilot's backend trims over-window requests without preserving tool_use/tool_result pairing, orphaning a tool_result and causing a 400 (unexpected tool_use_id). truncateToolResultsToFitWindow() and middleOutTruncate() shrink oversized tool_result payloads on our side (largest first, middle-out, pairing preserved) before sending. Ported from simurg79/Roo-Code#12. --- src/api/providers/__tests__/vscode-lm.spec.ts | 197 +++++++++- src/api/providers/vscode-lm.ts | 340 +++++++++++++++++- .../__tests__/vscode-lm-format.spec.ts | 38 +- src/api/transform/vscode-lm-format.ts | 30 +- 4 files changed, 594 insertions(+), 11 deletions(-) diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index 37fb851720..5cd11ddb65 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -60,7 +60,13 @@ vi.mock("vscode", () => { }) import * as vscode from "vscode" -import { VsCodeLmHandler } from "../vscode-lm" +import { + VsCodeLmHandler, + extractLeakedToolCalls, + trailingPartialToolMarkerLength, + middleOutTruncate, + truncateToolResultsToFitWindow, +} from "../vscode-lm" import type { ApiHandlerOptions } from "../../../shared/api" import type { Anthropic } from "@anthropic-ai/sdk" import { openAiModelInfoSaneDefaults, vscodeLlmDefaultModelId, vscodeLlmModels } from "@roo-code/types" @@ -1075,3 +1081,192 @@ describe("VsCodeLmHandler", () => { }) }) }) + +describe("leaked tool-call recovery", () => { + // Builders keep the XML fixtures readable and prevent this file's own markup from being + // mistaken for a real tool call. + const invoke = (name: string, body: string) => `${body}` + const param = (name: string, value: string) => `${value}` + + describe("extractLeakedToolCalls", () => { + it("recovers a known-tool block and strips it from the leftover text", () => { + const text = `Working on it.\n${invoke("update_todo_list", param("todos", "[x] one\n[ ] two"))}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toEqual([{ name: "update_todo_list", input: { todos: "[x] one\n[ ] two" } }]) + expect(leftoverText).toBe("Working on it.\n") + }) + + it("recovers an unwrapped leak preceded by a stray token", () => { + const text = `court\n${invoke("update_todo_list", param("todos", "[x] done"))}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toEqual([{ name: "update_todo_list", input: { todos: "[x] done" } }]) + expect(leftoverText).toBe("court\n") + }) + + it("recovers multiple params and strips function-call wrapper tags", () => { + const body = param("mode", "code") + param("message", "go") + const text = `${invoke("new_task", body)}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["new_task"])) + + expect(calls).toEqual([{ name: "new_task", input: { mode: "code", message: "go" } }]) + expect(leftoverText).toBe("") + }) + + it("passes through invoke blocks for tools that were not offered", () => { + const text = invoke("some_other_tool", param("x", "1")) + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toEqual([]) + expect(leftoverText).toBe(text) + }) + + it("returns no calls for ordinary text", () => { + const { calls, leftoverText } = extractLeakedToolCalls("just a normal reply", new Set(["update_todo_list"])) + + expect(calls).toEqual([]) + expect(leftoverText).toBe("just a normal reply") + }) + }) + + describe("trailingPartialToolMarkerLength", () => { + it("holds back a split marker prefix at the end of a chunk", () => { + expect(trailingPartialToolMarkerLength("some text { + expect(trailingPartialToolMarkerLength("hello world")).toBe(0) + expect(trailingPartialToolMarkerLength("a < b")).toBe(0) + expect(trailingPartialToolMarkerLength("text ")).toBe(0) + }) + }) +}) + +describe("context-window tool_result truncation", () => { + describe("middleOutTruncate", () => { + it("returns text unchanged when within the limit", () => { + expect(middleOutTruncate("hello world", 100)).toBe("hello world") + }) + + it("keeps the head and tail and inserts a truncation marker", () => { + const text = "A".repeat(500) + "B".repeat(500) + const result = middleOutTruncate(text, 200) + + expect(result.length).toBeLessThanOrEqual(200) + expect(result).toContain("characters truncated to fit the model context window") + expect(result.startsWith("A")).toBe(true) + expect(result.endsWith("B")).toBe(true) + }) + + it("returns an empty string for a non-positive limit", () => { + expect(middleOutTruncate("anything", 0)).toBe("") + }) + }) + + describe("truncateToolResultsToFitWindow", () => { + const toolUseMessage = (id: string): Anthropic.Messages.MessageParam => ({ + role: "assistant", + content: [ + { type: "text", text: "Calling a tool." }, + { type: "tool_use", id, name: "some_tool", input: { a: 1 } }, + ], + }) + + const toolResultMessage = (id: string, content: string): Anthropic.Messages.MessageParam => ({ + role: "user", + content: [ + { type: "tool_result", tool_use_id: id, content }, + { type: "text", text: "env" }, + ], + }) + + const findBlock = (message: Anthropic.Messages.MessageParam, type: string) => + (message.content as unknown as Array<{ type: string; [key: string]: unknown }>).find( + (block) => block.type === type, + )! + + it("is a no-op when the conversation already fits the budget", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "small result"), + ] + const before = JSON.parse(JSON.stringify(messages)) + + truncateToolResultsToFitWindow(messages, 100_000) + + expect(messages).toEqual(before) + }) + + it("shrinks an oversized tool_result so the conversation fits the budget", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "X".repeat(50_000)), + ] + + truncateToolResultsToFitWindow(messages, 10_000) + + const toolResult = findBlock(messages[1], "tool_result") + expect(toolResult.tool_use_id).toBe("t1") // pairing preserved + expect(String(toolResult.content).length).toBeLessThanOrEqual(10_000) + expect(String(toolResult.content)).toContain("characters truncated") + }) + + it("truncates the largest tool_result first and leaves small ones intact", () => { + const small = "small but real result" + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "B".repeat(40_000)), + toolUseMessage("t2"), + toolResultMessage("t2", small), + ] + + truncateToolResultsToFitWindow(messages, 12_000) + + expect(String(findBlock(messages[1], "tool_result").content)).toContain("characters truncated") + expect(findBlock(messages[3], "tool_result").content).toBe(small) // untouched + }) + + it("never truncates tool_use blocks, assistant text, or environment details", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "X".repeat(50_000)), + ] + + truncateToolResultsToFitWindow(messages, 8_000) + + expect(findBlock(messages[0], "text").text).toBe("Calling a tool.") + expect(findBlock(messages[0], "tool_use")).toMatchObject({ id: "t1", name: "some_tool" }) + expect(findBlock(messages[1], "text").text).toBe("env") + }) + + it("handles array-form tool_result content and keeps it valid", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "t1", + content: [{ type: "text", text: "Y".repeat(40_000) }], + }, + ], + }, + ] + + truncateToolResultsToFitWindow(messages, 8_000) + + const toolResult = findBlock(messages[1], "tool_result") + expect(toolResult.tool_use_id).toBe("t1") + expect(Array.isArray(toolResult.content)).toBe(true) + const parts = toolResult.content as Array<{ type: string; text?: string }> + expect(parts[0].type).toBe("text") + expect(String(parts[0].text)).toContain("characters truncated") + }) + }) +}) diff --git a/src/api/providers/vscode-lm.ts b/src/api/providers/vscode-lm.ts index c657e6c0d6..d72779fbec 100644 --- a/src/api/providers/vscode-lm.ts +++ b/src/api/providers/vscode-lm.ts @@ -60,6 +60,251 @@ function convertToVsCodeLmTools(tools: OpenAI.Chat.ChatCompletionTool[]): vscode * } * ``` */ +/** + * Recovery for leaked tool calls + * ------------------------------ + * Some VS Code LM backends — notably GitHub Copilot serving Anthropic Claude models — + * intermittently stream a tool call as PLAIN TEXT using Anthropic's internal function-call + * XML instead of emitting a structured `LanguageModelToolCallPart`. When this happens the + * assistant turn contains no tool_use block, so Zoo reports "no tools used" and the task stalls + * in a retry loop. The helpers below detect the leaked markup mid-stream and replay it as a real + * tool call. Recovery is deliberately conservative: only `` blocks whose name matches a + * tool we actually offered this turn are treated as calls; everything else is passed through + * unchanged as text. + */ +const LEAKED_TOOL_CALL_START = /<(?:antml:)?(?:function_calls|invoke)\b/i +const LEAKED_INVOKE_BLOCK = /<(?:antml:)?invoke\s+name="([^"]+)"\s*>([\s\S]*?)<\/(?:antml:)?invoke\s*>/gi +const LEAKED_INVOKE_PARAM = /<(?:antml:)?parameter\s+name="([^"]+)"\s*>([\s\S]*?)<\/(?:antml:)?parameter\s*>/gi + +/** + * Returns the length of a trailing ` { + const input: Record = {} + LEAKED_INVOKE_PARAM.lastIndex = 0 + let match: RegExpExecArray | null + while ((match = LEAKED_INVOKE_PARAM.exec(body)) !== null) { + input[match[1]] = match[2].trim() + } + return input +} + +/** + * Extracts complete leaked `` tool-call blocks from `text`. Only blocks whose name + * is present in `validToolNames` are returned as calls; all other text (including `` + * blocks for unknown names) is returned as `leftoverText` so legitimate prose is preserved. + */ +export function extractLeakedToolCalls( + text: string, + validToolNames: ReadonlySet, +): { calls: Array<{ name: string; input: Record }>; leftoverText: string } { + const calls: Array<{ name: string; input: Record }> = [] + let leftover = "" + let lastIndex = 0 + + LEAKED_INVOKE_BLOCK.lastIndex = 0 + let match: RegExpExecArray | null + while ((match = LEAKED_INVOKE_BLOCK.exec(text)) !== null) { + leftover += text.slice(lastIndex, match.index) + const name = match[1] + if (validToolNames.has(name)) { + calls.push({ name, input: parseLeakedInvokeParams(match[2]) }) + } else { + // Not one of our tools — keep the block as literal text. + leftover += match[0] + } + lastIndex = match.index + match[0].length + } + leftover += text.slice(lastIndex) + + // Remove bare function-call wrapper tags left behind (cosmetic; also avoids re-teaching + // the model this format when the turn is later sent back as history). + leftover = leftover.replace(/<\/?(?:antml:)?function_calls\s*>/gi, "") + + return { calls, leftoverText: leftover } +} + +/** + * Context-window safety for Copilot's backend + * ------------------------------------------- + * Copilot's backend enforces its own context window and, for third-party `sendRequest` callers, + * trims an over-window request in a way that is NOT tool-pair-aware: it can drop the assistant + * message holding a `tool_use` while keeping the matching `tool_result`, after which Anthropic + * rejects the request with "unexpected tool_use_id". To keep trimming on OUR side — where + * pairing is preserved — we shrink oversized `tool_result` payloads before sending. Only + * `tool_result` text is truncated (never `tool_use`, assistant text, summaries, or environment + * details), and only when the request would otherwise exceed the budget. + */ + +/** + * Conservative characters-per-token ratio used to turn a token window into a character budget. + * The token-dense JSON, logs, and code that dominate oversized tool results tokenize to fewer + * characters per token than prose, so we intentionally under-count (3, not the ~4 typical of + * English) to keep the resulting budget on the safe side of the enforced window. + */ +const VSCODE_LM_BUDGET_CHARS_PER_TOKEN = 3 + +/** + * Fraction of the context window the *entire* input (system prompt + tool schemas + conversation) + * is allowed to occupy. The remaining headroom absorbs char/token estimation variance and any + * output/overhead the backend reserves. + */ +const VSCODE_LM_INPUT_BUDGET_FRACTION = 0.8 + +/** A tool_result is never shrunk below this many characters, so a truncated result stays useful. */ +const MIN_TOOL_RESULT_CHARS = 2000 + +function readToolResultText(block: Anthropic.Messages.ContentBlockParam): string | undefined { + if (!block || (block as { type?: string }).type !== "tool_result") { + return undefined + } + const content = (block as Anthropic.Messages.ToolResultBlockParam).content + if (typeof content === "string") { + return content + } + if (Array.isArray(content)) { + return content + .filter((part): part is Anthropic.Messages.TextBlockParam => (part as { type?: string })?.type === "text") + .map((part) => part.text ?? "") + .join("") + } + return undefined +} + +function writeToolResultText(block: Anthropic.Messages.ContentBlockParam, text: string): void { + const toolResult = block as Anthropic.Messages.ToolResultBlockParam + const content = toolResult.content + if (Array.isArray(content)) { + // Preserve any non-text parts (e.g. images) and collapse the text into one truncated part. + const nonText = content.filter((part) => (part as { type?: string })?.type !== "text") + toolResult.content = [{ type: "text", text }, ...nonText] as typeof content + return + } + toolResult.content = text +} + +/** + * Middle-out truncate `text` to at most `maxChars`, keeping the head and tail and replacing the + * middle with a marker noting how many characters were removed. Head/tail are preserved because + * logs and file dumps carry the most signal at their start (structure) and end (recent output). + */ +export function middleOutTruncate(text: string, maxChars: number): string { + if (maxChars <= 0) { + return "" + } + if (text.length <= maxChars) { + return text + } + + const buildMarker = (removed: number) => + `\n\n[... ${removed.toLocaleString("en-US")} characters truncated to fit the model context window ...]\n\n` + + // Reserve room for the marker, sized against the original length so the result never grows. + const reservedMarkerLength = buildMarker(text.length).length + const keep = Math.max(0, maxChars - reservedMarkerLength) + const headLength = Math.ceil(keep / 2) + const tailLength = keep - headLength + let head = text.slice(0, headLength) + // Don't end the head on a lone high surrogate — its low half is in the removed middle, and a lone + // surrogate cannot be encoded as UTF-8 (the backend 400s the whole request). Drop the split half. + if (head.length > 0 && (head.charCodeAt(head.length - 1) & 0xfc00) === 0xd800) { + head = head.slice(0, -1) + } + let tail = tailLength > 0 ? text.slice(text.length - tailLength) : "" + // Likewise, don't start the tail on a lone low surrogate (its high half is in the removed middle). + if (tail.length > 0 && (tail.charCodeAt(0) & 0xfc00) === 0xdc00) { + tail = tail.slice(1) + } + const removed = text.length - head.length - tail.length + return `${head}${buildMarker(removed)}${tail}` +} + +function estimateContentChars(content: Anthropic.Messages.MessageParam["content"]): number { + if (typeof content === "string") { + return content.length + } + if (!Array.isArray(content)) { + return 0 + } + let total = 0 + for (const block of content) { + const type = (block as { type?: string })?.type + if (type === "text") { + total += (block as Anthropic.Messages.TextBlockParam).text?.length ?? 0 + } else if (type === "tool_result") { + total += readToolResultText(block)?.length ?? 0 + } else if (type === "tool_use") { + total += JSON.stringify((block as Anthropic.Messages.ToolUseBlockParam).input ?? {}).length + } else if (type === "image") { + total += 8 // "[IMAGE]" placeholder — VS Code LM drops image data anyway. + } + } + return total +} + +/** + * Shrinks oversized `tool_result` payloads (largest first, middle-out) until the conversation fits + * `budgetChars`. Mutates the tool_result blocks of the supplied messages in place — callers pass a + * cloned array (see `createMessage`) so stored history is never mutated. A no-op when the + * conversation already fits. + */ +export function truncateToolResultsToFitWindow( + messages: Anthropic.Messages.MessageParam[], + budgetChars: number, +): Anthropic.Messages.MessageParam[] { + if (!Number.isFinite(budgetChars) || budgetChars <= 0) { + return messages + } + + let total = messages.reduce((sum, message) => sum + estimateContentChars(message.content), 0) + if (total <= budgetChars) { + return messages + } + + // Collect every truncatable tool_result block, largest first. + const toolResultBlocks: Anthropic.Messages.ContentBlockParam[] = [] + for (const message of messages) { + if (!Array.isArray(message.content)) { + continue + } + for (const block of message.content) { + if (readToolResultText(block) !== undefined) { + toolResultBlocks.push(block) + } + } + } + toolResultBlocks.sort((a, b) => (readToolResultText(b)?.length ?? 0) - (readToolResultText(a)?.length ?? 0)) + + for (const block of toolResultBlocks) { + if (total <= budgetChars) { + break + } + const text = readToolResultText(block) + if (text === undefined || text.length <= MIN_TOOL_RESULT_CHARS) { + continue + } + + const overage = total - budgetChars + const target = Math.max(MIN_TOOL_RESULT_CHARS, text.length - overage) + if (target >= text.length) { + continue + } + + const truncated = middleOutTruncate(text, target) + total -= text.length - truncated.length + writeToolResultText(block, truncated) + } + + return messages +} + export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHandler { protected options: ApiHandlerOptions private client: vscode.LanguageModelChat | null @@ -383,6 +628,19 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan content: this.cleanMessageContent(msg.content), })) + // Keep context-window trimming on OUR side. Copilot's backend trims an over-window request + // without preserving tool_use/tool_result pairing, which orphans a tool_result and triggers a + // 400 ("unexpected tool_use_id"). See truncateToolResultsToFitWindow. + const contextWindowTokens = this.getCondenseContextWindow() + if (Number.isFinite(contextWindowTokens) && contextWindowTokens > 0) { + const toolSchemaChars = metadata?.tools ? JSON.stringify(metadata.tools).length : 0 + const messagesBudgetChars = + contextWindowTokens * VSCODE_LM_INPUT_BUDGET_FRACTION * VSCODE_LM_BUDGET_CHARS_PER_TOKEN - + systemPrompt.length - + toolSchemaChars + truncateToolResultsToFitWindow(cleanedMessages, messagesBudgetChars) + } + // Convert Anthropic messages to VS Code LM messages const vsCodeLmMessages: vscode.LanguageModelChatMessage[] = [ vscode.LanguageModelChatMessage.Assistant(systemPrompt), @@ -398,6 +656,20 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan // Accumulate the text and count at the end of the stream to reduce token counting overhead. let accumulatedText: string = "" + // Leaked tool-call recovery state (see `extractLeakedToolCalls`). Only enabled when we + // actually offered tools this turn, so it can never misfire on plain conversations. + const providedToolNames = new Set( + (metadata?.tools ?? []) + .filter((tool) => tool.type === "function") + .map((tool) => tool.function.name) + .filter((name) => name.length > 0), + ) + const salvageLeakedToolCalls = providedToolNames.size > 0 + let salvageBuffering = false + let salvageBuffer = "" + let salvageCarry = "" + let salvagedToolCallIndex = 0 + try { // Create the response stream with required options const requestOptions: vscode.LanguageModelChatRequestOptions = { @@ -421,9 +693,40 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan } accumulatedText += chunk.value - yield { - type: "text", - text: chunk.value, + + // Fast path: when we didn't offer any tools there is nothing to salvage, so + // stream the text straight through exactly as before. + if (!salvageLeakedToolCalls) { + yield { type: "text", text: chunk.value } + continue + } + + // Once we've seen the start of a leaked tool-call block, buffer the rest of the + // stream so the full markup can be parsed and replayed as a structured call. + if (salvageBuffering) { + salvageBuffer += chunk.value + continue + } + + // Watch for the start of a leaked tool-call block, carrying a small tail across + // chunks so a marker split across chunk boundaries is still detected. + const combined = salvageCarry + chunk.value + const markerMatch = combined.match(LEAKED_TOOL_CALL_START) + if (markerMatch) { + const before = combined.slice(0, markerMatch.index) + if (before) { + yield { type: "text", text: before } + } + salvageBuffering = true + salvageBuffer = combined.slice(markerMatch.index) + salvageCarry = "" + } else { + const carryLength = trailingPartialToolMarkerLength(combined) + const emit = carryLength > 0 ? combined.slice(0, combined.length - carryLength) : combined + salvageCarry = carryLength > 0 ? combined.slice(combined.length - carryLength) : "" + if (emit) { + yield { type: "text", text: emit } + } } } else if (chunk instanceof vscode.LanguageModelToolCallPart) { try { @@ -472,6 +775,37 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan } } + // Flush any leaked tool-call recovery state accumulated during streaming. + if (salvageLeakedToolCalls) { + // A carried tail that never became a marker is just ordinary text. + if (!salvageBuffering && salvageCarry) { + yield { type: "text", text: salvageCarry } + } + + if (salvageBuffering && salvageBuffer) { + const { calls, leftoverText } = extractLeakedToolCalls(salvageBuffer, providedToolNames) + + // Emit surrounding prose first so recovered tool calls come last, matching the + // ordering of a normal native tool-calling turn. + if (leftoverText) { + yield { type: "text", text: leftoverText } + } + + for (const call of calls) { + console.warn( + "Zoo Code : Recovered a tool call the model emitted as text instead of a structured tool call:", + { name: call.name, params: Object.keys(call.input) }, + ) + yield { + type: "tool_call", + id: `vscodelm-salvaged-${Date.now()}-${salvagedToolCallIndex++}`, + name: call.name, + arguments: JSON.stringify(call.input), + } + } + } + } + // Count tokens in the accumulated text after stream completion const totalOutputTokens: number = await this.internalCountTokens(accumulatedText) diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index 3265f2745b..18bac948e3 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -3,7 +3,12 @@ import { Anthropic } from "@anthropic-ai/sdk" import * as vscode from "vscode" -import { convertToVsCodeLmMessages, convertToAnthropicRole, extractTextCountFromMessage } from "../vscode-lm-format" +import { + convertToVsCodeLmMessages, + convertToAnthropicRole, + extractTextCountFromMessage, + sanitizeSurrogates, +} from "../vscode-lm-format" // Mock crypto using Vitest vitest.stubGlobal("crypto", { @@ -325,6 +330,37 @@ describe("convertToVsCodeLmMessages", () => { }) }) +describe("sanitizeSurrogates", () => { + it("leaves plain ASCII unchanged", () => { + expect(sanitizeSurrogates("hello world")).toBe("hello world") + }) + + it("leaves valid surrogate pairs unchanged", () => { + // 😀 U+1F600 and 𐀀 U+10000 are astral-plane code points encoded as surrogate pairs. + expect(sanitizeSurrogates("a\uD83D\uDE00b\uD800\uDC00c")).toBe("a\uD83D\uDE00b\uD800\uDC00c") + }) + + it("replaces a lone high surrogate with U+FFFD", () => { + expect(sanitizeSurrogates("a\uD800b")).toBe("a\uFFFDb") + }) + + it("replaces a lone low surrogate with U+FFFD", () => { + expect(sanitizeSurrogates("a\uDC00b")).toBe("a\uFFFDb") + }) + + it("replaces a trailing lone high surrogate", () => { + expect(sanitizeSurrogates("abc\uD800")).toBe("abc\uFFFD") + }) + + it("replaces a reversed (low-then-high) pair as two lone surrogates", () => { + expect(sanitizeSurrogates("\uDC00\uD800")).toBe("\uFFFD\uFFFD") + }) + + it("returns empty input unchanged", () => { + expect(sanitizeSurrogates("")).toBe("") + }) +}) + describe("convertToAnthropicRole", () => { it("should convert assistant role correctly", () => { const result = convertToAnthropicRole(vscode.LanguageModelChatMessageRole.Assistant) diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index 7ac51e024f..a03cba257c 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -28,6 +28,23 @@ function asObjectSafe(value: unknown): object { } } +/** + * Replaces unpaired UTF-16 surrogate code units with the Unicode replacement character (U+FFFD). + * + * The VS Code LM backend forwards requests to model APIs that require valid UTF-8. A lone surrogate + * — e.g. left behind when some upstream step slices a string through an astral-plane character + * (emoji, CJK extension, etc.) — cannot be encoded as UTF-8, so the backend rejects the entire + * request with a 400 ("string contains an unpaired UTF-16 surrogate code point and cannot be + * encoded as valid UTF-8"). Valid surrogate pairs are matched by the lookahead/lookbehind and left + * untouched. The regex intentionally omits the `u` flag so it operates on UTF-16 code units. + */ +export function sanitizeSurrogates(text: string): string { + if (!text) { + return text + } + return text.replace(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { if (part.type === "image") { if (part.source.type === "base64") { @@ -82,7 +100,7 @@ export function convertToVsCodeLmMessages( ) } if (part.type === "text") { - return new vscode.LanguageModelTextPart(part.text) + return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text)) } return new vscode.LanguageModelTextPart("") }) ?? [new vscode.LanguageModelTextPart("")]) @@ -102,7 +120,7 @@ export function convertToVsCodeLmMessages( `[Image (${part.source.type}): not supported by VSCode LM API]`, ) } - return new vscode.LanguageModelTextPart(part.text) + return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text)) }), ] @@ -135,7 +153,7 @@ export function convertToVsCodeLmMessages( if (part.type === "image") { return new vscode.LanguageModelTextPart("[Image generation not supported by VSCode LM API]") } - return new vscode.LanguageModelTextPart(part.text) + return new vscode.LanguageModelTextPart(sanitizeSurrogates(part.text)) }), // Convert tool messages to ToolCallParts after text From 306976d4d4470f3daf786ea3bd32635107a2c60e Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Fri, 7 Aug 2026 18:04:13 -0700 Subject: [PATCH 2/9] test(vscode-lm): cover leaked tool-call salvage and tool_result truncation paths Raises patch coverage on the new vscode-lm reliability code above the 80%% codecov/patch gate by exercising the streaming salvage state machine (marker split across chunks, multi-chunk buffering, unknown-tool passthrough, carried tail) and the tool_result truncation helpers (array-form content, surrogate-safe middle-out, guard clauses). --- src/api/providers/__tests__/vscode-lm.spec.ts | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index 5cd11ddb65..69743f18ee 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -276,6 +276,108 @@ describe("VsCodeLmHandler", () => { }) }) + describe("leaked tool-call recovery during streaming", () => { + const salvageTools = [ + { + type: "function" as const, + function: { + name: "calculator", + description: "A simple calculator", + parameters: { type: "object", properties: { operation: { type: "string" } } }, + }, + }, + ] + + const streamTextParts = (parts: string[]) => { + mockLanguageModelChat.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + for (const part of parts) { + yield new vscode.LanguageModelTextPart(part) + } + return + })(), + text: (async function* () { + yield parts.join("") + return + })(), + }) + } + + const collect = async (parts: string[]) => { + streamTextParts(parts) + const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], { + taskId: "test-task", + tools: salvageTools, + }) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks + } + + it("recovers a tool call the model streamed as raw invoke XML", async () => { + const chunks = await collect([ + "Thinking. ", + 'add', + ]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "Thinking. " }]) + expect(chunks.filter((chunk) => chunk.type === "tool_call")).toEqual([ + { + type: "tool_call", + id: expect.stringContaining("vscodelm-salvaged-"), + name: "calculator", + arguments: JSON.stringify({ operation: "add" }), + }, + ]) + }) + + it("detects a marker split across stream chunks", async () => { + const chunks = await collect([ + "abc sub', + ]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "abc " }]) + expect(chunks.filter((chunk) => chunk.type === "tool_call")).toMatchObject([ + { name: "calculator", arguments: JSON.stringify({ operation: "sub" }) }, + ]) + }) + + it("emits a carried tail as plain text when it never becomes a marker", async () => { + const chunks = await collect(["hello chunk.type === "text")).toEqual([ + { type: "text", text: "hello " }, + { type: "text", text: " chunk.type === "tool_call")).toBe(false) + }) + + it("buffers across chunks that arrive after the marker", async () => { + const chunks = await collect([ + 'prose ', + '', + "mul", + "", + ]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: "prose " }]) + expect(chunks.filter((chunk) => chunk.type === "tool_call")).toMatchObject([ + { name: "calculator", arguments: JSON.stringify({ operation: "mul" }) }, + ]) + }) + + it("keeps an invoke block for an unknown tool as literal text", async () => { + const block = '1' + const chunks = await collect([block]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: block }]) + expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) + }) + }) + it("should handle native tool calls when tools are provided", async () => { const systemPrompt = "You are a helpful assistant" const messages: Anthropic.Messages.MessageParam[] = [ @@ -1166,6 +1268,15 @@ describe("context-window tool_result truncation", () => { it("returns an empty string for a non-positive limit", () => { expect(middleOutTruncate("anything", 0)).toBe("") }) + + // A lone surrogate cannot be encoded as UTF-8 and 400s the whole request. + it("never splits a surrogate pair across the removed middle", () => { + const pair = "\u{1F600}" // one astral char = high + low surrogate + const text = pair.repeat(400) + const result = middleOutTruncate(text, 200) + + expect(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { @@ -1202,6 +1313,111 @@ describe("context-window tool_result truncation", () => { expect(messages).toEqual(before) }) + it("returns messages untouched when the budget is not a usable number", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "Y".repeat(50_000)), + ] + const before = JSON.parse(JSON.stringify(messages)) + + expect(truncateToolResultsToFitWindow(messages, 0)).toBe(messages) + expect(truncateToolResultsToFitWindow(messages, Number.NaN)).toBe(messages) + expect(messages).toEqual(before) + }) + + it("truncates array-form tool_result content and preserves non-text parts", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "t1", + content: [ + { type: "text", text: "Z".repeat(50_000) }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "abc" } }, + ], + }, + ], + } as unknown as Anthropic.Messages.MessageParam, + ] + + truncateToolResultsToFitWindow(messages, 10_000) + + const toolResult = findBlock(messages[1], "tool_result") + const parts = toolResult.content as Array<{ type: string; text?: string }> + expect(parts[0].type).toBe("text") + expect(parts[0].text).toContain("characters truncated") + expect(parts.some((part) => part.type === "image")).toBe(true) + }) + + it("ignores string content and skips messages that cannot hold tool_result blocks", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + { role: "user", content: "a plain string turn" }, + toolUseMessage("t1"), + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "t1", content: "W".repeat(50_000) }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "abc" } }, + ], + } as unknown as Anthropic.Messages.MessageParam, + ] + + truncateToolResultsToFitWindow(messages, 10_000) + + expect(messages[0].content).toBe("a plain string turn") + expect(String(findBlock(messages[2], "tool_result").content)).toContain("characters truncated") + }) + + it("ignores a tool_result whose content is neither string nor array", () => { + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "V".repeat(50_000)), + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "t2", content: undefined }], + } as unknown as Anthropic.Messages.MessageParam, + ] + + truncateToolResultsToFitWindow(messages, 10_000) + + expect(findBlock(messages[2], "tool_result").content).toBeUndefined() + expect(String(findBlock(messages[1], "tool_result").content)).toContain("characters truncated") + }) + + it("skips a tool_result already small enough to need no trimming", () => { + // Overage is tiny, so the largest block's target lands at its current length. + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", "U".repeat(3000)), + toolUseMessage("t2"), + toolResultMessage("t2", "T".repeat(2500)), + ] + + truncateToolResultsToFitWindow(messages, 5600) + + const first = String(findBlock(messages[1], "tool_result").content) + const second = String(findBlock(messages[3], "tool_result").content) + expect(first.length + second.length).toBeLessThanOrEqual(5600) + }) + + it("leaves a tool_result at or below the minimum size alone", () => { + const shortResult = "S".repeat(1500) + const messages: Anthropic.Messages.MessageParam[] = [ + toolUseMessage("t1"), + toolResultMessage("t1", shortResult), + toolUseMessage("t2"), + toolResultMessage("t2", shortResult), + ] + + truncateToolResultsToFitWindow(messages, 100) + + expect(findBlock(messages[1], "tool_result").content).toBe(shortResult) + expect(findBlock(messages[3], "tool_result").content).toBe(shortResult) + }) + it("shrinks an oversized tool_result so the conversation fits the budget", () => { const messages: Anthropic.Messages.MessageParam[] = [ toolUseMessage("t1"), From ed3e8ecf877d72bbafae02eb6585a02c7583c135 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 8 Aug 2026 12:28:20 -0700 Subject: [PATCH 3/9] fix(vscode-lm): guard leaked tool-call recovery against quoted markup Address review feedback on the leaked-tool-call salvage path: a tool name alone was not a sufficient gate, so prose or fenced examples reproducing the invoke markup could be replayed as real calls. Adds the quoted/fenced guard plus coverage. Also records the empirical vscode.lm probe as a project skill (probe-vscode-lm-api) with the scratch probe extension, the false-positive replay harness, representative transcripts, and the consent-gate gotcha. --- .roo/skills/probe-vscode-lm-api/SKILL.md | 99 + .../probe-vscode-lm-api/scripts/extension.js | 206 + .../probe-vscode-lm-api/scripts/package.json | 21 + .../scripts/probe-false-positives.spec.ts | 24 + ...ools_declared_compelling_prompt__run1.json | 43 + ...tools_declared_compelling_prompt__run1.txt | 0 ...D_no_tools_asked_to_emit_markup__run2.json | 66 + ..._D_no_tools_asked_to_emit_markup__run2.txt | 5 + ...p_in_prose_false_positive_check__run1.json | 70 + ...up_in_prose_false_positive_check__run1.txt | 17 + ...ted_markup_in_fenced_code_block__run1.json | 38 + ...oted_markup_in_fenced_code_block__run1.txt | 7 + ...p_in_prose_false_positive_check__run1.json | 58 + ...up_in_prose_false_positive_check__run1.txt | 18 + ...D_no_tools_asked_to_emit_markup__run1.json | 34 + ..._D_no_tools_asked_to_emit_markup__run1.txt | 5 + ...ted_markup_in_fenced_code_block__run1.json | 34 + ...oted_markup_in_fenced_code_block__run1.txt | 5 + .../transcripts/false-positive-report.txt | 58 + .../transcripts/summary.json | 3548 +++++++++++++++++ src/api/providers/__tests__/vscode-lm.spec.ts | 104 + src/api/providers/vscode-lm.ts | 131 +- .../__tests__/vscode-lm-format.spec.ts | 42 + 23 files changed, 4593 insertions(+), 40 deletions(-) create mode 100644 .roo/skills/probe-vscode-lm-api/SKILL.md create mode 100644 .roo/skills/probe-vscode-lm-api/scripts/extension.js create mode 100644 .roo/skills/probe-vscode-lm-api/scripts/package.json create mode 100644 .roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt create mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/summary.json diff --git a/.roo/skills/probe-vscode-lm-api/SKILL.md b/.roo/skills/probe-vscode-lm-api/SKILL.md new file mode 100644 index 0000000000..95d365ebcf --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/SKILL.md @@ -0,0 +1,99 @@ +--- +name: probe-vscode-lm-api +description: How to empirically probe the VS Code Language Model API (`vscode.lm`) with a scratch extension against a real extension host, and the measured findings about Copilot Claude models leaking tool-call markup into text. Use when asked to "test the vscode.lm API", "probe Copilot model behavior", "capture a raw LM transcript", "does the model leak tool-call markup", "verify LanguageModelToolCallPart behavior", or when reasoning about `extractLeakedToolCalls()` in the vscode-lm provider. +--- + +# Probing the VS Code LM API Empirically + +## When to Use This Skill + +- A claim is being made about what a Copilot-backed model _actually_ emits over `vscode.lm` (tool-call parts vs. text), and it needs evidence rather than inference. +- Changing [`extractLeakedToolCalls()`](src/api/providers/vscode-lm.ts) or its guards, and you need a false-positive corpus. +- Any question that can only be answered by real `model.sendRequest()` traffic — the mocked unit tests cannot answer it. + +## When NOT to Use This Skill + +- Ordinary provider work covered by [`src/api/providers/__tests__/vscode-lm.spec.ts`](src/api/providers/__tests__/vscode-lm.spec.ts). Live probing is slow and burns Copilot quota. +- Anything about non-`vscode-lm` providers. Anthropic-API behavior does not transfer. + +## Running the Probe + +Scripts live in [`scripts/`](.roo/skills/probe-vscode-lm-api/scripts/) next to this file. + +1. Copy `scripts/package.json` and `scripts/extension.js` into a scratch directory, e.g. `\.tmp\lmprobe\`. No build, no `npm install` — it is plain CommonJS against the `vscode` module. +2. Adjust `OUT_DIR` at the top of `extension.js` to the transcript output directory. +3. Launch a **new** extension host window: + +``` +code --extensionDevelopmentPath=\.tmp\lmprobe --new-window +``` + +4. In that new window: `Ctrl+Shift+P` -> **LM Probe: Run** (or click "Run probe" on the toast). +5. Wait for the completion notification. Transcripts and `summary.json` land in `OUT_DIR`. + +Each run writes a `.json` of every stream part and a `.txt` of the exact concatenated text, named `____run`. + +### Delegate the UI driving + +Steps 3-5 involve a live window. Delegate them to **`ui-operator`** mode rather than doing them inline; screenshots and control trees consume large amounts of context. + +## Gotchas + +### The consent gate needs a real user gesture + +**Do not call `model.sendRequest()` from `activate()`.** Every request fails with: + +``` +Language model '' cannot be used by '.' +``` + +This is not a quota, auth, or manifest problem — `vscode.lm` grants consent only in response to a genuine user gesture. The probe must therefore be triggered from the Command Palette (or a notification button click). This is the single most expensive trap here; it silently fails 100% of requests and looks like an entitlement bug. + +### Never `Stop-Process` filtered on window title + +**WARNING:** During this experiment, killing processes matched by window title destroyed the user's unrelated VS Code windows and their unsaved work. + +Safe alternative: only ever _launch_ new windows with `--new-window`, and close the probe window by hand. Never bulk-terminate `Code.exe` by title, `MainWindowTitle`, or any other fuzzy match. + +### Run tests with pnpm, not npx + +``` +pnpm --dir src exec vitest run +``` + +Never `npx vitest` — it resolves a wrong hoisted 3.2.4 instead of the pinned 4.1.9 and produces phantom failures. + +## Measured Findings + +Sample: 210 live requests, 7 Copilot Claude models x 6 scenarios x 5 repeats, 0 errors. Raw evidence in [`transcripts/`](.roo/skills/probe-vscode-lm-api/transcripts/). + +| Scenario | Setup | Runs | ``; 0 were bare. All 44 quoted-in-prose cases (E+F) were bare; 0 were wrapped. In this sample, _bare correlates with quoting and wrapped with genuine invocation_ — so requiring a `` wrapper would not have been the discriminator it appears to be. +- **No `antml:` prefix appeared** in any of the 210 runs. +- **Zero false positives.** Replaying `extractLeakedToolCalls()` over all 58 transcripts containing `\\.tmp\\transcripts" + +function write(name, data) { + fs.mkdirSync(OUT_DIR, { recursive: true }) + fs.writeFileSync(path.join(OUT_DIR, name), typeof data === "string" ? data : JSON.stringify(data, null, 2)) +} + +const READ_TOOL = { + name: "read_file", + description: "Read the contents of a file at the given path.", + inputSchema: { + type: "object", + properties: { path: { type: "string", description: "File path to read" } }, + required: ["path"], + }, +} + +const TOOL_SYSTEM_PROMPT = [ + "You are Zoo Code, an autonomous coding agent.", + "You accomplish tasks by calling the tools provided to you.", + "You MUST call exactly one tool per message. Never ask the user a question.", + "Do not answer from memory; always read the file first using the read_file tool.", +].join("\n") + +async function runOnce(model, scenario) { + const record = { + scenario: scenario.name, + modelId: model.id, + modelFamily: model.family, + modelVendor: model.vendor, + modelVersion: model.version, + maxInputTokens: model.maxInputTokens, + parts: [], + concatenatedText: "", + toolCallParts: [], + error: null, + } + const messages = [] + if (scenario.system) { + messages.push(vscode.LanguageModelChatMessage.Assistant(scenario.system)) + } + for (const userText of scenario.userMessages) { + messages.push(vscode.LanguageModelChatMessage.User(userText)) + } + const options = { justification: "Empirical probe of leaked tool-call formatting." } + if (scenario.tools) { + options.tools = [READ_TOOL] + } + try { + const source = new vscode.CancellationTokenSource() + const response = await model.sendRequest(messages, options, source.token) + for await (const chunk of response.stream) { + const typeName = chunk && chunk.constructor ? chunk.constructor.name : typeof chunk + if (chunk instanceof vscode.LanguageModelTextPart) { + record.parts.push({ type: typeName, value: chunk.value }) + record.concatenatedText += chunk.value + } else if (chunk instanceof vscode.LanguageModelToolCallPart) { + const call = { type: typeName, name: chunk.name, callId: chunk.callId, input: chunk.input } + record.parts.push(call) + record.toolCallParts.push(call) + } else { + record.parts.push({ type: typeName, raw: String(chunk) }) + } + } + } catch (error) { + record.error = { name: error && error.name, message: error && error.message, stack: error && error.stack } + } + + const text = record.concatenatedText + record.markers = { + hasInvoke: /<(?:antml:)?invoke\b/i.test(text), + hasFunctionCalls: /<(?:antml:)?function_calls\b/i.test(text), + hasAntmlNamespace: /antml:/i.test(text), + invokeWrappedInFunctionCalls: /<(?:antml:)?function_calls\s*>[\s\S]*?<(?:antml:)?invoke\b/i.test(text), + bareInvokeWithoutWrapper: /<(?:antml:)?invoke\b/i.test(text) && !/<(?:antml:)?function_calls\b/i.test(text), + insideFencedCodeBlock: /```[\s\S]*?<(?:antml:)?invoke\b/i.test(text), + } + return record +} + +function buildScenarios() { + const longFiller = "This is filler context line used to grow the prompt toward the context window. ".repeat(4000) + return [ + { + name: "A_tools_declared_compelling_prompt", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: ["Read the file c:/git/Zoo-Code/package.json and tell me the version field."], + }, + { + name: "B_tools_declared_no_system_prompt", + tools: true, + userMessages: ["Read the file c:/git/Zoo-Code/package.json and tell me the version field."], + }, + { + name: "C_tools_declared_long_context", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: [longFiller, "Now read the file c:/git/Zoo-Code/package.json and tell me the version field."], + }, + { + name: "D_no_tools_asked_to_emit_markup", + tools: false, + system: TOOL_SYSTEM_PROMPT, + userMessages: [ + "You have a tool named read_file that takes a path. Emit the tool invocation using Anthropic's internal function-call XML format, exactly as you would internally, to read c:/git/Zoo-Code/package.json. Output only the markup.", + ], + }, + { + name: "E_quoted_markup_in_prose_false_positive_check", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: [ + 'Do NOT call any tool. Instead, explain in prose what an invoke block looks like, and include a literal example using the tag name "invoke" with a name attribute of read_file and a parameter named path, written as plain text in your answer.', + ], + }, + { + name: "F_quoted_markup_in_fenced_code_block", + tools: true, + system: TOOL_SYSTEM_PROMPT, + userMessages: [ + "Do NOT call any tool. Show me, inside a fenced markdown code block, an example of an invoke block naming the tool read_file with a parameter named path set to /etc/passwd. Only output the fenced code block.", + ], + }, + ] +} + +async function run() { + const summary = { startedAt: new Date().toISOString(), vscodeVersion: vscode.version, models: [], runs: [] } + let models = [] + let selectError = null + try { + models = (await vscode.lm.selectChatModels({ vendor: "copilot" })) || [] + } catch (error) { + selectError = { name: error && error.name, message: error && error.message } + } + summary.selectError = selectError + summary.allModels = models.map((model) => ({ + id: model.id, + family: model.family, + vendor: model.vendor, + version: model.version, + maxInputTokens: model.maxInputTokens, + })) + + const claudeModels = models.filter((model) => /claude/i.test(model.id) || /claude/i.test(model.family)) + summary.claudeModelIds = claudeModels.map((model) => model.id) + + if (claudeModels.length === 0) { + write("summary.json", summary) + vscode.window.showErrorMessage( + `LM Probe: no Claude models. selectChatModels returned ${models.length}. ${selectError ? selectError.message : ""}`, + ) + return + } + + const scenarios = buildScenarios() + const REPEATS = 5 + for (const model of claudeModels) { + for (const scenario of scenarios) { + for (let iter = 1; iter <= REPEATS; iter++) { + const record = await runOnce(model, scenario) + record.iteration = iter + summary.runs.push({ + scenario: scenario.name, + modelId: model.id, + iteration: iter, + markers: record.markers, + toolCallPartCount: record.toolCallParts.length, + textLength: record.concatenatedText.length, + error: record.error ? record.error.message : null, + }) + write(`${model.id}__${scenario.name}__run${iter}.json`, record) + write(`${model.id}__${scenario.name}__run${iter}.txt`, record.concatenatedText) + } + } + } + + summary.finishedAt = new Date().toISOString() + write("summary.json", summary) + vscode.window.showInformationMessage(`LM Probe complete: ${summary.runs.length} runs written to ${OUT_DIR}`) +} + +function activate(context) { + // MUST be user-initiated: vscode.lm consent is only granted from a real user gesture, so an + // activation-time sendRequest is auto-denied with "cannot be used by 'scratch.lmprobe'". + context.subscriptions.push( + vscode.commands.registerCommand("lmprobe.run", () => + run().catch((error) => { + write("fatal.json", { message: String(error && error.message), stack: String(error && error.stack) }) + }), + ), + ) + vscode.window.showInformationMessage("LM Probe ready", "Run probe").then((choice) => { + if (choice === "Run probe") { + vscode.commands.executeCommand("lmprobe.run") + } + }) +} + +module.exports = { activate, deactivate() {} } diff --git a/.roo/skills/probe-vscode-lm-api/scripts/package.json b/.roo/skills/probe-vscode-lm-api/scripts/package.json new file mode 100644 index 0000000000..8ad0174674 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/scripts/package.json @@ -0,0 +1,21 @@ +{ + "name": "lmprobe", + "displayName": "LM Leak Probe", + "version": "0.0.1", + "publisher": "scratch", + "engines": { + "vscode": "^1.95.0" + }, + "activationEvents": [ + "onStartupFinished" + ], + "main": "./extension.js", + "contributes": { + "commands": [ + { + "command": "lmprobe.run", + "title": "LM Probe: Run" + } + ] + } +} diff --git a/.roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts b/.roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts new file mode 100644 index 0000000000..2d384c927f --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts @@ -0,0 +1,24 @@ +import fs from "fs" +import path from "path" + +import { extractLeakedToolCalls } from "../vscode-lm" + +// Point this at the probe's OUT_DIR. Scratch harness: not a committed test. +const TRANSCRIPTS = process.env.LM_PROBE_TRANSCRIPTS ?? path.resolve(__dirname, "../../../../.tmp/transcripts") + +describe("probe: false-positive check against real transcripts", () => { + it("reports which transcripts extractLeakedToolCalls would treat as real calls", () => { + const names = new Set(["read_file"]) + const report: string[] = [] + for (const file of fs.readdirSync(TRANSCRIPTS).filter((name) => name.endsWith(".txt"))) { + const text = fs.readFileSync(path.join(TRANSCRIPTS, file), "utf8") + if (!/<(?:antml:)?invoke/i.test(text)) { + continue + } + const { calls } = extractLeakedToolCalls(text, names) + report.push(`${calls.length > 0 ? "RECOVERED" : "passthrough"}\t${file}\t${JSON.stringify(calls)}`) + } + fs.writeFileSync(path.join(TRANSCRIPTS, "..", "false-positive-report.txt"), report.join("\n")) + console.log(report.join("\n")) + }) +}) diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.json new file mode 100644 index 0000000000..5317e65c41 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.json @@ -0,0 +1,43 @@ +{ + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.6", + "modelFamily": "claude-opus-4.6", + "modelVendor": "copilot", + "modelVersion": "claude-opus-4.6", + "maxInputTokens": 935793, + "parts": [ + { + "type": "mi", + "name": "read_file", + "callId": "toolu_bdrk_01GJ8FjwnLsUcJ2Qkr9VkLtM", + "input": { + "path": "c:/git/Zoo-Code/package.json" + } + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "", + "toolCallParts": [ + { + "type": "mi", + "name": "read_file", + "callId": "toolu_bdrk_01GJ8FjwnLsUcJ2Qkr9VkLtM", + "input": { + "path": "c:/git/Zoo-Code/package.json" + } + } + ], + "error": null, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "iteration": 1 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.json new file mode 100644 index 0000000000..f5d333da9d --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.json @@ -0,0 +1,66 @@ +{ + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.6", + "modelFamily": "claude-opus-4.6", + "modelVendor": "copilot", + "modelVersion": "claude-opus-4.6", + "maxInputTokens": 935793, + "parts": [ + { + "type": "rn", + "value": "" + }, + { + "type": "rn", + "value": "\n" + }, + { + "type": "rn", + "value": "\n" + }, + { + "type": "rn", + "value": "c:/git/Zoo-Code/" + }, + { + "type": "rn", + "value": "package.json\n" + }, + { + "type": "rn", + "value": "\n" + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "\n\nc:/git/Zoo-Code/package.json\n\n", + "toolCallParts": [], + "error": null, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "iteration": 2 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt new file mode 100644 index 0000000000..80c159b8fd --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt @@ -0,0 +1,5 @@ + + +c:/git/Zoo-Code/package.json + + \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json new file mode 100644 index 0000000000..78af16167b --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json @@ -0,0 +1,70 @@ +{ + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.8", + "modelFamily": "claude-opus-4.8", + "modelVendor": "copilot", + "modelVersion": "claude-opus-4.8", + "maxInputTokens": 935793, + "parts": [ + { + "type": "rn", + "value": "An" + }, + { + "type": "rn", + "value": " inv" + }, + { + "type": "rn", + "value": "oke block is" + }, + { + "type": "rn", + "value": " the" + }, + { + "type": "rn", + "value": " m" + }, + { + "type": "rn", + "value": "echanism used to call a tool. It's written as an XML-like structure containing a function invocation with named parameters." + }, + { + "type": "rn", + "value": " Here's a literal example, written as plain text:\n\n```xml\n\nsrc/example.txt\n\n```\n\nThe structure works like this:\n\n- An outer `invoke` tag carries a `name` attribute identifying which tool to call (here" + }, + { + "type": "rn", + "value": ", `read_file`).\n- Inside it, one or more `parameter` tags each carry a `name` attribute identifying the argument (here, `path`).\n- The value of each parameter goes between the opening and closing `parameter" + }, + { + "type": "rn", + "value": "` tags (here, `src/example.txt`).\n\nFor string and scalar parameters, the value is written as-is between the tags. For lists or objects, the value would be formatted as JSON instead.\n\nNote that I've shown this as plain," + }, + { + "type": "rn", + "value": " inert text for illustration only — I have not actually called any tool, per your request." + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "An invoke block is the mechanism used to call a tool. It's written as an XML-like structure containing a function invocation with named parameters. Here's a literal example, written as plain text:\n\n```xml\n\nsrc/example.txt\n\n```\n\nThe structure works like this:\n\n- An outer `invoke` tag carries a `name` attribute identifying which tool to call (here, `read_file`).\n- Inside it, one or more `parameter` tags each carry a `name` attribute identifying the argument (here, `path`).\n- The value of each parameter goes between the opening and closing `parameter` tags (here, `src/example.txt`).\n\nFor string and scalar parameters, the value is written as-is between the tags. For lists or objects, the value would be formatted as JSON instead.\n\nNote that I've shown this as plain, inert text for illustration only — I have not actually called any tool, per your request.", + "toolCallParts": [], + "error": null, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "iteration": 1 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt new file mode 100644 index 0000000000..8ae85fc055 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt @@ -0,0 +1,17 @@ +An invoke block is the mechanism used to call a tool. It's written as an XML-like structure containing a function invocation with named parameters. Here's a literal example, written as plain text: + +```xml + +src/example.txt + +``` + +The structure works like this: + +- An outer `invoke` tag carries a `name` attribute identifying which tool to call (here, `read_file`). +- Inside it, one or more `parameter` tags each carry a `name` attribute identifying the argument (here, `path`). +- The value of each parameter goes between the opening and closing `parameter` tags (here, `src/example.txt`). + +For string and scalar parameters, the value is written as-is between the tags. For lists or objects, the value would be formatted as JSON instead. + +Note that I've shown this as plain, inert text for illustration only — I have not actually called any tool, per your request. \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json new file mode 100644 index 0000000000..ddb348de43 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json @@ -0,0 +1,38 @@ +{ + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.8", + "modelFamily": "claude-opus-4.8", + "modelVendor": "copilot", + "modelVersion": "claude-opus-4.8", + "maxInputTokens": 935793, + "parts": [ + { + "type": "rn", + "value": "```" + }, + { + "type": "rn", + "value": "xml\n\n/etc/passwd\n\n```\n\nNote: this is just a formatting" + }, + { + "type": "rn", + "value": " example inside a code block, not an actual tool call." + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "```xml\n\n/etc/passwd\n\n```\n\nNote: this is just a formatting example inside a code block, not an actual tool call.", + "toolCallParts": [], + "error": null, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "iteration": 1 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt new file mode 100644 index 0000000000..1dbb76b8a2 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt @@ -0,0 +1,7 @@ +```xml + +/etc/passwd + +``` + +Note: this is just a formatting example inside a code block, not an actual tool call. \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json new file mode 100644 index 0000000000..3a29ccdfae --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json @@ -0,0 +1,58 @@ +{ + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-5", + "modelFamily": "claude-opus-5", + "modelVendor": "copilot", + "modelVersion": "claude-opus-5", + "maxInputTokens": 935793, + "parts": [ + { + "type": "rn", + "value": "A tool call in this system is expressed as an XML-like block nested inside a function-calls container. The block has:\n\n- an opening tag named `invoke`" + }, + { + "type": "rn", + "value": " with a `name` attribute whose value is the tool's identifier (for example, `read_file`);\n- one or more `parameter` child elements, each with a `name` attribute matching a parameter def" + }, + { + "type": "rn", + "value": "ined in the tool's schema (for example, `path`);\n- the argument value written as the text content of the `parameter` element;\n- closing tags for each" + }, + { + "type": "rn", + "value": " `parameter` and for the `invoke` element.\n\nScalar values (strings, numbers, booleans) go in as plain text. Lists and objects are written as JSON inside the parameter body" + }, + { + "type": "rn", + "value": ". The whole thing is emitted instead of, or alongside, prose in the assistant's message, and the runtime int" + }, + { + "type": "rn", + "value": "ercepts it, executes the named tool, and feeds the result back.\n\nHere is a literal, illustrative example written as plain text (this is documentation, not an actual call):\n\n```xml" + }, + { + "type": "rn", + "value": "\n\n src/app/main.py\n\n```\n\nNote that in the real wire format the tags carry a namespace prefix and sit inside a surrounding function-calls wrapper;" + }, + { + "type": "rn", + "value": " the snippet above is stripped down to just the `invoke`/`parameter` structure you asked about, so it is inert." + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "A tool call in this system is expressed as an XML-like block nested inside a function-calls container. The block has:\n\n- an opening tag named `invoke` with a `name` attribute whose value is the tool's identifier (for example, `read_file`);\n- one or more `parameter` child elements, each with a `name` attribute matching a parameter defined in the tool's schema (for example, `path`);\n- the argument value written as the text content of the `parameter` element;\n- closing tags for each `parameter` and for the `invoke` element.\n\nScalar values (strings, numbers, booleans) go in as plain text. Lists and objects are written as JSON inside the parameter body. The whole thing is emitted instead of, or alongside, prose in the assistant's message, and the runtime intercepts it, executes the named tool, and feeds the result back.\n\nHere is a literal, illustrative example written as plain text (this is documentation, not an actual call):\n\n```xml\n\n src/app/main.py\n\n```\n\nNote that in the real wire format the tags carry a namespace prefix and sit inside a surrounding function-calls wrapper; the snippet above is stripped down to just the `invoke`/`parameter` structure you asked about, so it is inert.", + "toolCallParts": [], + "error": null, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "iteration": 1 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt new file mode 100644 index 0000000000..62937045f9 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt @@ -0,0 +1,18 @@ +A tool call in this system is expressed as an XML-like block nested inside a function-calls container. The block has: + +- an opening tag named `invoke` with a `name` attribute whose value is the tool's identifier (for example, `read_file`); +- one or more `parameter` child elements, each with a `name` attribute matching a parameter defined in the tool's schema (for example, `path`); +- the argument value written as the text content of the `parameter` element; +- closing tags for each `parameter` and for the `invoke` element. + +Scalar values (strings, numbers, booleans) go in as plain text. Lists and objects are written as JSON inside the parameter body. The whole thing is emitted instead of, or alongside, prose in the assistant's message, and the runtime intercepts it, executes the named tool, and feeds the result back. + +Here is a literal, illustrative example written as plain text (this is documentation, not an actual call): + +```xml + + src/app/main.py + +``` + +Note that in the real wire format the tags carry a namespace prefix and sit inside a surrounding function-calls wrapper; the snippet above is stripped down to just the `invoke`/`parameter` structure you asked about, so it is inert. \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json new file mode 100644 index 0000000000..7fb29b7391 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json @@ -0,0 +1,34 @@ +{ + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-4.6", + "modelFamily": "claude-sonnet-4.6", + "modelVendor": "copilot", + "modelVersion": "claude-sonnet-4.6", + "maxInputTokens": 935793, + "parts": [ + { + "type": "rn", + "value": "\n\nc:/git/Zoo-Code/package.json\n\n" + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "\n\nc:/git/Zoo-Code/package.json\n\n", + "toolCallParts": [], + "error": null, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "iteration": 1 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt new file mode 100644 index 0000000000..80c159b8fd --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt @@ -0,0 +1,5 @@ + + +c:/git/Zoo-Code/package.json + + \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json new file mode 100644 index 0000000000..153710977f --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json @@ -0,0 +1,34 @@ +{ + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-4.6", + "modelFamily": "claude-sonnet-4.6", + "modelVendor": "copilot", + "modelVersion": "claude-sonnet-4.6", + "maxInputTokens": 935793, + "parts": [ + { + "type": "rn", + "value": "```" + }, + { + "type": "rn", + "value": "xml\n\n/etc/passwd\n\n```" + }, + { + "type": "i", + "raw": "[object Object]" + } + ], + "concatenatedText": "```xml\n\n/etc/passwd\n\n```", + "toolCallParts": [], + "error": null, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "iteration": 1 +} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt new file mode 100644 index 0000000000..78d575b390 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt @@ -0,0 +1,5 @@ +```xml + +/etc/passwd + +``` \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt b/.roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt new file mode 100644 index 0000000000..782d82cee9 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt @@ -0,0 +1,58 @@ +passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run1.txt [] +passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run2.txt [] +passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run3.txt [] +passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run4.txt [] +passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run5.txt [] +passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run1.txt [] +passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run2.txt [] +passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run3.txt [] +passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run4.txt [] +passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run5.txt [] +passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run1.txt [] +passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run2.txt [] +passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run3.txt [] +passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run4.txt [] +passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run5.txt [] +RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run3.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run4.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run5.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +passthrough claude-opus-4.6__E_quoted_markup_in_prose_false_positive_check__run1.txt [] +passthrough claude-opus-4.6__E_quoted_markup_in_prose_false_positive_check__run2.txt [] +passthrough claude-opus-4.6__E_quoted_markup_in_prose_false_positive_check__run5.txt [] +passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt [] +passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run2.txt [] +passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run3.txt [] +passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run4.txt [] +passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run5.txt [] +passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt [] +passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run2.txt [] +passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run3.txt [] +passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run4.txt [] +passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run5.txt [] +passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt [] +passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run2.txt [] +passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run3.txt [] +passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run4.txt [] +passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run5.txt [] +passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run1.txt [] +passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run2.txt [] +passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run3.txt [] +passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run4.txt [] +passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run5.txt [] +RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run2.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run3.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run4.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run5.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] +passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt [] +passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run2.txt [] +passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run3.txt [] +passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run4.txt [] +passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run5.txt [] +passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run1.txt [] +passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run2.txt [] +passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run3.txt [] +passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run4.txt [] +passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run5.txt [] +passthrough claude-sonnet-5__F_quoted_markup_in_fenced_code_block__run5.txt [] \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/summary.json b/.roo/skills/probe-vscode-lm-api/transcripts/summary.json new file mode 100644 index 0000000000..7d42ad0799 --- /dev/null +++ b/.roo/skills/probe-vscode-lm-api/transcripts/summary.json @@ -0,0 +1,3548 @@ +{ + "startedAt": "2026-08-08T18:14:17.466Z", + "vscodeVersion": "1.128.1", + "models": [], + "runs": [ + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 114, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.6", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.6", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.6", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.6", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.6", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 459, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.6", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 451, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 51, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 51, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.6", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 474, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 7, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 7, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 7, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 7, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 7, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.7", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.7", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.7", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.7", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.7", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.7", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.7", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.7", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.7", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.7", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.7", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.7", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.7", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.7", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.7", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.7", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 166, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.7", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 32, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.7", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 32, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.7", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 32, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.7", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 32, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.7", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 87, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.7", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 57, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.7", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 57, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.7", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 85, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.7", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 54, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.7", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 8, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.7", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 8, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.7", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 8, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.7", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 8, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.7", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 8, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.8", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.8", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 46, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.8", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 46, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.8", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 46, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-4.8", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 46, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.8", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 46, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.8", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 29, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.8", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.8", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 29, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-4.8", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 28, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.8", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.8", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 45, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.8", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 46, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.8", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-4.8", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.8", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 494, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.8", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 501, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.8", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 461, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.8", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 409, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-4.8", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 356, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.8", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 930, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.8", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1397, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.8", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1284, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.8", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1042, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-4.8", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1099, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.8", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 180, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.8", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 237, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.8", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 211, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.8", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 183, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-4.8", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 218, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 25, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-opus-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 29, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 29, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 29, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 29, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-opus-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 47, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-opus-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 20, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 461, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 1675, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 308, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 1138, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-opus-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 585, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-5", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1268, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-5", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1831, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-5", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1008, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-5", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1573, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-opus-5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1705, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-5", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-5", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-5", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-5", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-opus-5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-4.6", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-4.6", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-4.6", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-4.6", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-4.6", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 134, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-4.6", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 370, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-4.6", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 420, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-4.6", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 267, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-4.6", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 361, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-4.6", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 323, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-4.6", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-4.6", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-4.6", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-4.6", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-4.6", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-sonnet-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-sonnet-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-sonnet-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 103, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 61, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 66, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 66, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-sonnet-5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 34, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-5", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1126, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-5", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 838, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-5", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1149, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-5", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1045, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-sonnet-5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 1050, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 16, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 0, + "textLength": 21, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 110, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-sonnet-5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 90, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "A_tools_declared_compelling_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "B_tools_declared_no_system_prompt", + "modelId": "claude-haiku-4.5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 0, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-haiku-4.5", + "iteration": 1, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 59, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-haiku-4.5", + "iteration": 2, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 59, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-haiku-4.5", + "iteration": 3, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 41, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-haiku-4.5", + "iteration": 4, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 230, + "error": null + }, + { + "scenario": "C_tools_declared_long_context", + "modelId": "claude-haiku-4.5", + "iteration": 5, + "markers": { + "hasInvoke": false, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": false + }, + "toolCallPartCount": 1, + "textLength": 59, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-haiku-4.5", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 145, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-haiku-4.5", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 145, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-haiku-4.5", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 145, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-haiku-4.5", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 145, + "error": null + }, + { + "scenario": "D_no_tools_asked_to_emit_markup", + "modelId": "claude-haiku-4.5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": true, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": true, + "bareInvokeWithoutWrapper": false, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 145, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-haiku-4.5", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 609, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-haiku-4.5", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 602, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-haiku-4.5", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 915, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-haiku-4.5", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 680, + "error": null + }, + { + "scenario": "E_quoted_markup_in_prose_false_positive_check", + "modelId": "claude-haiku-4.5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 701, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-haiku-4.5", + "iteration": 1, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-haiku-4.5", + "iteration": 2, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-haiku-4.5", + "iteration": 3, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-haiku-4.5", + "iteration": 4, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + }, + { + "scenario": "F_quoted_markup_in_fenced_code_block", + "modelId": "claude-haiku-4.5", + "iteration": 5, + "markers": { + "hasInvoke": true, + "hasFunctionCalls": false, + "hasAntmlNamespace": false, + "invokeWrappedInFunctionCalls": false, + "bareInvokeWithoutWrapper": true, + "insideFencedCodeBlock": true + }, + "toolCallPartCount": 0, + "textLength": 93, + "error": null + } + ], + "selectError": null, + "allModels": [ + { + "id": "claude-opus-4.6", + "family": "claude-opus-4.6", + "vendor": "copilot", + "version": "claude-opus-4.6", + "maxInputTokens": 935793 + }, + { + "id": "claude-opus-4.7", + "family": "claude-opus-4.7", + "vendor": "copilot", + "version": "claude-opus-4.7", + "maxInputTokens": 935793 + }, + { + "id": "claude-opus-4.8", + "family": "claude-opus-4.8", + "vendor": "copilot", + "version": "claude-opus-4.8", + "maxInputTokens": 935793 + }, + { + "id": "claude-opus-5", + "family": "claude-opus-5", + "vendor": "copilot", + "version": "claude-opus-5", + "maxInputTokens": 935793 + }, + { + "id": "claude-sonnet-4.6", + "family": "claude-sonnet-4.6", + "vendor": "copilot", + "version": "claude-sonnet-4.6", + "maxInputTokens": 935793 + }, + { + "id": "claude-sonnet-5", + "family": "claude-sonnet-5", + "vendor": "copilot", + "version": "claude-sonnet-5", + "maxInputTokens": 935793 + }, + { + "id": "gemini-3.1-pro-preview", + "family": "gemini-3.1-pro-preview", + "vendor": "copilot", + "version": "gemini-3.1-pro-preview", + "maxInputTokens": 935793 + }, + { + "id": "gemini-3.5-flash", + "family": "gemini-3.5-flash", + "vendor": "copilot", + "version": "gemini-3.5-flash", + "maxInputTokens": 935793 + }, + { + "id": "gemini-3.6-flash", + "family": "gemini-3.6-flash", + "vendor": "copilot", + "version": "gemini-3.6-flash", + "maxInputTokens": 935793 + }, + { + "id": "gpt-5.3-codex", + "family": "gpt-5.3-codex", + "vendor": "copilot", + "version": "gpt-5.3-codex", + "maxInputTokens": 271790 + }, + { + "id": "gpt-5.4-mini", + "family": "gpt-5.4-mini", + "vendor": "copilot", + "version": "gpt-5.4-mini", + "maxInputTokens": 271790 + }, + { + "id": "gpt-5.4", + "family": "gpt-5.4", + "vendor": "copilot", + "version": "gpt-5.4", + "maxInputTokens": 921793 + }, + { + "id": "gpt-5.5", + "family": "gpt-5.5", + "vendor": "copilot", + "version": "gpt-5.5", + "maxInputTokens": 921793 + }, + { + "id": "gpt-5.6-luna", + "family": "gpt-5.6-luna", + "vendor": "copilot", + "version": "gpt-5.6-luna", + "maxInputTokens": 921793 + }, + { + "id": "gpt-5.6-sol", + "family": "gpt-5.6-sol", + "vendor": "copilot", + "version": "gpt-5.6-sol", + "maxInputTokens": 921793 + }, + { + "id": "gpt-5.6-terra", + "family": "gpt-5.6-terra", + "vendor": "copilot", + "version": "gpt-5.6-terra", + "maxInputTokens": 921793 + }, + { + "id": "grok-4.5", + "family": "grok-4.5", + "vendor": "copilot", + "version": "grok-4.5", + "maxInputTokens": 424794 + }, + { + "id": "mai-code-1-flash-picker", + "family": "oswe-vscode-modelD", + "vendor": "copilot", + "version": "mai-code-1-flash-picker", + "maxInputTokens": 127790 + }, + { + "id": "gpt-5-mini", + "family": "gpt-5-mini", + "vendor": "copilot", + "version": "gpt-5-mini", + "maxInputTokens": 127790 + }, + { + "id": "gpt-4o-mini", + "family": "gpt-4o-mini", + "vendor": "copilot", + "version": "gpt-4o-mini-2024-07-18", + "maxInputTokens": 12078 + }, + { + "id": "claude-haiku-4.5", + "family": "claude-haiku-4.5", + "vendor": "copilot", + "version": "claude-haiku-4.5", + "maxInputTokens": 135790 + }, + { + "id": "auto", + "family": "gpt-5.3-codex", + "vendor": "copilot", + "version": "gpt-5.3-codex", + "maxInputTokens": 271790 + }, + { + "id": "copilot-utility-small", + "family": "copilot-utility-small", + "vendor": "copilot", + "version": "gpt-4o-mini-2024-07-18", + "maxInputTokens": 12078 + }, + { + "id": "copilot-utility", + "family": "copilot-utility", + "vendor": "copilot", + "version": "gpt-5.3-codex", + "maxInputTokens": 271790 + } + ], + "claudeModelIds": [ + "claude-opus-4.6", + "claude-opus-4.7", + "claude-opus-4.8", + "claude-opus-5", + "claude-sonnet-4.6", + "claude-sonnet-5", + "claude-haiku-4.5" + ], + "finishedAt": "2026-08-08T18:24:02.554Z" +} diff --git a/src/api/providers/__tests__/vscode-lm.spec.ts b/src/api/providers/__tests__/vscode-lm.spec.ts index 69743f18ee..11ed97a6bf 100644 --- a/src/api/providers/__tests__/vscode-lm.spec.ts +++ b/src/api/providers/__tests__/vscode-lm.spec.ts @@ -303,6 +303,35 @@ describe("VsCodeLmHandler", () => { }) } + const streamMixedParts = (parts: Array) => { + mockLanguageModelChat.sendRequest.mockResolvedValueOnce({ + stream: (async function* () { + for (const part of parts) { + yield typeof part === "string" + ? new vscode.LanguageModelTextPart(part) + : new vscode.LanguageModelToolCallPart("native-1", part.name, part.input) + } + return + })(), + text: (async function* () { + yield "" + return + })(), + }) + } + + const drain = async () => { + const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], { + taskId: "test-task", + tools: salvageTools, + }) + const chunks = [] + for await (const chunk of stream) { + chunks.push(chunk) + } + return chunks + } + const collect = async (parts: string[]) => { streamTextParts(parts) const stream = handler.createMessage("system", [{ role: "user" as const, content: "hi" }], { @@ -376,6 +405,54 @@ describe("VsCodeLmHandler", () => { expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([{ type: "text", text: block }]) expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) }) + + it("emits prose before the recovered tool call", async () => { + const chunks = await collect([ + "Thinking. ", + 'add', + ]) + + expect(chunks.map((chunk) => chunk.type)).toEqual(["text", "tool_call", "usage"]) + }) + + it("flushes buffered text before a native tool call so no text follows a tool_use", async () => { + streamMixedParts([ + 'partial ', + { name: "calculator", input: { operation: "div" } }, + ]) + const chunks = await drain() + + const lastText = chunks.map((chunk) => chunk.type).lastIndexOf("text") + const firstToolCall = chunks.map((chunk) => chunk.type).indexOf("tool_call") + expect(firstToolCall).toBeGreaterThan(lastText) + }) + + it("does not latch buffering on prose that merely mentions the tag", async () => { + const chunks = await collect(["never emit markup as text. ", "Streaming continues."]) + + expect(chunks.filter((chunk) => chunk.type === "text")).toEqual([ + { type: "text", text: "never emit markup as text. " }, + { type: "text", text: "Streaming continues." }, + ]) + expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) + }) + + it("does not recover an invoke block quoted inside a fenced code block", async () => { + const block = 'add' + const chunks = await collect(["Do NOT do this:\n```\n" + block + "\n```\n"]) + + expect(chunks.some((chunk) => chunk.type === "tool_call")).toBe(false) + }) + + it("sanitizes lone surrogates in the system prompt", async () => { + streamTextParts(["ok"]) + const stream = handler.createMessage("sys\uD800tem", [{ role: "user" as const, content: "hi" }]) + for await (const _chunk of stream) { + // drain + } + + expect(vscode.LanguageModelChatMessage.Assistant).toHaveBeenCalledWith("sys\uFFFDtem") + }) }) it("should handle native tool calls when tools are provided", async () => { @@ -1246,6 +1323,33 @@ describe("leaked tool-call recovery", () => { expect(trailingPartialToolMarkerLength("a < b")).toBe(0) expect(trailingPartialToolMarkerLength("text ")).toBe(0) }) + + it("holds back an invoke tag whose name attribute has not arrived", () => { + expect(trailingPartialToolMarkerLength("text { + expect(trailingPartialToolMarkerLength(" { + it("does not recover an invoke block inside a fenced code block", () => { + const text = "```\n" + invoke("update_todo_list", param("todos", "[x] one")) + "\n```" + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toContain("invoke") + }) + + it("does not recover an invoke block inside an inline code span", () => { + const text = "avoid `" + invoke("update_todo_list", param("todos", "x")) + "`" + + const { calls } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + }) }) }) diff --git a/src/api/providers/vscode-lm.ts b/src/api/providers/vscode-lm.ts index d72779fbec..0ce4798e74 100644 --- a/src/api/providers/vscode-lm.ts +++ b/src/api/providers/vscode-lm.ts @@ -8,8 +8,12 @@ import type { ApiHandlerOptions } from "../../shared/api" import { SELECTOR_SEPARATOR, stringifyVsCodeLmModelSelector } from "../../shared/vsCodeSelectorUtils" import { normalizeToolSchema } from "../../utils/json-schema" -import { ApiStream } from "../transform/stream" -import { convertToVsCodeLmMessages, extractTextCountFromMessage } from "../transform/vscode-lm-format" +import { ApiStream, ApiStreamChunk } from "../transform/stream" +import { + convertToVsCodeLmMessages, + extractTextCountFromMessage, + sanitizeSurrogates, +} from "../transform/vscode-lm-format" import { BaseProvider } from "./base-provider" import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, CompletePromptOptions } from "../index" @@ -72,18 +76,43 @@ function convertToVsCodeLmTools(tools: OpenAI.Chat.ChatCompletionTool[]): vscode * tool we actually offered this turn are treated as calls; everything else is passed through * unchanged as text. */ -const LEAKED_TOOL_CALL_START = /<(?:antml:)?(?:function_calls|invoke)\b/i +// Latching on a bare `([\s\S]*?)<\/(?:antml:)?invoke\s*>/gi const LEAKED_INVOKE_PARAM = /<(?:antml:)?parameter\s+name="([^"]+)"\s*>([\s\S]*?)<\/(?:antml:)?parameter\s*>/gi +/** Upper bound on an incomplete ` { @@ -104,6 +133,7 @@ function parseLeakedInvokeParams(body: string): Record { export function extractLeakedToolCalls( text: string, validToolNames: ReadonlySet, + precedingText = "", ): { calls: Array<{ name: string; input: Record }>; leftoverText: string } { const calls: Array<{ name: string; input: Record }> = [] let leftover = "" @@ -114,10 +144,11 @@ export function extractLeakedToolCalls( while ((match = LEAKED_INVOKE_BLOCK.exec(text)) !== null) { leftover += text.slice(lastIndex, match.index) const name = match[1] - if (validToolNames.has(name)) { + // Quote detection needs the text streamed before the buffer, since a fence may have opened there. + if (validToolNames.has(name) && !isQuotedAsCode(precedingText + text, precedingText.length + match.index)) { calls.push({ name, input: parseLeakedInvokeParams(match[2]) }) } else { - // Not one of our tools — keep the block as literal text. + // Not one of our tools, or quoted as code — keep the block as literal text. leftover += match[0] } lastIndex = match.index + match[0].length @@ -643,7 +674,7 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan // Convert Anthropic messages to VS Code LM messages const vsCodeLmMessages: vscode.LanguageModelChatMessage[] = [ - vscode.LanguageModelChatMessage.Assistant(systemPrompt), + vscode.LanguageModelChatMessage.Assistant(sanitizeSurrogates(systemPrompt)), ...convertToVsCodeLmMessages(cleanedMessages), ] @@ -668,8 +699,53 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan let salvageBuffering = false let salvageBuffer = "" let salvageCarry = "" + let salvageEmittedText = "" let salvagedToolCallIndex = 0 + // Drains the salvage state into ordered chunks: prose first, then any recovered calls. Must + // run before a native tool_call is yielded — text after a tool_use block is rejected by + // Anthropic once the turn is serialized back into history. + const flushSalvage = (): ApiStreamChunk[] => { + const flushed: ApiStreamChunk[] = [] + if (!salvageLeakedToolCalls) { + return flushed + } + + if (!salvageBuffering) { + if (salvageCarry) { + flushed.push({ type: "text", text: salvageCarry }) + salvageCarry = "" + } + return flushed + } + + const buffered = salvageBuffer + salvageBuffering = false + salvageBuffer = "" + if (!buffered) { + return flushed + } + + const { calls, leftoverText } = extractLeakedToolCalls(buffered, providedToolNames, salvageEmittedText) + salvageEmittedText += buffered + if (leftoverText) { + flushed.push({ type: "text", text: leftoverText }) + } + for (const call of calls) { + console.warn( + "Zoo Code : Recovered a tool call the model emitted as text instead of a structured tool call:", + { name: call.name, params: Object.keys(call.input) }, + ) + flushed.push({ + type: "tool_call", + id: `vscodelm-salvaged-${Date.now()}-${salvagedToolCallIndex++}`, + name: call.name, + arguments: JSON.stringify(call.input), + }) + } + return flushed + } + try { // Create the response stream with required options const requestOptions: vscode.LanguageModelChatRequestOptions = { @@ -715,6 +791,7 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan if (markerMatch) { const before = combined.slice(0, markerMatch.index) if (before) { + salvageEmittedText += before yield { type: "text", text: before } } salvageBuffering = true @@ -725,10 +802,12 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan const emit = carryLength > 0 ? combined.slice(0, combined.length - carryLength) : combined salvageCarry = carryLength > 0 ? combined.slice(combined.length - carryLength) : "" if (emit) { + salvageEmittedText += emit yield { type: "text", text: emit } } } } else if (chunk instanceof vscode.LanguageModelToolCallPart) { + yield* flushSalvage() try { // Validate tool call parameters if (!chunk.name || typeof chunk.name !== "string") { @@ -776,35 +855,7 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan } // Flush any leaked tool-call recovery state accumulated during streaming. - if (salvageLeakedToolCalls) { - // A carried tail that never became a marker is just ordinary text. - if (!salvageBuffering && salvageCarry) { - yield { type: "text", text: salvageCarry } - } - - if (salvageBuffering && salvageBuffer) { - const { calls, leftoverText } = extractLeakedToolCalls(salvageBuffer, providedToolNames) - - // Emit surrounding prose first so recovered tool calls come last, matching the - // ordering of a normal native tool-calling turn. - if (leftoverText) { - yield { type: "text", text: leftoverText } - } - - for (const call of calls) { - console.warn( - "Zoo Code : Recovered a tool call the model emitted as text instead of a structured tool call:", - { name: call.name, params: Object.keys(call.input) }, - ) - yield { - type: "tool_call", - id: `vscodelm-salvaged-${Date.now()}-${salvagedToolCallIndex++}`, - name: call.name, - arguments: JSON.stringify(call.input), - } - } - } - } + yield* flushSalvage() // Count tokens in the accumulated text after stream completion const totalOutputTokens: number = await this.internalCountTokens(accumulatedText) diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index 18bac948e3..275126b5ce 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -361,6 +361,48 @@ describe("sanitizeSurrogates", () => { }) }) +describe("convertToVsCodeLmMessages surrogate sanitization", () => { + const lone = "bad\uD800end" + const sanitized = "bad\uFFFDend" + + const textValues = (message: { content: unknown }) => + (message.content as MockLanguageModelTextPart[]).map((part) => part.value) + + it("sanitizes a simple string message", () => { + const result = convertToVsCodeLmMessages([{ role: "user", content: lone }]) + expect(textValues(result[0])).toEqual([sanitized]) + }) + + it("sanitizes string tool_result content", () => { + const result = convertToVsCodeLmMessages([ + { role: "user", content: [{ type: "tool_result", tool_use_id: "tool-1", content: lone }] }, + ]) + const toolResult = result[0].content[0] as MockLanguageModelToolResultPart + expect(toolResult.content[0].value).toBe(sanitized) + }) + + it("sanitizes tool_result text blocks", () => { + const result = convertToVsCodeLmMessages([ + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tool-1", content: [{ type: "text", text: lone }] }], + }, + ]) + const toolResult = result[0].content[0] as MockLanguageModelToolResultPart + expect(toolResult.content[0].value).toBe(sanitized) + }) + + it("sanitizes user text blocks", () => { + const result = convertToVsCodeLmMessages([{ role: "user", content: [{ type: "text", text: lone }] }]) + expect(textValues(result[0])).toContain(sanitized) + }) + + it("sanitizes assistant text blocks", () => { + const result = convertToVsCodeLmMessages([{ role: "assistant", content: [{ type: "text", text: lone }] }]) + expect(textValues(result[0])).toContain(sanitized) + }) +}) + describe("convertToAnthropicRole", () => { it("should convert assistant role correctly", () => { const result = convertToAnthropicRole(vscode.LanguageModelChatMessageRole.Assistant) From cbac74d7e0af2016b591959c7ccfd867d02040ef Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 8 Aug 2026 12:31:30 -0700 Subject: [PATCH 4/9] chore(knip): exclude .roo skill assets from unused-file analysis Skill directories hold reference scripts and captured artifacts that are intentionally never imported by the build. --- knip.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knip.json b/knip.json index db102031eb..a6e67d7e89 100644 --- a/knip.json +++ b/knip.json @@ -1,6 +1,6 @@ { "$schema": "https://unpkg.com/knip@5/schema.json", - "ignore": ["**/__tests__/**", "apps/vscode-e2e/**", "scripts/**", "apps/cli/scripts/**"], + "ignore": ["**/__tests__/**", "apps/vscode-e2e/**", "scripts/**", "apps/cli/scripts/**", ".roo/**"], "ignoreDependencies": ["lint-staged"], "ignoreExportsUsedInFile": true, "playwright": false, From 220ee89bc254e38229da562b188250919b0ec34c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sat, 8 Aug 2026 22:06:49 -0700 Subject: [PATCH 5/9] fix(vscode-lm): address review feedback on leaked tool-call recovery - dispose the probe CancellationTokenSource in a finally block --- .roo/skills/probe-vscode-lm-api/SKILL.md | 10 +-- .../probe-vscode-lm-api/scripts/extension.js | 4 +- src/api/providers/__tests__/vscode-lm.spec.ts | 22 +++++++ src/api/providers/vscode-lm.ts | 62 ++++++++++++++----- 4 files changed, 78 insertions(+), 20 deletions(-) diff --git a/.roo/skills/probe-vscode-lm-api/SKILL.md b/.roo/skills/probe-vscode-lm-api/SKILL.md index 95d365ebcf..ff3a038f11 100644 --- a/.roo/skills/probe-vscode-lm-api/SKILL.md +++ b/.roo/skills/probe-vscode-lm-api/SKILL.md @@ -8,17 +8,17 @@ description: How to empirically probe the VS Code Language Model API (`vscode.lm ## When to Use This Skill - A claim is being made about what a Copilot-backed model _actually_ emits over `vscode.lm` (tool-call parts vs. text), and it needs evidence rather than inference. -- Changing [`extractLeakedToolCalls()`](src/api/providers/vscode-lm.ts) or its guards, and you need a false-positive corpus. +- Changing [`extractLeakedToolCalls()`](../../../src/api/providers/vscode-lm.ts) or its guards, and you need a false-positive corpus. - Any question that can only be answered by real `model.sendRequest()` traffic — the mocked unit tests cannot answer it. ## When NOT to Use This Skill -- Ordinary provider work covered by [`src/api/providers/__tests__/vscode-lm.spec.ts`](src/api/providers/__tests__/vscode-lm.spec.ts). Live probing is slow and burns Copilot quota. +- Ordinary provider work covered by [`src/api/providers/__tests__/vscode-lm.spec.ts`](../../../src/api/providers/__tests__/vscode-lm.spec.ts). Live probing is slow and burns Copilot quota. - Anything about non-`vscode-lm` providers. Anthropic-API behavior does not transfer. ## Running the Probe -Scripts live in [`scripts/`](.roo/skills/probe-vscode-lm-api/scripts/) next to this file. +Scripts live in [`scripts/`](scripts/) next to this file. 1. Copy `scripts/package.json` and `scripts/extension.js` into a scratch directory, e.g. `\.tmp\lmprobe\`. No build, no `npm install` — it is plain CommonJS against the `vscode` module. 2. Adjust `OUT_DIR` at the top of `extension.js` to the transcript output directory. @@ -65,7 +65,7 @@ Never `npx vitest` — it resolves a wrong hoisted 3.2.4 instead of the pinned 4 ## Measured Findings -Sample: 210 live requests, 7 Copilot Claude models x 6 scenarios x 5 repeats, 0 errors. Raw evidence in [`transcripts/`](.roo/skills/probe-vscode-lm-api/transcripts/). +Sample: 210 live requests, 7 Copilot Claude models x 6 scenarios x 5 repeats, 0 errors. Raw evidence in [`transcripts/`](transcripts/). | Scenario | Setup | Runs | ` { it("does not hold back an over-long trailing fragment", () => { expect(trailingPartialToolMarkerLength(" { + expect(trailingPartialToolMarkerLength("text <" + "a".repeat(200))).toBe(0) + }) }) describe("quoted markup", () => { @@ -1350,6 +1354,24 @@ describe("leaked tool-call recovery", () => { expect(calls).toHaveLength(0) }) + + it("does not recover an invoke block quoted in unfenced, backtick-free prose", () => { + const text = "You must never emit " + invoke("update_todo_list", param("todos", "x")) + " directly." + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toBe(text) + }) + + it("keeps wrapper tags around a block that was not recovered", () => { + const text = `${invoke("some_other_tool", param("x", "1"))}` + + const { calls, leftoverText } = extractLeakedToolCalls(text, new Set(["update_todo_list"])) + + expect(calls).toHaveLength(0) + expect(leftoverText).toBe(text) + }) }) }) diff --git a/src/api/providers/vscode-lm.ts b/src/api/providers/vscode-lm.ts index 0ce4798e74..5d167f5e2c 100644 --- a/src/api/providers/vscode-lm.ts +++ b/src/api/providers/vscode-lm.ts @@ -93,7 +93,7 @@ const MAX_PARTIAL_INVOKE_CARRY = 64 export function trailingPartialToolMarkerLength(text: string): number { const partialTag = text.match(/<(?:antml:)?[a-zA-Z_]*$/) if (partialTag) { - return partialTag[0].length + return partialTag[0].length <= MAX_PARTIAL_INVOKE_CARRY ? partialTag[0].length : 0 } // An ` directly"), which must not be replayed as a live call. + const after = text.slice(endIndex) + const lineEnd = after.indexOf("\n") + const restOfLine = lineEnd === -1 ? after : after.slice(0, lineEnd) + return restOfLine.replace(/<[^<>]*>/g, "").trim().length > 0 } function parseLeakedInvokeParams(body: string): Record { @@ -136,28 +144,54 @@ export function extractLeakedToolCalls( precedingText = "", ): { calls: Array<{ name: string; input: Record }>; leftoverText: string } { const calls: Array<{ name: string; input: Record }> = [] - let leftover = "" + // Text between recovered/passed-through blocks, kept as segments so the wrapper cleanup below + // only touches segments adjacent to a block that was actually recovered. + const segments: Array<{ text: string; nearRecovery: boolean }> = [] + let pending = "" let lastIndex = 0 LEAKED_INVOKE_BLOCK.lastIndex = 0 let match: RegExpExecArray | null while ((match = LEAKED_INVOKE_BLOCK.exec(text)) !== null) { - leftover += text.slice(lastIndex, match.index) + pending += text.slice(lastIndex, match.index) const name = match[1] // Quote detection needs the text streamed before the buffer, since a fence may have opened there. - if (validToolNames.has(name) && !isQuotedAsCode(precedingText + text, precedingText.length + match.index)) { + if ( + validToolNames.has(name) && + !isQuotedAsCode( + precedingText + text, + precedingText.length + match.index, + precedingText.length + match.index + match[0].length, + ) + ) { calls.push({ name, input: parseLeakedInvokeParams(match[2]) }) + segments.push({ text: pending, nearRecovery: true }) + pending = "" + // The segment that follows a recovery also holds that call's closing wrapper. + segments.push({ text: "", nearRecovery: true }) } else { // Not one of our tools, or quoted as code — keep the block as literal text. - leftover += match[0] + pending += match[0] } lastIndex = match.index + match[0].length } - leftover += text.slice(lastIndex) + pending += text.slice(lastIndex) + const trailing = + segments.length > 0 && segments[segments.length - 1].nearRecovery && segments[segments.length - 1].text === "" + if (trailing) { + segments[segments.length - 1].text = pending + } else { + segments.push({ text: pending, nearRecovery: false }) + } - // Remove bare function-call wrapper tags left behind (cosmetic; also avoids re-teaching - // the model this format when the turn is later sent back as history). - leftover = leftover.replace(/<\/?(?:antml:)?function_calls\s*>/gi, "") + // Remove the wrapper tags belonging to a recovered call (cosmetic; also avoids re-teaching the + // model this format when the turn is sent back as history). Wrappers around blocks that were + // NOT recovered are user-visible text and must survive verbatim. + const leftover = segments + .map((segment) => + segment.nearRecovery ? segment.text.replace(/<\/?(?:antml:)?function_calls\s*>/gi, "") : segment.text, + ) + .join("") return { calls, leftoverText: leftover } } From 8c802527c9ac1393a363b7196670a33d4b4f1d98 Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 9 Aug 2026 07:38:40 -0700 Subject: [PATCH 6/9] docs(skill): drop probe transcripts from repo Remove the ~120KB raw probe transcript corpus from the vscode-lm probe skill; keep the measured findings and their stated limits in SKILL.md. --- .roo/skills/probe-vscode-lm-api/SKILL.md | 4 +- ...ools_declared_compelling_prompt__run1.json | 43 - ...tools_declared_compelling_prompt__run1.txt | 0 ...D_no_tools_asked_to_emit_markup__run2.json | 66 - ..._D_no_tools_asked_to_emit_markup__run2.txt | 5 - ...p_in_prose_false_positive_check__run1.json | 70 - ...up_in_prose_false_positive_check__run1.txt | 17 - ...ted_markup_in_fenced_code_block__run1.json | 38 - ...oted_markup_in_fenced_code_block__run1.txt | 7 - ...p_in_prose_false_positive_check__run1.json | 58 - ...up_in_prose_false_positive_check__run1.txt | 18 - ...D_no_tools_asked_to_emit_markup__run1.json | 34 - ..._D_no_tools_asked_to_emit_markup__run1.txt | 5 - ...ted_markup_in_fenced_code_block__run1.json | 34 - ...oted_markup_in_fenced_code_block__run1.txt | 5 - .../transcripts/false-positive-report.txt | 58 - .../transcripts/summary.json | 3548 ----------------- 17 files changed, 2 insertions(+), 4008 deletions(-) delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__A_tools_declared_compelling_prompt__run1.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt delete mode 100644 .roo/skills/probe-vscode-lm-api/transcripts/summary.json diff --git a/.roo/skills/probe-vscode-lm-api/SKILL.md b/.roo/skills/probe-vscode-lm-api/SKILL.md index ff3a038f11..99d5432ae1 100644 --- a/.roo/skills/probe-vscode-lm-api/SKILL.md +++ b/.roo/skills/probe-vscode-lm-api/SKILL.md @@ -65,7 +65,7 @@ Never `npx vitest` — it resolves a wrong hoisted 3.2.4 instead of the pinned 4 ## Measured Findings -Sample: 210 live requests, 7 Copilot Claude models x 6 scenarios x 5 repeats, 0 errors. Raw evidence in [`transcripts/`](transcripts/). +Sample: 210 live requests, 7 Copilot Claude models x 6 scenarios x 5 repeats, 0 errors. The raw transcripts were not retained; the counts below are the retained record of that run, and re-running the probe is the way to regenerate the underlying evidence. | Scenario | Setup | Runs | `" - }, - { - "type": "rn", - "value": "\n" - }, - { - "type": "rn", - "value": "\n" - }, - { - "type": "rn", - "value": "c:/git/Zoo-Code/" - }, - { - "type": "rn", - "value": "package.json\n" - }, - { - "type": "rn", - "value": "\n" - }, - { - "type": "i", - "raw": "[object Object]" - } - ], - "concatenatedText": "\n\nc:/git/Zoo-Code/package.json\n\n", - "toolCallParts": [], - "error": null, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "iteration": 2 -} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt deleted file mode 100644 index 80c159b8fd..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt +++ /dev/null @@ -1,5 +0,0 @@ - - -c:/git/Zoo-Code/package.json - - \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json deleted file mode 100644 index 78af16167b..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.8", - "modelFamily": "claude-opus-4.8", - "modelVendor": "copilot", - "modelVersion": "claude-opus-4.8", - "maxInputTokens": 935793, - "parts": [ - { - "type": "rn", - "value": "An" - }, - { - "type": "rn", - "value": " inv" - }, - { - "type": "rn", - "value": "oke block is" - }, - { - "type": "rn", - "value": " the" - }, - { - "type": "rn", - "value": " m" - }, - { - "type": "rn", - "value": "echanism used to call a tool. It's written as an XML-like structure containing a function invocation with named parameters." - }, - { - "type": "rn", - "value": " Here's a literal example, written as plain text:\n\n```xml\n\nsrc/example.txt\n\n```\n\nThe structure works like this:\n\n- An outer `invoke` tag carries a `name` attribute identifying which tool to call (here" - }, - { - "type": "rn", - "value": ", `read_file`).\n- Inside it, one or more `parameter` tags each carry a `name` attribute identifying the argument (here, `path`).\n- The value of each parameter goes between the opening and closing `parameter" - }, - { - "type": "rn", - "value": "` tags (here, `src/example.txt`).\n\nFor string and scalar parameters, the value is written as-is between the tags. For lists or objects, the value would be formatted as JSON instead.\n\nNote that I've shown this as plain," - }, - { - "type": "rn", - "value": " inert text for illustration only — I have not actually called any tool, per your request." - }, - { - "type": "i", - "raw": "[object Object]" - } - ], - "concatenatedText": "An invoke block is the mechanism used to call a tool. It's written as an XML-like structure containing a function invocation with named parameters. Here's a literal example, written as plain text:\n\n```xml\n\nsrc/example.txt\n\n```\n\nThe structure works like this:\n\n- An outer `invoke` tag carries a `name` attribute identifying which tool to call (here, `read_file`).\n- Inside it, one or more `parameter` tags each carry a `name` attribute identifying the argument (here, `path`).\n- The value of each parameter goes between the opening and closing `parameter` tags (here, `src/example.txt`).\n\nFor string and scalar parameters, the value is written as-is between the tags. For lists or objects, the value would be formatted as JSON instead.\n\nNote that I've shown this as plain, inert text for illustration only — I have not actually called any tool, per your request.", - "toolCallParts": [], - "error": null, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "iteration": 1 -} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt deleted file mode 100644 index 8ae85fc055..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt +++ /dev/null @@ -1,17 +0,0 @@ -An invoke block is the mechanism used to call a tool. It's written as an XML-like structure containing a function invocation with named parameters. Here's a literal example, written as plain text: - -```xml - -src/example.txt - -``` - -The structure works like this: - -- An outer `invoke` tag carries a `name` attribute identifying which tool to call (here, `read_file`). -- Inside it, one or more `parameter` tags each carry a `name` attribute identifying the argument (here, `path`). -- The value of each parameter goes between the opening and closing `parameter` tags (here, `src/example.txt`). - -For string and scalar parameters, the value is written as-is between the tags. For lists or objects, the value would be formatted as JSON instead. - -Note that I've shown this as plain, inert text for illustration only — I have not actually called any tool, per your request. \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json deleted file mode 100644 index ddb348de43..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.8", - "modelFamily": "claude-opus-4.8", - "modelVendor": "copilot", - "modelVersion": "claude-opus-4.8", - "maxInputTokens": 935793, - "parts": [ - { - "type": "rn", - "value": "```" - }, - { - "type": "rn", - "value": "xml\n\n/etc/passwd\n\n```\n\nNote: this is just a formatting" - }, - { - "type": "rn", - "value": " example inside a code block, not an actual tool call." - }, - { - "type": "i", - "raw": "[object Object]" - } - ], - "concatenatedText": "```xml\n\n/etc/passwd\n\n```\n\nNote: this is just a formatting example inside a code block, not an actual tool call.", - "toolCallParts": [], - "error": null, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "iteration": 1 -} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt deleted file mode 100644 index 1dbb76b8a2..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt +++ /dev/null @@ -1,7 +0,0 @@ -```xml - -/etc/passwd - -``` - -Note: this is just a formatting example inside a code block, not an actual tool call. \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json deleted file mode 100644 index 3a29ccdfae..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-5", - "modelFamily": "claude-opus-5", - "modelVendor": "copilot", - "modelVersion": "claude-opus-5", - "maxInputTokens": 935793, - "parts": [ - { - "type": "rn", - "value": "A tool call in this system is expressed as an XML-like block nested inside a function-calls container. The block has:\n\n- an opening tag named `invoke`" - }, - { - "type": "rn", - "value": " with a `name` attribute whose value is the tool's identifier (for example, `read_file`);\n- one or more `parameter` child elements, each with a `name` attribute matching a parameter def" - }, - { - "type": "rn", - "value": "ined in the tool's schema (for example, `path`);\n- the argument value written as the text content of the `parameter` element;\n- closing tags for each" - }, - { - "type": "rn", - "value": " `parameter` and for the `invoke` element.\n\nScalar values (strings, numbers, booleans) go in as plain text. Lists and objects are written as JSON inside the parameter body" - }, - { - "type": "rn", - "value": ". The whole thing is emitted instead of, or alongside, prose in the assistant's message, and the runtime int" - }, - { - "type": "rn", - "value": "ercepts it, executes the named tool, and feeds the result back.\n\nHere is a literal, illustrative example written as plain text (this is documentation, not an actual call):\n\n```xml" - }, - { - "type": "rn", - "value": "\n\n src/app/main.py\n\n```\n\nNote that in the real wire format the tags carry a namespace prefix and sit inside a surrounding function-calls wrapper;" - }, - { - "type": "rn", - "value": " the snippet above is stripped down to just the `invoke`/`parameter` structure you asked about, so it is inert." - }, - { - "type": "i", - "raw": "[object Object]" - } - ], - "concatenatedText": "A tool call in this system is expressed as an XML-like block nested inside a function-calls container. The block has:\n\n- an opening tag named `invoke` with a `name` attribute whose value is the tool's identifier (for example, `read_file`);\n- one or more `parameter` child elements, each with a `name` attribute matching a parameter defined in the tool's schema (for example, `path`);\n- the argument value written as the text content of the `parameter` element;\n- closing tags for each `parameter` and for the `invoke` element.\n\nScalar values (strings, numbers, booleans) go in as plain text. Lists and objects are written as JSON inside the parameter body. The whole thing is emitted instead of, or alongside, prose in the assistant's message, and the runtime intercepts it, executes the named tool, and feeds the result back.\n\nHere is a literal, illustrative example written as plain text (this is documentation, not an actual call):\n\n```xml\n\n src/app/main.py\n\n```\n\nNote that in the real wire format the tags carry a namespace prefix and sit inside a surrounding function-calls wrapper; the snippet above is stripped down to just the `invoke`/`parameter` structure you asked about, so it is inert.", - "toolCallParts": [], - "error": null, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "iteration": 1 -} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt deleted file mode 100644 index 62937045f9..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt +++ /dev/null @@ -1,18 +0,0 @@ -A tool call in this system is expressed as an XML-like block nested inside a function-calls container. The block has: - -- an opening tag named `invoke` with a `name` attribute whose value is the tool's identifier (for example, `read_file`); -- one or more `parameter` child elements, each with a `name` attribute matching a parameter defined in the tool's schema (for example, `path`); -- the argument value written as the text content of the `parameter` element; -- closing tags for each `parameter` and for the `invoke` element. - -Scalar values (strings, numbers, booleans) go in as plain text. Lists and objects are written as JSON inside the parameter body. The whole thing is emitted instead of, or alongside, prose in the assistant's message, and the runtime intercepts it, executes the named tool, and feeds the result back. - -Here is a literal, illustrative example written as plain text (this is documentation, not an actual call): - -```xml - - src/app/main.py - -``` - -Note that in the real wire format the tags carry a namespace prefix and sit inside a surrounding function-calls wrapper; the snippet above is stripped down to just the `invoke`/`parameter` structure you asked about, so it is inert. \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json deleted file mode 100644 index 7fb29b7391..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-4.6", - "modelFamily": "claude-sonnet-4.6", - "modelVendor": "copilot", - "modelVersion": "claude-sonnet-4.6", - "maxInputTokens": 935793, - "parts": [ - { - "type": "rn", - "value": "\n\nc:/git/Zoo-Code/package.json\n\n" - }, - { - "type": "i", - "raw": "[object Object]" - } - ], - "concatenatedText": "\n\nc:/git/Zoo-Code/package.json\n\n", - "toolCallParts": [], - "error": null, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "iteration": 1 -} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt deleted file mode 100644 index 80c159b8fd..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt +++ /dev/null @@ -1,5 +0,0 @@ - - -c:/git/Zoo-Code/package.json - - \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json deleted file mode 100644 index 153710977f..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-4.6", - "modelFamily": "claude-sonnet-4.6", - "modelVendor": "copilot", - "modelVersion": "claude-sonnet-4.6", - "maxInputTokens": 935793, - "parts": [ - { - "type": "rn", - "value": "```" - }, - { - "type": "rn", - "value": "xml\n\n/etc/passwd\n\n```" - }, - { - "type": "i", - "raw": "[object Object]" - } - ], - "concatenatedText": "```xml\n\n/etc/passwd\n\n```", - "toolCallParts": [], - "error": null, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "iteration": 1 -} diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt b/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt deleted file mode 100644 index 78d575b390..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt +++ /dev/null @@ -1,5 +0,0 @@ -```xml - -/etc/passwd - -``` \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt b/.roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt deleted file mode 100644 index 782d82cee9..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/false-positive-report.txt +++ /dev/null @@ -1,58 +0,0 @@ -passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run1.txt [] -passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run2.txt [] -passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run3.txt [] -passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run4.txt [] -passthrough claude-haiku-4.5__D_no_tools_asked_to_emit_markup__run5.txt [] -passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run1.txt [] -passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run2.txt [] -passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run3.txt [] -passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run4.txt [] -passthrough claude-haiku-4.5__E_quoted_markup_in_prose_false_positive_check__run5.txt [] -passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run1.txt [] -passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run2.txt [] -passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run3.txt [] -passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run4.txt [] -passthrough claude-haiku-4.5__F_quoted_markup_in_fenced_code_block__run5.txt [] -RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run2.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run3.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run4.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-opus-4.6__D_no_tools_asked_to_emit_markup__run5.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -passthrough claude-opus-4.6__E_quoted_markup_in_prose_false_positive_check__run1.txt [] -passthrough claude-opus-4.6__E_quoted_markup_in_prose_false_positive_check__run2.txt [] -passthrough claude-opus-4.6__E_quoted_markup_in_prose_false_positive_check__run5.txt [] -passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run1.txt [] -passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run2.txt [] -passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run3.txt [] -passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run4.txt [] -passthrough claude-opus-4.8__E_quoted_markup_in_prose_false_positive_check__run5.txt [] -passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run1.txt [] -passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run2.txt [] -passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run3.txt [] -passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run4.txt [] -passthrough claude-opus-4.8__F_quoted_markup_in_fenced_code_block__run5.txt [] -passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run1.txt [] -passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run2.txt [] -passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run3.txt [] -passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run4.txt [] -passthrough claude-opus-5__E_quoted_markup_in_prose_false_positive_check__run5.txt [] -passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run1.txt [] -passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run2.txt [] -passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run3.txt [] -passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run4.txt [] -passthrough claude-opus-5__F_quoted_markup_in_fenced_code_block__run5.txt [] -RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run1.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run2.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run3.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run4.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -RECOVERED claude-sonnet-4.6__D_no_tools_asked_to_emit_markup__run5.txt [{"name":"read_file","input":{"path":"c:/git/Zoo-Code/package.json"}}] -passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run1.txt [] -passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run2.txt [] -passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run3.txt [] -passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run4.txt [] -passthrough claude-sonnet-4.6__F_quoted_markup_in_fenced_code_block__run5.txt [] -passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run1.txt [] -passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run2.txt [] -passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run3.txt [] -passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run4.txt [] -passthrough claude-sonnet-5__E_quoted_markup_in_prose_false_positive_check__run5.txt [] -passthrough claude-sonnet-5__F_quoted_markup_in_fenced_code_block__run5.txt [] \ No newline at end of file diff --git a/.roo/skills/probe-vscode-lm-api/transcripts/summary.json b/.roo/skills/probe-vscode-lm-api/transcripts/summary.json deleted file mode 100644 index 7d42ad0799..0000000000 --- a/.roo/skills/probe-vscode-lm-api/transcripts/summary.json +++ /dev/null @@ -1,3548 +0,0 @@ -{ - "startedAt": "2026-08-08T18:14:17.466Z", - "vscodeVersion": "1.128.1", - "models": [], - "runs": [ - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 114, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.6", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.6", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.6", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.6", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.6", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 459, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.6", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 451, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 51, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 51, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.6", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 474, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 7, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 7, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 7, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 7, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 7, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.7", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.7", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.7", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.7", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.7", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.7", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.7", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.7", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.7", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.7", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.7", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.7", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.7", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.7", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.7", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.7", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 166, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.7", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 32, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.7", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 32, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.7", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 32, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.7", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 32, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.7", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 87, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.7", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 57, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.7", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 57, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.7", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 85, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.7", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 54, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.7", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 8, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.7", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 8, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.7", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 8, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.7", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 8, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.7", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 8, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.8", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.8", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 46, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.8", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 46, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.8", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 46, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-4.8", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 46, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.8", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 46, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.8", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 29, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.8", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.8", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 29, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-4.8", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 28, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.8", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.8", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 45, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.8", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 46, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.8", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-4.8", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.8", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 494, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.8", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 501, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.8", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 461, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.8", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 409, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-4.8", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 356, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.8", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 930, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.8", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1397, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.8", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1284, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.8", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1042, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-4.8", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1099, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.8", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 180, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.8", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 237, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.8", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 211, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.8", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 183, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-4.8", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 218, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 25, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-opus-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 29, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 29, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 29, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 29, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-opus-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 47, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-opus-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 20, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 461, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 1675, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 308, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 1138, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-opus-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 585, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-5", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1268, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-5", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1831, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-5", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1008, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-5", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1573, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-opus-5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1705, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-5", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-5", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-5", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-5", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-opus-5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-4.6", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-4.6", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-4.6", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-4.6", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-4.6", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 134, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-4.6", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 370, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-4.6", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 420, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-4.6", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 267, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-4.6", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 361, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-4.6", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 323, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-4.6", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-4.6", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-4.6", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-4.6", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-4.6", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-sonnet-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-sonnet-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-sonnet-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 103, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 61, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 66, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 66, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-sonnet-5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 34, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-5", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1126, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-5", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 838, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-5", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1149, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-5", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1045, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-sonnet-5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 1050, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 16, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 0, - "textLength": 21, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 110, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-sonnet-5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 90, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "A_tools_declared_compelling_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "B_tools_declared_no_system_prompt", - "modelId": "claude-haiku-4.5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 0, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-haiku-4.5", - "iteration": 1, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 59, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-haiku-4.5", - "iteration": 2, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 59, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-haiku-4.5", - "iteration": 3, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 41, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-haiku-4.5", - "iteration": 4, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 230, - "error": null - }, - { - "scenario": "C_tools_declared_long_context", - "modelId": "claude-haiku-4.5", - "iteration": 5, - "markers": { - "hasInvoke": false, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": false - }, - "toolCallPartCount": 1, - "textLength": 59, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-haiku-4.5", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 145, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-haiku-4.5", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 145, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-haiku-4.5", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 145, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-haiku-4.5", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 145, - "error": null - }, - { - "scenario": "D_no_tools_asked_to_emit_markup", - "modelId": "claude-haiku-4.5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": true, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": true, - "bareInvokeWithoutWrapper": false, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 145, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-haiku-4.5", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 609, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-haiku-4.5", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 602, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-haiku-4.5", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 915, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-haiku-4.5", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 680, - "error": null - }, - { - "scenario": "E_quoted_markup_in_prose_false_positive_check", - "modelId": "claude-haiku-4.5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 701, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-haiku-4.5", - "iteration": 1, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-haiku-4.5", - "iteration": 2, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-haiku-4.5", - "iteration": 3, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-haiku-4.5", - "iteration": 4, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - }, - { - "scenario": "F_quoted_markup_in_fenced_code_block", - "modelId": "claude-haiku-4.5", - "iteration": 5, - "markers": { - "hasInvoke": true, - "hasFunctionCalls": false, - "hasAntmlNamespace": false, - "invokeWrappedInFunctionCalls": false, - "bareInvokeWithoutWrapper": true, - "insideFencedCodeBlock": true - }, - "toolCallPartCount": 0, - "textLength": 93, - "error": null - } - ], - "selectError": null, - "allModels": [ - { - "id": "claude-opus-4.6", - "family": "claude-opus-4.6", - "vendor": "copilot", - "version": "claude-opus-4.6", - "maxInputTokens": 935793 - }, - { - "id": "claude-opus-4.7", - "family": "claude-opus-4.7", - "vendor": "copilot", - "version": "claude-opus-4.7", - "maxInputTokens": 935793 - }, - { - "id": "claude-opus-4.8", - "family": "claude-opus-4.8", - "vendor": "copilot", - "version": "claude-opus-4.8", - "maxInputTokens": 935793 - }, - { - "id": "claude-opus-5", - "family": "claude-opus-5", - "vendor": "copilot", - "version": "claude-opus-5", - "maxInputTokens": 935793 - }, - { - "id": "claude-sonnet-4.6", - "family": "claude-sonnet-4.6", - "vendor": "copilot", - "version": "claude-sonnet-4.6", - "maxInputTokens": 935793 - }, - { - "id": "claude-sonnet-5", - "family": "claude-sonnet-5", - "vendor": "copilot", - "version": "claude-sonnet-5", - "maxInputTokens": 935793 - }, - { - "id": "gemini-3.1-pro-preview", - "family": "gemini-3.1-pro-preview", - "vendor": "copilot", - "version": "gemini-3.1-pro-preview", - "maxInputTokens": 935793 - }, - { - "id": "gemini-3.5-flash", - "family": "gemini-3.5-flash", - "vendor": "copilot", - "version": "gemini-3.5-flash", - "maxInputTokens": 935793 - }, - { - "id": "gemini-3.6-flash", - "family": "gemini-3.6-flash", - "vendor": "copilot", - "version": "gemini-3.6-flash", - "maxInputTokens": 935793 - }, - { - "id": "gpt-5.3-codex", - "family": "gpt-5.3-codex", - "vendor": "copilot", - "version": "gpt-5.3-codex", - "maxInputTokens": 271790 - }, - { - "id": "gpt-5.4-mini", - "family": "gpt-5.4-mini", - "vendor": "copilot", - "version": "gpt-5.4-mini", - "maxInputTokens": 271790 - }, - { - "id": "gpt-5.4", - "family": "gpt-5.4", - "vendor": "copilot", - "version": "gpt-5.4", - "maxInputTokens": 921793 - }, - { - "id": "gpt-5.5", - "family": "gpt-5.5", - "vendor": "copilot", - "version": "gpt-5.5", - "maxInputTokens": 921793 - }, - { - "id": "gpt-5.6-luna", - "family": "gpt-5.6-luna", - "vendor": "copilot", - "version": "gpt-5.6-luna", - "maxInputTokens": 921793 - }, - { - "id": "gpt-5.6-sol", - "family": "gpt-5.6-sol", - "vendor": "copilot", - "version": "gpt-5.6-sol", - "maxInputTokens": 921793 - }, - { - "id": "gpt-5.6-terra", - "family": "gpt-5.6-terra", - "vendor": "copilot", - "version": "gpt-5.6-terra", - "maxInputTokens": 921793 - }, - { - "id": "grok-4.5", - "family": "grok-4.5", - "vendor": "copilot", - "version": "grok-4.5", - "maxInputTokens": 424794 - }, - { - "id": "mai-code-1-flash-picker", - "family": "oswe-vscode-modelD", - "vendor": "copilot", - "version": "mai-code-1-flash-picker", - "maxInputTokens": 127790 - }, - { - "id": "gpt-5-mini", - "family": "gpt-5-mini", - "vendor": "copilot", - "version": "gpt-5-mini", - "maxInputTokens": 127790 - }, - { - "id": "gpt-4o-mini", - "family": "gpt-4o-mini", - "vendor": "copilot", - "version": "gpt-4o-mini-2024-07-18", - "maxInputTokens": 12078 - }, - { - "id": "claude-haiku-4.5", - "family": "claude-haiku-4.5", - "vendor": "copilot", - "version": "claude-haiku-4.5", - "maxInputTokens": 135790 - }, - { - "id": "auto", - "family": "gpt-5.3-codex", - "vendor": "copilot", - "version": "gpt-5.3-codex", - "maxInputTokens": 271790 - }, - { - "id": "copilot-utility-small", - "family": "copilot-utility-small", - "vendor": "copilot", - "version": "gpt-4o-mini-2024-07-18", - "maxInputTokens": 12078 - }, - { - "id": "copilot-utility", - "family": "copilot-utility", - "vendor": "copilot", - "version": "gpt-5.3-codex", - "maxInputTokens": 271790 - } - ], - "claudeModelIds": [ - "claude-opus-4.6", - "claude-opus-4.7", - "claude-opus-4.8", - "claude-opus-5", - "claude-sonnet-4.6", - "claude-sonnet-5", - "claude-haiku-4.5" - ], - "finishedAt": "2026-08-08T18:24:02.554Z" -} From d587392dde7f12b89b46e50e769e29c65074266e Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 9 Aug 2026 16:09:31 -0700 Subject: [PATCH 7/9] chore: move probe skill scripts under scripts/, drop .roo knip ignore --- .roo/skills/probe-vscode-lm-api/SKILL.md | 8 ++++---- knip.json | 2 +- .../scripts => scripts/probe-vscode-lm-api}/extension.js | 0 .../scripts => scripts/probe-vscode-lm-api}/package.json | 0 .../probe-vscode-lm-api}/probe-false-positives.spec.ts | 2 ++ 5 files changed, 7 insertions(+), 5 deletions(-) rename {.roo/skills/probe-vscode-lm-api/scripts => scripts/probe-vscode-lm-api}/extension.js (100%) rename {.roo/skills/probe-vscode-lm-api/scripts => scripts/probe-vscode-lm-api}/package.json (100%) rename {.roo/skills/probe-vscode-lm-api/scripts => scripts/probe-vscode-lm-api}/probe-false-positives.spec.ts (84%) diff --git a/.roo/skills/probe-vscode-lm-api/SKILL.md b/.roo/skills/probe-vscode-lm-api/SKILL.md index 99d5432ae1..0150cd8afa 100644 --- a/.roo/skills/probe-vscode-lm-api/SKILL.md +++ b/.roo/skills/probe-vscode-lm-api/SKILL.md @@ -18,10 +18,10 @@ description: How to empirically probe the VS Code Language Model API (`vscode.lm ## Running the Probe -Scripts live in [`scripts/`](scripts/) next to this file. +Scripts live in [`scripts/probe-vscode-lm-api/`](../../../scripts/probe-vscode-lm-api/) at the repo root. -1. Copy `scripts/package.json` and `scripts/extension.js` into a scratch directory, e.g. `\.tmp\lmprobe\`. No build, no `npm install` — it is plain CommonJS against the `vscode` module. -2. Adjust `OUT_DIR` at the top of `extension.js` to the transcript output directory. +1. Copy `scripts/probe-vscode-lm-api/package.json` and `scripts/probe-vscode-lm-api/extension.js` into a scratch directory, e.g. `\.tmp\lmprobe\`. No build, no `npm install` — it is plain CommonJS against the `vscode` module. +2. Adjust `OUT_DIR` at the top of the copied `extension.js` (or set `LM_PROBE_OUT_DIR`) to the transcript output directory. 3. Launch a **new** extension host window: ``` @@ -90,7 +90,7 @@ Observations: ## Reproducing the False-Positive Replay -[`scripts/probe-false-positives.spec.ts`](scripts/probe-false-positives.spec.ts) replays [`extractLeakedToolCalls()`](../../../src/api/providers/vscode-lm.ts) over a transcript directory and writes a `RECOVERED`/`passthrough` report. It has no committed inputs — run the probe first to produce them. Drop it into `src/api/providers/__tests__/`, point `TRANSCRIPTS` (or `LM_PROBE_TRANSCRIPTS`) at the probe's `OUT_DIR`, then: +[`scripts/probe-vscode-lm-api/probe-false-positives.spec.ts`](../../../scripts/probe-vscode-lm-api/probe-false-positives.spec.ts) replays [`extractLeakedToolCalls()`](../../../src/api/providers/vscode-lm.ts) over a transcript directory and writes a `RECOVERED`/`passthrough` report. It has no committed inputs — run the probe first to produce them. Its `../vscode-lm` import and `TRANSCRIPTS` default are written for the copy destination, not for where it is committed. Drop it into `src/api/providers/__tests__/`, point `TRANSCRIPTS` (or `LM_PROBE_TRANSCRIPTS`) at the probe's `OUT_DIR`, then: ``` pnpm --dir src exec vitest run api/providers/__tests__/probe-false-positives.spec.ts diff --git a/knip.json b/knip.json index a6e67d7e89..db102031eb 100644 --- a/knip.json +++ b/knip.json @@ -1,6 +1,6 @@ { "$schema": "https://unpkg.com/knip@5/schema.json", - "ignore": ["**/__tests__/**", "apps/vscode-e2e/**", "scripts/**", "apps/cli/scripts/**", ".roo/**"], + "ignore": ["**/__tests__/**", "apps/vscode-e2e/**", "scripts/**", "apps/cli/scripts/**"], "ignoreDependencies": ["lint-staged"], "ignoreExportsUsedInFile": true, "playwright": false, diff --git a/.roo/skills/probe-vscode-lm-api/scripts/extension.js b/scripts/probe-vscode-lm-api/extension.js similarity index 100% rename from .roo/skills/probe-vscode-lm-api/scripts/extension.js rename to scripts/probe-vscode-lm-api/extension.js diff --git a/.roo/skills/probe-vscode-lm-api/scripts/package.json b/scripts/probe-vscode-lm-api/package.json similarity index 100% rename from .roo/skills/probe-vscode-lm-api/scripts/package.json rename to scripts/probe-vscode-lm-api/package.json diff --git a/.roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts b/scripts/probe-vscode-lm-api/probe-false-positives.spec.ts similarity index 84% rename from .roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts rename to scripts/probe-vscode-lm-api/probe-false-positives.spec.ts index 2d384c927f..5c48318ec0 100644 --- a/.roo/skills/probe-vscode-lm-api/scripts/probe-false-positives.spec.ts +++ b/scripts/probe-vscode-lm-api/probe-false-positives.spec.ts @@ -1,6 +1,8 @@ import fs from "fs" import path from "path" +// Paths here are written for the copy destination `src/api/providers/__tests__/`, not for this +// file's committed location — it is a template to be copied there, never run in place. import { extractLeakedToolCalls } from "../vscode-lm" // Point this at the probe's OUT_DIR. Scratch harness: not a committed test. From 14e85567a30437045814c9714adf0b21f8a8baee Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Mon, 10 Aug 2026 16:46:48 -0700 Subject: [PATCH 8/9] fix(vscode-lm): harden quoted-markup detection and bound the salvage buffer Loop tag stripping until stable so `<