diff --git a/.changeset/fix-code-index-and-metadata-stability.md b/.changeset/fix-code-index-and-metadata-stability.md new file mode 100644 index 00000000000..74b1d5b97de --- /dev/null +++ b/.changeset/fix-code-index-and-metadata-stability.md @@ -0,0 +1,5 @@ +--- +"roo-code-continue": patch +--- + +Fix: code-index incremental updates now batch embeddings by the configured `codeIndex.embeddingBatchSize`, so a single large file no longer floods the embedder with oversized requests (SCNet gateway rejects batches over 20/25 items with 400 errors). Also serializes task-metadata read-modify-write cycles to eliminate concurrent lock contention on `task_metadata.json` (`Lock file is already being held`). diff --git a/.changeset/fix-nightly-display-name.md b/.changeset/fix-nightly-display-name.md new file mode 100644 index 00000000000..9f463f1b461 --- /dev/null +++ b/.changeset/fix-nightly-display-name.md @@ -0,0 +1,7 @@ +--- +"roo-code-continue": patch +--- + +Fix nightly publish failing on the VS Code Marketplace. + +- Rename the nightly build display name from "Roo Code Nightly" to "Roo Code Continue Nightly" (in `apps/vscode-nightly/package.nls.nightly.json`). The Marketplace requires display names to be globally unique across all publishers; "Roo Code Nightly" was already taken by another extension, causing `vsce publish` to fail with "This extension display name is taken". diff --git a/.changeset/fix-unbound-models-api-shape.md b/.changeset/fix-unbound-models-api-shape.md new file mode 100644 index 00000000000..51f8b3fc741 --- /dev/null +++ b/.changeset/fix-unbound-models-api-shape.md @@ -0,0 +1,5 @@ +--- +"roo-code-continue": patch +--- + +Fix Unbound model fetching: the `/models` endpoint now returns a dictionary keyed by model id with camelCase fields (e.g. `maxTokens`, `contextWindow`, `inputTokenPrice`), while the fetcher still expected an array with snake_case fields. This caused "response did not contain an array of models" errors and an empty model list. The fetcher now accepts both dictionary and array shapes, reads both camelCase and snake_case fields, and coerces numeric strings. diff --git a/.changeset/fix-vercel-ai-gateway-non-language-models.md b/.changeset/fix-vercel-ai-gateway-non-language-models.md new file mode 100644 index 00000000000..6a7ef5e4c58 --- /dev/null +++ b/.changeset/fix-vercel-ai-gateway-non-language-models.md @@ -0,0 +1,5 @@ +--- +"roo-code-continue": patch +--- + +Fix Vercel AI Gateway model fetching: speech/transcription models (e.g. fish-audio, openai/tts, openai/whisper) omit `context_window`/`max_tokens`, which failed the whole `/models` schema validation and dropped all language models. The schema now treats those fields as optional and the non-language filter handles exclusion. diff --git a/AGENTS.md b/AGENTS.md index 67d352f0131..18e51c0feb5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,21 @@ This file provides guidance to agents when working with code in this repository. +## Repository Background: Roo Code → Roo Code Continue + +This repository is a **fork of [Roo Code](https://github.com/RooVetGit/Roo-Code)**, whose original upstream repository has been **archived and is no longer maintained**. In order to keep publishing to the VS Code Marketplace under this fork, the extension was renamed: + +- **Extension identity** (`package.json` `name`): `roo-code-continue` (Nightly: `roo-code-continue-nightly`) +- **Display name** (via `package.nls.*.json` → `extension.displayName`): **"Roo Code Continue"** (Nightly: "Roo Code Continue Nightly") + +When making changes, keep the following naming rules in mind: + +- **User-facing / Marketplace-facing names** (display name, command titles, activity bar titles, README, i18n strings) should reference the new "Roo Code Continue" branding. Do NOT revert these to "Roo Code". +- **Internal identifiers that affect settings/import compatibility MUST keep the original `roo-cline` prefix.** This includes: VS Code `configuration` setting keys (e.g. `roo-cline.allowedCommands`, `roo-cline.customStoragePath`, `roo-cline.apiRequestTimeout`), commands (`roo-cline.plusButtonClicked`, ...), view/webview IDs (`roo-cline.SidebarProvider`), submenus, and keybindings. Renaming these would break existing user settings, keybindings, and the ability to import settings from the original Roo Code — so they must stay exactly as-is. +- The Nightly variant (in `apps/vscode-nightly/`) is generated at build time via `generatePackageJson()` which substitutes `roo-cline → roo-code-continue-nightly` for `contributes` keys only; settings `properties` keep the `roo-cline.` prefix for the same compatibility reason. +- `src/utils/migrateSettings.ts` handles legacy file-name migration (`cline_custom_modes.json`, `cline_mcp_settings.json`) and default-command cleanup; do not change the migrated file names. +- The VS Code Marketplace requires a **globally unique display name**. Before introducing a new display name, verify it is not already taken, otherwise `vsce publish` fails with "This extension display name is taken". + - Settings View Pattern: When working on `SettingsView`, inputs must bind to the local `cachedState`, NOT the live `useExtensionState()`. The `cachedState` acts as a buffer for user edits, isolating them from the `ContextProxy` source-of-truth until the user explicitly clicks "Save". Wiring inputs directly to the live state causes race conditions. ## Version Management & Release Workflow diff --git a/apps/vscode-nightly/package.nls.nightly.json b/apps/vscode-nightly/package.nls.nightly.json index b76bbf4eb8e..42c3e8bebfb 100644 --- a/apps/vscode-nightly/package.nls.nightly.json +++ b/apps/vscode-nightly/package.nls.nightly.json @@ -1,7 +1,7 @@ { - "extension.displayName": "Roo Code Nightly", - "views.contextMenu.label": "Roo Code Nightly", - "views.terminalMenu.label": "Roo Code Nightly", - "views.activitybar.title": "Roo Code Nightly", - "configuration.title": "Roo Code Nightly" + "extension.displayName": "Roo Code Continue Nightly", + "views.contextMenu.label": "Roo Code Continue Nightly", + "views.terminalMenu.label": "Roo Code Continue Nightly", + "views.activitybar.title": "Roo Code Continue Nightly", + "configuration.title": "Roo Code Continue Nightly" } diff --git a/src/api/providers/fetchers/__tests__/unbound.spec.ts b/src/api/providers/fetchers/__tests__/unbound.spec.ts new file mode 100644 index 00000000000..8a4121d0c12 --- /dev/null +++ b/src/api/providers/fetchers/__tests__/unbound.spec.ts @@ -0,0 +1,160 @@ +import axios from "axios" + +import { getUnboundModels } from "../unbound" + +vitest.mock("axios") +const mockedAxios = axios as any + +describe("getUnboundModels", () => { + beforeEach(() => { + vitest.clearAllMocks() + }) + + it("parses a dictionary response (current API shape) with camelCase fields", async () => { + mockedAxios.get.mockResolvedValueOnce({ + data: { + "anthropic/claude-haiku-4-5": { + // The live API returns these as numeric strings. + maxTokens: "32000", + contextWindow: "200000", + supportsImages: true, + supportsPromptCaching: true, + inputTokenPrice: "1.000000", + outputTokenPrice: "5.000000", + cacheReadPrice: "0.500000", + cacheWritePrice: "6.250000", + description: "Claude Haiku 4.5", + }, + "openai/gpt-4o-mini": { + maxTokens: 16384, + contextWindow: 128000, + supportsImages: false, + supportsPromptCaching: false, + inputTokenPrice: "0.15", + outputTokenPrice: "0.60", + }, + }, + }) + + const models = await getUnboundModels("test-key") + + expect(mockedAxios.get).toHaveBeenCalledWith( + "https://api.getunbound.ai/models", + expect.objectContaining({ headers: { Authorization: "Bearer test-key" } }), + ) + + expect(Object.keys(models)).toEqual(["anthropic/claude-haiku-4-5", "openai/gpt-4o-mini"]) + + const haiku = models["anthropic/claude-haiku-4-5"] + expect(haiku.maxTokens).toBe(32000) + expect(haiku.contextWindow).toBe(200000) + expect(haiku.supportsImages).toBe(true) + expect(haiku.supportsPromptCache).toBe(true) + // parseApiPrice converts $/M tokens to micro-units + expect(haiku.inputPrice).toBe(1_000_000) + expect(haiku.outputPrice).toBe(5_000_000) + expect(haiku.cacheReadsPrice).toBe(500_000) + expect(haiku.cacheWritesPrice).toBe(6_250_000) + expect(haiku.description).toBe("Claude Haiku 4.5") + + const mini = models["openai/gpt-4o-mini"] + expect(mini.supportsImages).toBe(false) + expect(mini.supportsPromptCache).toBe(false) + expect(mini.inputPrice).toBe(150_000) + expect(mini.outputPrice).toBe(600_000) + }) + + it("parses a legacy array response with snake_case fields", async () => { + mockedAxios.get.mockResolvedValueOnce({ + data: [ + { + id: "anthropic/claude-sonnet-4", + max_output_tokens: 64000, + context_window: 200000, + supports_vision: true, + supports_caching: true, + input_price: "3.00", + output_price: "15.00", + caching_price: "3.75", + cached_price: "0.30", + description: "Claude Sonnet 4", + }, + ], + }) + + const models = await getUnboundModels() + + expect(Object.keys(models)).toEqual(["anthropic/claude-sonnet-4"]) + + const model = models["anthropic/claude-sonnet-4"] + expect(model.maxTokens).toBe(64000) + expect(model.contextWindow).toBe(200000) + expect(model.supportsImages).toBe(true) + expect(model.supportsPromptCache).toBe(true) + expect(model.inputPrice).toBe(3_000_000) + expect(model.outputPrice).toBe(15_000_000) + expect(model.cacheWritesPrice).toBe(3_750_000) + expect(model.cacheReadsPrice).toBe(300_000) + }) + + it("handles a response wrapped in a data property", async () => { + mockedAxios.get.mockResolvedValueOnce({ + data: { + data: { + "test/model": { + maxTokens: 8192, + contextWindow: 200000, + }, + }, + }, + }) + + const models = await getUnboundModels() + + expect(models["test/model"]).toBeDefined() + expect(models["test/model"].maxTokens).toBe(8192) + }) + + it("applies defaults when fields are missing", async () => { + mockedAxios.get.mockResolvedValueOnce({ + data: { + "test/model": { + // No fields provided + }, + }, + }) + + const models = await getUnboundModels() + + expect(models["test/model"].maxTokens).toBe(8192) + expect(models["test/model"].contextWindow).toBe(200_000) + expect(models["test/model"].supportsPromptCache).toBe(false) + expect(models["test/model"].supportsImages).toBe(false) + expect(models["test/model"].inputPrice).toBeUndefined() + expect(models["test/model"].description).toBeUndefined() + }) + + it("returns empty object and logs an error when response is not an object or array", async () => { + const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(() => {}) + mockedAxios.get.mockResolvedValueOnce({ data: "not-an-object" }) + + const models = await getUnboundModels() + + expect(models).toEqual({}) + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining("response did not contain a models object or array"), + ) + consoleErrorSpy.mockRestore() + }) + + it("handles API errors gracefully", async () => { + const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(() => {}) + mockedAxios.get.mockRejectedValueOnce(new Error("Network error")) + + const models = await getUnboundModels() + + expect(models).toEqual({}) + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("Error fetching Unbound models")) + consoleErrorSpy.mockRestore() + }) +}) diff --git a/src/api/providers/fetchers/__tests__/vercel-ai-gateway.spec.ts b/src/api/providers/fetchers/__tests__/vercel-ai-gateway.spec.ts index 3a4a234de96..00ffdf53810 100644 --- a/src/api/providers/fetchers/__tests__/vercel-ai-gateway.spec.ts +++ b/src/api/providers/fetchers/__tests__/vercel-ai-gateway.spec.ts @@ -112,6 +112,52 @@ describe("Vercel AI Gateway Fetchers", () => { consoleErrorSpy.mockRestore() }) + it("does not fail the whole list when speech/transcription models omit context_window/max_tokens", async () => { + const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(() => {}) + // Mirrors the real /models response: fish-audio/s1 is index 107 in + // the live API and has no context_window/max_tokens fields. + const response = { + data: { + object: "list", + data: [ + { + id: "anthropic/claude-sonnet-4", + object: "model", + created: 1640995200, + owned_by: "anthropic", + name: "Claude Sonnet 4", + description: "Claude Sonnet 4", + context_window: 200000, + max_tokens: 64000, + type: "language", + pricing: { input: "3.00", output: "15.00" }, + }, + { + id: "fish-audio/s1", + object: "model", + created: 1755815280, + owned_by: "fish-audio", + name: "S1", + description: "Fish Audio S1 TTS", + type: "speech", + pricing: { input: "0.000015", speech_input_character_cost: "0.000015" }, + }, + ], + }, + } + mockedAxios.get.mockResolvedValueOnce(response) + + const models = await getVercelAiGatewayModels() + + // No schema error should be logged - the list validates fine. + expect(consoleErrorSpy).not.toHaveBeenCalled() + + // The language model is still parsed; the speech model is filtered out. + expect(models["anthropic/claude-sonnet-4"]).toBeDefined() + expect(models["fish-audio/s1"]).toBeUndefined() + consoleErrorSpy.mockRestore() + }) + it("continues processing with partially valid schema", async () => { const consoleErrorSpy = vitest.spyOn(console, "error").mockImplementation(() => {}) const invalidResponse = { diff --git a/src/api/providers/fetchers/unbound.ts b/src/api/providers/fetchers/unbound.ts index 38410118093..6e68d53fd0b 100644 --- a/src/api/providers/fetchers/unbound.ts +++ b/src/api/providers/fetchers/unbound.ts @@ -4,6 +4,81 @@ import type { ModelInfo } from "@roo-code/types" import { parseApiPrice } from "../../../shared/cost" +// The Unbound /models endpoint has historically returned different shapes: +// 1. An array of model objects (with snake_case fields like `max_output_tokens`) +// 2. A dictionary keyed by model id, with camelCase fields (e.g. `maxTokens`, +// `contextWindow`, `inputTokenPrice`) - this is what the API returns today. +// We normalize both shapes into a Record. +type UnboundRawModel = Record & { id?: string } + +function toEntries(rawModels: unknown): Array<[string, UnboundRawModel]> { + if (Array.isArray(rawModels)) { + const entries: Array<[string, UnboundRawModel]> = [] + + for (const item of rawModels) { + if (item && typeof item === "object") { + const rawModel = item as UnboundRawModel + const id = rawModel.id + if (typeof id === "string" && id.length > 0) { + entries.push([id, rawModel]) + } + } + } + + return entries + } + + if (rawModels && typeof rawModels === "object") { + return Object.entries(rawModels as Record).map(([id, value]) => [ + id, + (value && typeof value === "object" ? value : {}) as UnboundRawModel, + ]) + } + + return [] +} + +function pick(rawModel: UnboundRawModel, camelCase: string, snakeCase: string): unknown { + return rawModel[camelCase] ?? rawModel[snakeCase] +} + +function toFiniteNumber(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value + } + + if (typeof value === "string" && value.trim().length > 0) { + const parsed = Number(value) + return Number.isFinite(parsed) ? parsed : undefined + } + + return undefined +} + +function parseUnboundModel(id: string, rawModel: UnboundRawModel): ModelInfo { + const maxTokens = toFiniteNumber(pick(rawModel, "maxTokens", "max_output_tokens")) + const contextWindow = toFiniteNumber(pick(rawModel, "contextWindow", "context_window")) + const supportsPromptCache = pick(rawModel, "supportsPromptCaching", "supports_caching") + const supportsImages = pick(rawModel, "supportsImages", "supports_vision") + const inputTokenPrice = pick(rawModel, "inputTokenPrice", "input_price") + const outputTokenPrice = pick(rawModel, "outputTokenPrice", "output_price") + const cacheReadPrice = pick(rawModel, "cacheReadPrice", "cached_price") + const cacheWritePrice = pick(rawModel, "cacheWritePrice", "caching_price") + const description = pick(rawModel, "description", "description") + + return { + maxTokens: maxTokens ?? 8192, + contextWindow: contextWindow ?? 200_000, + supportsPromptCache: supportsPromptCache === true, + supportsImages: supportsImages === true, + inputPrice: parseApiPrice(inputTokenPrice), + outputPrice: parseApiPrice(outputTokenPrice), + cacheReadsPrice: parseApiPrice(cacheReadPrice), + cacheWritesPrice: parseApiPrice(cacheWritePrice), + ...(typeof description === "string" && description.length > 0 ? { description } : {}), + } +} + export async function getUnboundModels(apiKey?: string | null): Promise> { const models: Record = {} @@ -15,22 +90,15 @@ export async function getUnboundModels(apiKey?: string | null): Promise() private checkpointPossibleFiles = new Set() + // Serializes the read-modify-write cycle on the task metadata file so concurrent + // edits from multiple tools/watchers never interleave. Without this, concurrent + // calls lose updates AND each races safeWriteJson's proper-lockfile, flooding the + // log with "Lock file is already being held" and dropping metadata writes. + private _metadataLock: Promise = Promise.resolve() + private withMetadataLock(fn: () => Promise): Promise { + const result = this._metadataLock.then(fn, fn) + this._metadataLock = result.then( + () => {}, + () => {}, + ) + return result + } + constructor(provider: ClineProvider, taskId: string) { this.providerRef = new WeakRef(provider) this.taskId = taskId @@ -140,63 +154,67 @@ export class FileContextTracker { // Adds a file to the metadata tracker // This handles the business logic of determining if the file is new, stale, or active. // It also updates the metadata with the latest read/edit dates. + // The whole read-modify-write cycle is serialized through withMetadataLock so concurrent + // calls never read a stale snapshot or race the underlying file lock in saveTaskMetadata. async addFileToFileContextTracker(taskId: string, filePath: string, source: RecordSource) { - try { - const metadata = await this.getTaskMetadata(taskId) - const now = Date.now() - - // Mark existing entries for this file as stale - metadata.files_in_context.forEach((entry) => { - if (entry.path === filePath && entry.record_state === "active") { - entry.record_state = "stale" + return this.withMetadataLock(async () => { + try { + const metadata = await this.getTaskMetadata(taskId) + const now = Date.now() + + // Mark existing entries for this file as stale + metadata.files_in_context.forEach((entry) => { + if (entry.path === filePath && entry.record_state === "active") { + entry.record_state = "stale" + } + }) + + // Helper to get the latest date for a specific field and file + const getLatestDateForField = (path: string, field: keyof FileMetadataEntry): number | null => { + const relevantEntries = metadata.files_in_context + .filter((entry) => entry.path === path && entry[field]) + .sort((a, b) => (b[field] as number) - (a[field] as number)) + + return relevantEntries.length > 0 ? (relevantEntries[0][field] as number) : null } - }) - // Helper to get the latest date for a specific field and file - const getLatestDateForField = (path: string, field: keyof FileMetadataEntry): number | null => { - const relevantEntries = metadata.files_in_context - .filter((entry) => entry.path === path && entry[field]) - .sort((a, b) => (b[field] as number) - (a[field] as number)) - - return relevantEntries.length > 0 ? (relevantEntries[0][field] as number) : null - } + let newEntry: FileMetadataEntry = { + path: filePath, + record_state: "active", + record_source: source, + roo_read_date: getLatestDateForField(filePath, "roo_read_date"), + roo_edit_date: getLatestDateForField(filePath, "roo_edit_date"), + user_edit_date: getLatestDateForField(filePath, "user_edit_date"), + } - let newEntry: FileMetadataEntry = { - path: filePath, - record_state: "active", - record_source: source, - roo_read_date: getLatestDateForField(filePath, "roo_read_date"), - roo_edit_date: getLatestDateForField(filePath, "roo_edit_date"), - user_edit_date: getLatestDateForField(filePath, "user_edit_date"), - } + switch (source) { + // user_edited: The user has edited the file + case "user_edited": + newEntry.user_edit_date = now + this.recentlyModifiedFiles.add(filePath) + break + + // roo_edited: Roo has edited the file + case "roo_edited": + newEntry.roo_read_date = now + newEntry.roo_edit_date = now + this.checkpointPossibleFiles.add(filePath) + this.markFileAsEditedByRoo(filePath) + break + + // read_tool/file_mentioned: Roo has read the file via a tool or file mention + case "read_tool": + case "file_mentioned": + newEntry.roo_read_date = now + break + } - switch (source) { - // user_edited: The user has edited the file - case "user_edited": - newEntry.user_edit_date = now - this.recentlyModifiedFiles.add(filePath) - break - - // roo_edited: Roo has edited the file - case "roo_edited": - newEntry.roo_read_date = now - newEntry.roo_edit_date = now - this.checkpointPossibleFiles.add(filePath) - this.markFileAsEditedByRoo(filePath) - break - - // read_tool/file_mentioned: Roo has read the file via a tool or file mention - case "read_tool": - case "file_mentioned": - newEntry.roo_read_date = now - break + metadata.files_in_context.push(newEntry) + await this.saveTaskMetadata(taskId, metadata) + } catch (error) { + console.error("Failed to add file to metadata:", error) } - - metadata.files_in_context.push(newEntry) - await this.saveTaskMetadata(taskId, metadata) - } catch (error) { - console.error("Failed to add file to metadata:", error) - } + }) } // Returns (and then clears) the set of recently modified files diff --git a/src/core/context-tracking/__tests__/FileContextTracker.spec.ts b/src/core/context-tracking/__tests__/FileContextTracker.spec.ts new file mode 100644 index 00000000000..3bcab06c230 --- /dev/null +++ b/src/core/context-tracking/__tests__/FileContextTracker.spec.ts @@ -0,0 +1,120 @@ +// npx vitest run src/core/context-tracking/__tests__/FileContextTracker.spec.ts + +import { FileContextTracker } from "../FileContextTracker" +import type { TaskMetadata } from "../FileContextTrackerTypes" + +vi.mock("vscode", () => ({ + Uri: { file: vi.fn((p: string) => ({ fsPath: p })) }, + RelativePattern: vi.fn(), + EventEmitter: vi.fn(() => ({ event: vi.fn(), fire: vi.fn(), dispose: vi.fn() })), + workspace: { + createFileSystemWatcher: vi.fn(), + workspaceFolders: [{ uri: { fsPath: "/mock/workspace" } }], + }, +})) + +vi.mock("../../webview/ClineProvider", () => ({ ClineProvider: vi.fn() })) +vi.mock("../../../utils/safeWriteJson", () => ({ safeWriteJson: vi.fn().mockResolvedValue(undefined) })) + +describe("FileContextTracker", () => { + const mockProvider = { + contextProxy: { globalStorageUri: { fsPath: "/mock/global-storage" } }, + } as any + + beforeEach(() => { + vi.clearAllMocks() + }) + + it("adds a file entry and persists it via saveTaskMetadata", async () => { + const tracker = new FileContextTracker(mockProvider, "task-1") + + const baseMetadata: TaskMetadata = { files_in_context: [] } + vi.spyOn(tracker, "getTaskMetadata").mockResolvedValue(baseMetadata) + const saveSpy = vi.spyOn(tracker, "saveTaskMetadata").mockResolvedValue(undefined) + + await tracker.addFileToFileContextTracker("task-1", "src/a.ts", "read_tool") + + expect(saveSpy).toHaveBeenCalledTimes(1) + const saved = saveSpy.mock.calls[0][1] as TaskMetadata + expect(saved.files_in_context).toHaveLength(1) + expect(saved.files_in_context[0].path).toBe("src/a.ts") + expect(saved.files_in_context[0].record_source).toBe("read_tool") + expect(saved.files_in_context[0].record_state).toBe("active") + }) + + it("serializes concurrent updates so no entries are lost (no interleaved read-modify-write)", async () => { + const tracker = new FileContextTracker(mockProvider, "task-1") + + // Simulate the on-disk state: only updated once a save actually completes. + let diskMetadata: TaskMetadata = { files_in_context: [] } + const gateResolvers: Array<() => void> = [] + const gate = () => new Promise((resolve) => gateResolvers.push(resolve)) + + const saveSpy = vi.spyOn(tracker, "saveTaskMetadata").mockImplementation(async (_taskId, metadata) => { + await gate() // Slow write: hold each save open so callers overlap + diskMetadata = metadata + }) + vi.spyOn(tracker, "getTaskMetadata").mockImplementation(async () => diskMetadata) + + // Fire three concurrent updates (e.g. multiple tools/watchers touching files at once) + const p1 = tracker.addFileToFileContextTracker("task-1", "src/a.ts", "read_tool") + const p2 = tracker.addFileToFileContextTracker("task-1", "src/b.ts", "user_edited") + const p3 = tracker.addFileToFileContextTracker("task-1", "src/c.ts", "roo_edited") + + // Release each save one at a time; with the lock, each subsequent call must + // observe the state persisted by the previous one. + for (let i = 0; i < 3; i++) { + await vi.waitFor(() => { + expect(gateResolvers.length).toBeGreaterThanOrEqual(i + 1) + }) + gateResolvers[i]() + } + + await Promise.all([p1, p2, p3]) + + // Every update must have been persisted in order, with none lost. + expect(saveSpy).toHaveBeenCalledTimes(3) + expect(diskMetadata.files_in_context).toHaveLength(3) + const paths = diskMetadata.files_in_context.map((entry) => entry.path) + expect(paths).toEqual(["src/a.ts", "src/b.ts", "src/c.ts"]) + }) + + it("continues the lock chain when a save fails so later updates still persist", async () => { + const tracker = new FileContextTracker(mockProvider, "task-1") + + let diskMetadata: TaskMetadata = { files_in_context: [] } + const gateResolvers: Array<() => void> = [] + const gate = () => new Promise((resolve) => gateResolvers.push(resolve)) + + const saveSpy = vi.spyOn(tracker, "saveTaskMetadata").mockImplementation(async (_taskId, metadata) => { + await gate() + diskMetadata = metadata + }) + // Mirror real behaviour: getTaskMetadata re-reads the persisted state from disk, + // so un-persisted in-memory mutations from a failed save are NOT visible. + vi.spyOn(tracker, "getTaskMetadata").mockImplementation(async () => ({ + files_in_context: [...diskMetadata.files_in_context], + })) + + // The first save fails (e.g. a transient write error); the second must still run. + saveSpy.mockRejectedValueOnce(new Error("Lock file is already being held")) + + const p1 = tracker.addFileToFileContextTracker("task-1", "src/a.ts", "read_tool") + const p2 = tracker.addFileToFileContextTracker("task-1", "src/b.ts", "read_tool") + + // p1's save rejects immediately; p2 is queued behind the lock and runs afterwards. + await vi.waitFor(() => { + expect(saveSpy).toHaveBeenCalledTimes(2) + }) + expect(gateResolvers).toHaveLength(1) + + gateResolvers[0]() + + await Promise.all([p1, p2]) + + // The chain was not broken by p1's failure: p2 still read fresh state and persisted + // its own update. (The failed write's entry is intentionally absent from disk.) + expect(saveSpy).toHaveBeenCalledTimes(2) + expect(diskMetadata.files_in_context.map((e) => e.path)).toEqual(["src/b.ts"]) + }) +}) diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index 775f7d9dbf3..22c715254dd 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -3,6 +3,7 @@ import * as vscode from "vscode" import { FileWatcher } from "../file-watcher" +import { codeParser } from "../parser" // Mock dependencies vi.mock("../../cache-manager") @@ -274,6 +275,109 @@ describe("FileWatcher", () => { }) }) + describe("embedding batching", () => { + it("should split a single file's blocks into batches matching the configured embedding batch size", async () => { + // Create a FileWatcher with an explicit small batch size (e.g. user config sets 5) + const batchedWatcher = new FileWatcher( + "/mock/workspace", + mockContext, + mockCacheManager, + mockEmbedder, + mockVectorStore, + mockIgnoreInstance, + undefined, + 5, + ) + + // Mock the parser to return 12 code blocks + const mockBlocks = Array.from({ length: 12 }, (_, index) => ({ + file_path: `/mock/workspace/src/file.ts`, + content: `block ${index}`, + start_line: index * 10 + 1, + end_line: index * 10 + 10, + })) + vi.mocked(codeParser.parseFile).mockResolvedValue(mockBlocks as any) + + // Embedder returns a vector derived from the text so we can verify + // embeddings are correctly spliced back together across batches + mockEmbedder.createEmbeddings.mockImplementation(async (texts: string[]) => ({ + embeddings: texts.map((text) => { + const blockIndex = Number(text.split(" ")[1]) + return [blockIndex, 0.5, 0.25] + }), + })) + + // Simulate no prior hash so the file is not skipped + mockCacheManager.getHash.mockReturnValue(undefined) + + const result = await batchedWatcher.processFile("/mock/workspace/src/file.ts") + + expect(result.status).toBe("processed_for_batching") + // 12 blocks / batch size 5 => 3 calls (5, 5, 2) + expect(mockEmbedder.createEmbeddings).toHaveBeenCalledTimes(3) + expect(mockEmbedder.createEmbeddings).toHaveBeenNthCalledWith(1, [ + "block 0", + "block 1", + "block 2", + "block 3", + "block 4", + ]) + expect(mockEmbedder.createEmbeddings).toHaveBeenNthCalledWith(2, [ + "block 5", + "block 6", + "block 7", + "block 8", + "block 9", + ]) + expect(mockEmbedder.createEmbeddings).toHaveBeenNthCalledWith(3, ["block 10", "block 11"]) + + // All 12 points should be produced with the correct embeddings + expect(result.pointsToUpsert).toHaveLength(12) + expect(result.pointsToUpsert![0].vector).toEqual([0, 0.5, 0.25]) + expect(result.pointsToUpsert![11].vector).toEqual([11, 0.5, 0.25]) + }) + + it("should make a single embedding call when blocks fit within the batch size", async () => { + const mockBlocks = [ + { + file_path: "/mock/workspace/src/file.ts", + content: "block a", + start_line: 1, + end_line: 10, + }, + { + file_path: "/mock/workspace/src/file.ts", + content: "block b", + start_line: 11, + end_line: 20, + }, + ] + vi.mocked(codeParser.parseFile).mockResolvedValue(mockBlocks as any) + mockEmbedder.createEmbeddings.mockImplementation(async (texts: string[]) => ({ + embeddings: texts.map((_, i) => [i, 0.1]), + })) + mockCacheManager.getHash.mockReturnValue(undefined) + + const result = await fileWatcher.processFile("/mock/workspace/src/file.ts") + + expect(result.status).toBe("processed_for_batching") + expect(mockEmbedder.createEmbeddings).toHaveBeenCalledTimes(1) + expect(mockEmbedder.createEmbeddings).toHaveBeenCalledWith(["block a", "block b"]) + expect(result.pointsToUpsert).toHaveLength(2) + }) + + it("should not call the embedder when the file has no parseable blocks", async () => { + vi.mocked(codeParser.parseFile).mockResolvedValue([]) + mockCacheManager.getHash.mockReturnValue(undefined) + + const result = await fileWatcher.processFile("/mock/workspace/src/file.ts") + + expect(result.status).toBe("processed_for_batching") + expect(mockEmbedder.createEmbeddings).not.toHaveBeenCalled() + expect(result.pointsToUpsert).toHaveLength(0) + }) + }) + describe("dispose", () => { it("should dispose of the watcher when disposed", async () => { await fileWatcher.initialize() diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index 81483648258..cfc4d57920c 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -468,8 +468,17 @@ export class FileWatcher implements IFileWatcher { // Prepare points for batch processing let pointsToUpsert: PointStruct[] = [] if (this.embedder && blocks.length > 0) { - const texts = blocks.map((block) => block.content) - const { embeddings } = await this.embedder.createEmbeddings(texts) + // Split blocks into batches respecting the configured embedding batch size + // so a single file never sends more items than the embedder allows per request + // (e.g. SCNet gateway rejects batches larger than 20/25 items with a 400 error) + const batchSize = Math.max(1, this.batchSegmentThreshold) + const embeddings: number[][] = [] + for (let i = 0; i < blocks.length; i += batchSize) { + const batchBlocks = blocks.slice(i, i + batchSize) + const batchTexts = batchBlocks.map((block) => block.content) + const { embeddings: batchEmbeddings } = await this.embedder.createEmbeddings(batchTexts) + embeddings.push(...batchEmbeddings) + } pointsToUpsert = blocks.map((block, index) => { const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath) diff --git a/src/vitest.config.ts b/src/vitest.config.ts index 3a6854d5328..51a540346ef 100644 --- a/src/vitest.config.ts +++ b/src/vitest.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ testTimeout: 20_000, hookTimeout: 20_000, onConsoleLog, + pool: "forks", }, resolve: { alias: { diff --git a/webview-ui/src/utils/__tests__/sourceMapInitializer.spec.ts b/webview-ui/src/utils/__tests__/sourceMapInitializer.spec.ts index affef21a468..6b2d7d13d46 100644 --- a/webview-ui/src/utils/__tests__/sourceMapInitializer.spec.ts +++ b/webview-ui/src/utils/__tests__/sourceMapInitializer.spec.ts @@ -10,11 +10,111 @@ vi.mock("../vscode", () => { } }) -import { reportWebviewError } from "../sourceMapInitializer" +import { initializeSourceMaps, reportWebviewError } from "../sourceMapInitializer" import { vscode } from "../vscode" const mockPostMessage = vi.mocked(vscode.postMessage) +describe("initializeSourceMaps preloading", () => { + const originalNodeEnv = process.env.NODE_ENV + const originalFetch = globalThis.fetch + const originalCreateElement = document.createElement.bind(document) + + beforeEach(() => { + process.env.NODE_ENV = "production" + document.head.innerHTML = "" + document.body.innerHTML = "" + vi.restoreAllMocks() + }) + + afterEach(() => { + process.env.NODE_ENV = originalNodeEnv + globalThis.fetch = originalFetch + document.createElement = originalCreateElement + }) + + it("preloads only the map referenced by sourceMappingURL", async () => { + const scriptSrc = "vscode-webview://resource/assets/index-abc.js" + const mapUrl = "vscode-webview://resource/assets/index-abc.js.map" + + const script = document.createElement("script") + script.src = scriptSrc + document.body.appendChild(script) + + // Fetch 1 = script content (contains sourceMappingURL), fetch 2 = the map probe. + globalThis.fetch = vi + .fn() + .mockResolvedValueOnce({ text: async () => "//# sourceMappingURL=index-abc.js.map" } as Response) + .mockResolvedValueOnce({ ok: true } as Response) + + initializeSourceMaps() + + // Let the promise chain resolve. + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(globalThis.fetch).toHaveBeenCalledTimes(2) + expect(globalThis.fetch).toHaveBeenLastCalledWith(mapUrl) + + const preloads = document.head.querySelectorAll('link[rel="preload"]') + expect(preloads).toHaveLength(1) + expect(preloads[0].getAttribute("href")).toBe(mapUrl) + }) + + it("does not preload a source map that is missing (404)", async () => { + const scriptSrc = "vscode-webview://resource/assets/index-abc.js" + + const script = document.createElement("script") + script.src = scriptSrc + document.body.appendChild(script) + + globalThis.fetch = vi + .fn() + .mockResolvedValueOnce({ text: async () => "//# sourceMappingURL=index-abc.js.map" } as Response) + .mockResolvedValueOnce({ ok: false, status: 404 } as Response) + + initializeSourceMaps() + + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(document.head.querySelectorAll('link[rel="preload"]')).toHaveLength(0) + }) + + it("does nothing when a script has no sourceMappingURL comment", async () => { + const script = document.createElement("script") + script.src = "vscode-webview://resource/assets/vendor.js" + document.body.appendChild(script) + + globalThis.fetch = vi + .fn() + .mockResolvedValueOnce({ text: async () => "console.log('no source map')" } as Response) + + initializeSourceMaps() + + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(globalThis.fetch).toHaveBeenCalledTimes(1) + expect(document.head.querySelectorAll('link[rel="preload"]')).toHaveLength(0) + }) + + it("skips scripts without a src", async () => { + // Only append a script element that is not yet connected to the DOM, + // so jsdom does not try to execute it as an inline script. + const script = document.createElement("script") + + const fetchSpy = vi.fn() + globalThis.fetch = fetchSpy + + initializeSourceMaps() + + await new Promise((resolve) => setTimeout(resolve, 0)) + + expect(fetchSpy).not.toHaveBeenCalled() + + // The detached script element is still collectable but never runs. + expect(script.src).toBe("") + }) +}) + describe("reportWebviewError", () => { it("posts a webviewError message with Error details", () => { const error = new Error("boom") diff --git a/webview-ui/src/utils/sourceMapInitializer.ts b/webview-ui/src/utils/sourceMapInitializer.ts index eb88fb01774..c695e267849 100644 --- a/webview-ui/src/utils/sourceMapInitializer.ts +++ b/webview-ui/src/utils/sourceMapInitializer.ts @@ -93,56 +93,57 @@ export function initializeSourceMaps(): void { } }) - // Preload source maps for all scripts + // Preload source maps for all scripts. + // We own the build pipeline (see webview-ui/vite.config.ts and the + // sourcemapPlugin), which appends a sourceMappingURL comment to every + // generated chunk. Reading that comment is the single source of truth for + // the map's real file name, so we do NOT guess candidate names (which + // produced spurious 404s in the webview logs). try { const scripts = document.getElementsByTagName("script") for (let i = 0; i < scripts.length; i++) { const script = scripts[i] - if (script.src) { - // Try multiple source map locations - const possibleMapUrls = [ - `${script.src}.map`, - `${script.src}?source-map=true`, - script.src.replace(/\.js$/, ".js.map"), - script.src.replace(/\.js$/, ".map.json"), - script.src.replace(/\.js$/, ".sourcemap"), - ] - - // Preload all possible source map locations - for (const mapUrl of possibleMapUrls) { - const link = document.createElement("link") - link.rel = "preload" - link.as = "fetch" - link.href = mapUrl - link.crossOrigin = "anonymous" - document.head.appendChild(link) - } + if (!script.src) { + continue + } - // Also check for inline sourceMappingURL comments - fetch(script.src) - .then((response) => response.text()) - .then((content) => { - const sourceMappingURLMatch = content.match(/\/\/[#@]\s*sourceMappingURL=([^\s]+)/) - if (sourceMappingURLMatch && sourceMappingURLMatch[1]) { - const sourceMappingURL = sourceMappingURLMatch[1] - - // If it's not a data: URL, preload it - if (!sourceMappingURL.startsWith("data:")) { - const scriptUrlObj = new URL(script.src) - const baseUrl = scriptUrlObj.href.substring(0, scriptUrlObj.href.lastIndexOf("/") + 1) - const fullUrl = new URL(sourceMappingURL, baseUrl).href - - const link = document.createElement("link") - link.rel = "preload" - link.as = "fetch" - link.href = fullUrl - link.crossOrigin = "anonymous" - document.head.appendChild(link) + // Resolve the map URL from the inline sourceMappingURL comment. + fetch(script.src) + .then((response) => response.text()) + .then((content) => { + const sourceMappingURLMatch = content.match(/\/\/[#@]\s*sourceMappingURL=([^\s]+)/) + if (!sourceMappingURLMatch || !sourceMappingURLMatch[1]) { + return + } + + const sourceMappingURL = sourceMappingURLMatch[1] + + // Inline data: maps need no preload. + if (sourceMappingURL.startsWith("data:")) { + return + } + + const scriptUrlObj = new URL(script.src) + const baseUrl = scriptUrlObj.href.substring(0, scriptUrlObj.href.lastIndexOf("/") + 1) + const fullUrl = new URL(sourceMappingURL, baseUrl).href + + // Probe first so we only preload maps that actually exist. + return fetch(fullUrl) + .then((mapResponse) => { + if (!mapResponse.ok) { + console.debug(`Source map not found (skipping preload): ${fullUrl}`) + return } - } - }) - .catch((e) => console.debug("Error checking for inline sourceMappingURL:", e)) - } + const link = document.createElement("link") + link.rel = "preload" + link.as = "fetch" + link.href = fullUrl + link.crossOrigin = "anonymous" + document.head.appendChild(link) + }) + .catch((e) => console.debug("Error probing source map:", e)) + }) + .catch((e) => console.debug("Error checking for inline sourceMappingURL:", e)) } } catch (e) { console.error("Error preloading source maps:", e)