Unwedge the code source: a URL in a template literal, an 8192-token chunk, and an unbounded retry

src/__tests__/admin-ops-endpoints.test.ts

@@ -397,6 +397,11 @@ describe("admin ops control surface", () => {
         // that would write zero rows is now visible BEFORE it runs.
         next_acquire: "full",
         next_acquire_reason: "no-stored-config-fingerprint",
+        // No item has been quarantined, so nothing is missing from the index.
+        // The key is present regardless: an operator reading this shape must
+        // not have to guess whether an empty list means "none" or "not
+        // reported".
+        quarantined_items: [],
       },
     ]);
   });

src/__tests__/code-chunker-oversize.test.ts

@@ -0,0 +1,135 @@
+import { describe, it, expect } from "vitest";
+import { chunkCode } from "../indexing/chunking/code.js";
+import type { SourceConfig } from "../types.js";
+
+// Regression for the ten-day `code`-source wedge on mcp.copilotkit.ai.
+//
+// The named failing item was CopilotKit's
+// packages/web-inspector/dev/threads-state-lab.ts. Production logged:
+//
+//   [pipeline:code] Failed to index packages/web-inspector/dev/threads-state-lab.ts:
+//     BadRequestError: 400 Invalid 'input[2]': maximum input length is 8192 tokens.
+//
+// `input[2]` — the THIRD chunk — was 35,964 characters covering lines 159-1121
+// of a 1121-line file, because the chunker's block-state tracker got stuck.
+// The trigger is line 262:
+//
+//   wsUrl: `ws://127.0.0.1:5177/inspector-lab-runtime/${key}/realtime`,
+//
+// `stripStringsAndLineComments` tracked ' and " but NOT ` , so the `//` inside
+// the template literal read as a line comment. The rest of the line (including
+// the CLOSING backtick) was discarded, leaving an ODD backtick count, so the
+// tracker latched `inTemplateString = true` and never cleared it. Every
+// subsequent blank line was rejected as a split point, so the remaining ~960
+// lines collapsed into one chunk that blew past the embedding model's token
+// limit — which failed the item, held the source's state token, and froze the
+// index.
+//
+// Two invariants are pinned here:
+//   1. a `//` (or `/*`) inside a template literal must not latch the tracker;
+//   2. no chunk may exceed the hard size backstop, whatever the source looks
+//      like — a chunker that cannot find split points must still bound output.
+
+const CODE_CONFIG = {
+  name: "code",
+  type: "code",
+  chunk: { target_lines: 80, overlap_lines: 10 },
+} as unknown as SourceConfig;
+
+/** Reproduce the production file's shape: a URL-in-template-literal, then a
+ *  long tail of ordinary blank-line-separated declarations. */
+function buildFileWithUrlInTemplateLiteral(tailBlocks: number): string {
+  const head = [
+    "const ENABLE_URL =",
+    '  "https://intelligence.copilotkit.ai/intelligence/enable";',
+    "",
+    "function runtimeInfo(key: string) {",
+    "  return {",
+    "    wsUrl: `ws://127.0.0.1:5177/inspector-lab-runtime/${key}/realtime`,",
+    "  };",
+    "}",
+    "",
+  ];
+  const tail: string[] = [];
+  for (let i = 0; i < tailBlocks; i++) {
+    tail.push(
+      `export const SCENARIO_${i} = {`,
+      `  id: "scenario-${i}",`,
+      `  label: "Scenario number ${i} with a reasonably long descriptive label",`,
+      `  description: "Fixture payload ${i} used by the threads state lab dev harness",`,
+      "};",
+      "",
+    );
+  }
+  return head.concat(tail).join("\n");
+}
+
+describe("code chunker: template literals and the hard size backstop", () => {
+  it("does not treat `//` inside a template literal as a line comment", () => {
+    // 200 tail blocks = 1200 lines. With the bug, EVERY line after the wsUrl
+    // line is swallowed into a single chunk because the tracker believes it is
+    // still inside a template literal, so no blank line qualifies as a split
+    // point.
+    const content = buildFileWithUrlInTemplateLiteral(200);
+    const chunks = chunkCode(
+      content,
+      "packages/web-inspector/dev/threads-state-lab.ts",
+      CODE_CONFIG,
+    );
+
+    // ~1200 lines at target_lines 80 must produce many chunks, not two.
+    expect(chunks.length).toBeGreaterThan(10);
+
+    // And no single chunk may swallow the whole tail.
+    const largest = Math.max(...chunks.map((c) => c.content.length));
+    expect(largest).toBeLessThan(15_000);
+  });
+
+  it("bounds chunk size even when the source offers NO split points", () => {
+    // A file with no blank lines at all: the mechanical fallback splits on
+    // target_lines, but nothing bounds the CHARACTER size of those lines. A
+    // single 80-line range of long lines still overflows the embedding model.
+    const longLine = "  ".concat("const x = ", '"'.padEnd(600, "y"), '";');
+    const content = Array.from({ length: 400 }, () => longLine).join("\n");
+    const chunks = chunkCode(
+      content,
+      "packages/dense/no-blank-lines.ts",
+      CODE_CONFIG,
+    );
+
+    for (const chunk of chunks) {
+      expect(chunk.content.length).toBeLessThanOrEqual(12_000);
+    }
+  });
+
+  it("still splits an ordinary file on blank-line boundaries", () => {
+    // Guard against "fixed the latch by never entering template state at all".
+    const content = buildFileWithUrlInTemplateLiteral(40);
+    const chunks = chunkCode(content, "packages/a/b.ts", CODE_CONFIG);
+    expect(chunks.length).toBeGreaterThan(1);
+    // Chunks stay line-addressable and in order.
+    for (let i = 1; i < chunks.length; i++) {
+      expect(chunks[i].startLine ?? 0).toBeGreaterThan(
+        chunks[i - 1].startLine ?? 0,
+      );
+    }
+  });
+
+  it("keeps a genuine multi-line template literal intact as one region", () => {
+    // The latch exists for a reason: a blank line INSIDE a template literal is
+    // not a safe split point. Fixing the `//` case must not lose that.
+    const content = [
+      "const q = `",
+      "SELECT 1",
+      "",
+      "FROM t",
+      "`;",
+      "",
+      "const after = 1;",
+      "",
+    ].join("\n");
+    const chunks = chunkCode(content, "a.ts", CODE_CONFIG);
+    // Short file — one chunk, and crucially no crash / no latch leaking out.
+    expect(chunks).toHaveLength(1);
+  });
+});

src/__tests__/embeddings-oversize-input.test.ts

@@ -0,0 +1,162 @@
+import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+
+// Regression for the ten-day `code`-source wedge on mcp.copilotkit.ai.
+//
+// OpenAIEmbeddingProvider guarded oversized inputs with a CHARACTER cap:
+//
+//   const MAX_CHARS = 30_000;   // "~8192 tokens with safety margin"
+//
+// That is a unit error. 30,000 characters is only ~8192 tokens at ≥3.66
+// chars/token; dense source code tokenizes far denser than prose, so a 30,000
+// character code chunk is comfortably over 10,000 tokens. Production hit it:
+//
+//   BadRequestError: 400 Invalid 'input[2]': maximum input length is 8192 tokens.
+//
+// A 400 is not retryable, so the item failed, the source's state token was
+// held, and the whole `code` source froze for ten days.
+//
+// The invariant: an input the API rejects purely for LENGTH must be recovered
+// from in-provider by shrinking and retrying, not surfaced as a hard failure.
+// No fixed character cap can be correct for every tokenizer, so the provider
+// has to converge on the real limit empirically.
+//
+// The fake below is the REAL OpenAI SDK talking to a REAL HTTP server that
+// returns production's exact 400 shape. Its limit is expressed in characters
+// (a stand-in for the tokenizer) and deliberately set BELOW MAX_CHARS, which
+// is precisely the condition the char cap cannot see.
+
+const SERVER_CHAR_LIMIT = 4_000;
+
+interface EmbeddingsRequest {
+  input: string[];
+  model: string;
+}
+
+let server: http.Server;
+let baseUrl: string;
+let requests: EmbeddingsRequest[] = [];
+let savedBaseUrl: string | undefined;
+
+beforeAll(async () => {
+  server = http.createServer((req, res) => {
+    let body = "";
+    req.on("data", (c) => (body += c));
+    req.on("end", () => {
+      const parsed = JSON.parse(body) as EmbeddingsRequest;
+      requests.push(parsed);
+      const overIndex = parsed.input.findIndex(
+        (t) => t.length > SERVER_CHAR_LIMIT,
+      );
+      if (overIndex !== -1) {
+        // Production's exact error shape.
+        res.writeHead(400, { "content-type": "application/json" });
+        res.end(
+          JSON.stringify({
+            error: {
+              message: `Invalid 'input[${overIndex}]': maximum input length is 8192 tokens.`,
+              type: "invalid_request_error",
+              param: `input[${overIndex}]`,
+              code: null,
+            },
+          }),
+        );
+        return;
+      }
+      res.writeHead(200, { "content-type": "application/json" });
+      res.end(
+        JSON.stringify({
+          object: "list",
+          model: parsed.model,
+          data: parsed.input.map((_, i) => ({
+            object: "embedding",
+            index: i,
+            embedding: Array.from({ length: 8 }, () => 0.01),
+          })),
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        }),
+      );
+    });
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as AddressInfo;
+  baseUrl = `http://127.0.0.1:${port}/v1`;
+  savedBaseUrl = process.env.OPENAI_BASE_URL;
+  process.env.OPENAI_BASE_URL = baseUrl;
+});
+
+afterAll(async () => {
+  if (savedBaseUrl === undefined) delete process.env.OPENAI_BASE_URL;
+  else process.env.OPENAI_BASE_URL = savedBaseUrl;
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+});
+
+async function makeProvider() {
+  // Import AFTER OPENAI_BASE_URL is set: the SDK client is built in the
+  // provider constructor and reads the env var there.
+  const { OpenAIEmbeddingProvider } = await import("../indexing/embeddings.js");
+  return new OpenAIEmbeddingProvider("test-key", "text-embedding-3-small", 8);
+}
+
+describe("OpenAIEmbeddingProvider: an over-length input must not hard-fail", () => {
+  it("recovers from a length-400 by shrinking the offending input and retrying", async () => {
+    requests = [];
+    const provider = await makeProvider();
+    const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    // 20,000 chars: UNDER the 30,000 char cap, so the existing guard lets it
+    // through untouched — and the server rejects it. This is the production
+    // condition exactly.
+    const oversized = "x".repeat(20_000);
+    const vectors = await provider.embedBatch(["small", oversized]);
+
+    warnSpy.mockRestore();
+
+    expect(vectors).toHaveLength(2);
+    expect(vectors[1]).toHaveLength(8);
+    // It converged by shrinking, not by luck: more than one request was made
+    // and the final one was within the server's limit.
+    expect(requests.length).toBeGreaterThan(1);
+    const last = requests[requests.length - 1];
+    expect(Math.max(...last.input.map((t) => t.length))).toBeLessThanOrEqual(
+      SERVER_CHAR_LIMIT,
+    );
+    // Healthy siblings in the batch are not mangled.
+    expect(last.input[0]).toBe("small");
+  });
+
+  it("still fails loudly on a NON-length 400 (no infinite shrink loop)", async () => {
+    // Guard against "swallow every 400". A bad API key or an unknown param
+    // must still throw.
+    requests = [];
+    const badServer = http.createServer((req, res) => {
+      let body = "";
+      req.on("data", (c) => (body += c));
+      req.on("end", () => {
+        res.writeHead(400, { "content-type": "application/json" });
+        res.end(
+          JSON.stringify({
+            error: {
+              message: "Unknown parameter: 'dimensions'.",
+              type: "invalid_request_error",
+            },
+          }),
+        );
+      });
+    });
+    await new Promise<void>((r) => badServer.listen(0, "127.0.0.1", r));
+    const { port } = badServer.address() as AddressInfo;
+    const prev = process.env.OPENAI_BASE_URL;
+    process.env.OPENAI_BASE_URL = `http://127.0.0.1:${port}/v1`;
+    try {
+      const provider = await makeProvider();
+      await expect(provider.embedBatch(["anything"])).rejects.toThrow(
+        /Unknown parameter/,
+      );
+    } finally {
+      process.env.OPENAI_BASE_URL = prev;
+      await new Promise<void>((r) => badServer.close(() => r()));
+    }
+  });
+});

src/__tests__/orchestrator-config-fingerprint.test.ts

@@ -116,11 +116,11 @@ vi.mock("../indexing/pipeline.js", () => ({
         h.indexed.add(item.id);
         h.indexedThisRun.push(item.id);
       }
-      return { failedIds: [] };
+      return { failedIds: [], failures: [] };
     }
     async removeItems(ids: string[]) {
       for (const id of ids) h.indexed.delete(id);
-      return { failedIds: [] };
+      return { failedIds: [], failures: [] };
     }
   },
 }));