diff --git a/apps/memos-local-plugin/core/capture/capture.ts b/apps/memos-local-plugin/core/capture/capture.ts index 9d52f749e..73e975a47 100644 --- a/apps/memos-local-plugin/core/capture/capture.ts +++ b/apps/memos-local-plugin/core/capture/capture.ts @@ -133,8 +133,17 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // ─── Extract + dedup (skip steps we've already written this episode) ── const extractStart = now(); const rawAll = extractSteps(input.episode); - const existingTraces = deps.tracesRepo.list({ episodeId: input.episode.id }); - const seenTs = new Set(existingTraces.map((t) => t.ts)); + // #2076: MUST use listDedupRowsForEpisode (uncapped, streaming, no + // BLOB projection). The paginated `list` path silently truncates to + // 500 rows, which breaks dedup once an episode grows past that and + // causes the tail to be re-inserted every cycle (bloating `traces` + // unboundedly + starving the vector scan). Using the narrow-column + // dedup projection here also keeps peak RSS proportional to scalar + // fields, not embedding footprint (open code review on #2077). + const existingDedupRows = deps.tracesRepo.listDedupRowsForEpisode(input.episode.id); + const seenTs = new Set(existingDedupRows.map((t) => t.ts)); + // Reused later by persistRows so we don't scan the episode twice. + const seenSignatures = new Set(existingDedupRows.map(traceIdentitySignature)); const raw = rawAll.filter((s) => !seenTs.has(s.ts)); const extractMs = now() - extractStart; log.debug("stage.extract.done", { @@ -183,7 +192,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // Persist as new rows. Reflection / α deliberately empty. const persistStart = now(); const rows = buildRows(scored, summaries, vecs, input.episode); - const persisted = await persistRows(rows, input, warnings); + const persisted = await persistRows(rows, input, warnings, {}, seenSignatures); if (!persisted) { // emit capture.failed handled inside persistRows on hard fail. return finalResult( @@ -251,12 +260,16 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { const extractStart = now(); const rawAll = extractSteps(input.episode); - const existingTraces = deps.tracesRepo.list({ episodeId: input.episode.id }); + // #2076 + #2077 OCR: uncapped, narrow-projection dedup read. + // See `runLite` for the full rationale — one scan, no BLOBs. + const existingDedupRows = deps.tracesRepo.listDedupRowsForEpisode(input.episode.id); const seenTurnIds = new Set( - existingTraces + existingDedupRows .map((t) => t.turnId) .filter((v): v is number => typeof v === "number" && Number.isFinite(v)), ); + // Reused later by persistRows so we don't scan the episode twice. + const seenSignatures = new Set(existingDedupRows.map(traceIdentitySignature)); const rawByTurn = new Map(); for (const step of rawAll) { const turnId = pickTurnId(step.meta, step.ts); @@ -308,7 +321,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { }); const persisted = await persistRows(rows, input, warnings, { skipActionVectorRetry: true, - }); + }, seenSignatures); if (!persisted) { return finalResult( input, @@ -768,9 +781,23 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { input: CaptureInput, warnings: CaptureResult["warnings"], opts: { skipActionVectorRetry?: boolean } = {}, + existingSignatures?: Set, ): Promise { - const existingBeforeInsert = deps.tracesRepo.list({ episodeId: input.episode.id }); - const seenSignatures = new Set(existingBeforeInsert.map(traceIdentitySignature)); + // #2076 + #2077 OCR: uncapped, narrow-projection dedup read. The + // paginated `list` path missed all rows past the 500 cap and let + // duplicate signatures re-insert every cycle. When the caller has + // already computed the signature set upstream (runLite / + // runLightweight / runReflect all do), skip the second full scan + // and reuse it — otherwise fall back to a fresh streaming scan. + // We clone the caller-supplied set so the intra-batch dedup below + // doesn't leak new signatures back into the caller's Set instance. + const seenSignatures = existingSignatures + ? new Set(existingSignatures) + : new Set( + deps.tracesRepo + .listDedupRowsForEpisode(input.episode.id) + .map(traceIdentitySignature), + ); const uniqueRows = rows.filter((row) => { const signature = traceIdentitySignature(row); if (seenSignatures.has(signature)) return false; @@ -891,7 +918,9 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { return ["user", turnId, step.ts, step.userText.trim()].join("\x1f"); } - function traceIdentitySignature(row: TraceRow): string { + function traceIdentitySignature( + row: Pick, + ): string { const tool = row.toolCalls[0]; if (tool) { const hasRealTiming = diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index c00f617d9..894552b56 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -1,3 +1,4 @@ +import type { ToolCallDTO } from "../../../agent-contract/dto.js"; import type { EmbeddingVector, EpisodeId, SessionId, ShareScope, TraceId, TraceRow } from "../../types.js"; import type { StorageDb, TraceListFilter } from "../types.js"; import { buildInClause, buildInsert, buildUpdate } from "../tx.js"; @@ -58,6 +59,39 @@ export type TraceSearchMeta = { error_signatures_json?: string; }; +/** + * Narrow row shape returned by {@link listDedupRowsForEpisode}. Includes + * exactly the fields the capture-side dedup path needs (see + * `traceIdentitySignature` / `runLite` / `runLightweight` in + * `core/capture/capture.ts`). Deliberately excludes the two big BLOB + * columns (`vec_summary`, `vec_action`) so an episode with 500k rows + * can be scanned without pulling ~4 GB of embeddings into JS memory — + * the root pathology in #2076. + */ +export interface TraceDedupRow { + ts: number; + turnId: number; + userText: string; + agentText: string; + toolCalls: ToolCallDTO[]; +} + +interface RawDedupRow { + ts: number; + turn_id: number; + user_text: string; + agent_text: string; + tool_calls_json: string; +} + +const DEDUP_COLUMNS = [ + "ts", + "turn_id", + "user_text", + "agent_text", + "tool_calls_json", +] as const; + export function makeTracesRepo(db: StorageDb) { const insert = db.prepare(buildInsert({ table: "traces", columns: COLUMNS })); const upsert = db.prepare( @@ -219,11 +253,99 @@ export function makeTracesRepo(db: StorageDb) { } if (tr.sql) fragments.push(tr.sql); const where = joinWhere(fragments); - const page = buildPageClauses(filter, "ts"); + const shouldUseUncappedEpisodeScan = + Boolean(filter.episodeId) && + filter.limit === undefined && + filter.offset === undefined; + const page = shouldUseUncappedEpisodeScan + ? `ORDER BY ts ${filter.newestFirst === false ? "ASC" : "DESC"}` + : buildPageClauses(filter, "ts"); const sql = `SELECT ${COLUMNS.join(", ")} FROM traces ${where} ${page}`; return db.prepare(sql).all(params).map(mapRow); }, + /** + * Full episode-scoped trace fetch with NO pagination cap. + * + * Fetches ALL columns including the large `vec_summary` and + * `vec_action` BLOB columns, mapped into full `TraceRow` shape. + * Use this only when the caller genuinely needs those BLOBs + * (e.g. `runReflect`, which re-embeds and rewrites every field). + * + * **For dedup-only reads** — where only `ts`, `turnId`, + * `userText`, `agentText`, and `toolCalls` are needed — use + * {@link listDedupRowsForEpisode} instead to avoid loading + * multi-GB of embeddings into JS memory. + * + * Why an uncapped read exists at all: the paginated + * `list({ episodeId })` path silently truncates to + * `PageOptions.limit` (default 500). That cap breaks capture-side + * dedup (#2076): when an episode grows past the cap, the next + * runLite / runReflect only sees the newest 500 rows, treats + * every older step as "novel", and re-inserts the whole tail + * every cycle. In the reporter's 4.2 GB / 6.8 GB failure, + * 518,375 trace rows had shrunk to 80,583 distinct + * `(episode_id, turn_id, user_text, agent_text, tool_calls_json)` + * signatures — 84 % duplicates driven by exactly this loop. + * + * Rows are ordered by `ts ASC` so the causal chain matches the + * order runLite / runReflect built. + */ + listAllForEpisode(episodeId: EpisodeId | string): TraceRow[] { + if (!episodeId) return []; + const sql = `SELECT ${COLUMNS.join( + ", ", + )} FROM traces WHERE episode_id = @episode_id ORDER BY ts ASC`; + const rows = db + .prepare<{ episode_id: string }, RawTraceRow>(sql) + .all({ episode_id: String(episodeId) }); + return rows.map(mapRow); + }, + + /** + * Narrow-projection episode fetch for capture-side dedup. + * + * Sibling to {@link listAllForEpisode}, but projects only the five + * scalar dedup-identity columns + * (`ts` / `turn_id` / `user_text` / `agent_text` / `tool_calls_json`) + * and NEVER touches `vec_summary` / `vec_action`. That is the real + * saving: on the reporter's DB in #2076 the two BLOB columns + * dominated row size, so skipping them cuts per-row bytes by + * ~1000× regardless of how many rows the episode contains. + * + * Uses `stmt.iterate()` internally to avoid better-sqlite3's + * intermediate `.all()` allocation, but the result is still + * materialised into a `TraceDedupRow[]` and returned to the + * caller — this is not a streaming API. Peak JS memory therefore + * scales linearly with row count × the small scalar payload, + * which is what capture-side dedup actually needs. + * + * Every capture-side dedup call-site (`runLite`, `runLightweight`, + * `persistRows` in `core/capture/capture.ts`) uses this helper + * so no BLOBs load during the dedup pass. + * + * Same episode-scoping / `ts ASC` ordering contract as + * `listAllForEpisode`. + */ + listDedupRowsForEpisode(episodeId: EpisodeId | string): TraceDedupRow[] { + if (!episodeId) return []; + const sql = `SELECT ${DEDUP_COLUMNS.join( + ", ", + )} FROM traces WHERE episode_id = @episode_id ORDER BY ts ASC`; + const stmt = db.prepare<{ episode_id: string }, RawDedupRow>(sql); + const rows: TraceDedupRow[] = []; + for (const r of stmt.iterate({ episode_id: String(episodeId) })) { + rows.push({ + ts: r.ts, + turnId: r.turn_id, + userText: r.user_text, + agentText: r.agent_text, + toolCalls: fromJsonText(r.tool_calls_json, []), + }); + } + return rows; + }, + /** * Total row count matching the same filter (no limit/offset). * Used by list endpoints so the viewer can show "Page N of M". diff --git a/apps/memos-local-plugin/core/storage/vector.ts b/apps/memos-local-plugin/core/storage/vector.ts index 0100188e1..4aa27269d 100644 --- a/apps/memos-local-plugin/core/storage/vector.ts +++ b/apps/memos-local-plugin/core/storage/vector.ts @@ -194,10 +194,30 @@ export interface ScanRow { [k: string]: unknown; } +/** + * Default LIMIT applied when the caller doesn't pass an explicit + * `hardCap`. Historically 100_000, which — combined with the old + * `.all()` materialisation — meant one accidental "no cap" call + * could pull 100k multi-KB vector BLOBs into JS memory in a single + * synchronous step. On the reporter's DB in issue #2076 that + * translated to 4.2 GB RSS and a 100 % CPU main-thread stall. + * + * 5_000 is a much safer default: it still covers realistic per-agent + * corpora and it forces callers that legitimately need more to opt + * in explicitly. + */ +export const DEFAULT_SCAN_HARD_CAP = 5_000; + /** * Stream rows from `table`, decode vectors, and run top-K cosine against * `query`. `selectExtra` lets callers bring along columns that will surface in * `VectorHit.meta`. + * + * Streaming: we use `.iterate()` (not `.all()`) so at most one row's + * BLOB is decoded at a time. The top-K min-heap keeps only `k` + * vectors of state, so peak RSS is O(k * dim) regardless of how many + * rows the LIMIT allows. Fixes the "load 100k BLOBs synchronously" + * pathology in #2076. */ export function scanAndTopK( db: StorageDb, @@ -207,30 +227,72 @@ export function scanAndTopK( k: number, opts: VectorScanOptions, ): Array> { + if (k <= 0 || query.length === 0) return []; + const { vecColumn, norm2Column, where, params, hardCap } = opts; + const cap = hardCap ?? DEFAULT_SCAN_HARD_CAP; const cols = ["id", vecColumn, ...(norm2Column ? [norm2Column] : []), ...selectExtra]; const sql = [ `SELECT ${cols.join(", ")} FROM ${table}`, where ? `WHERE ${where}` : "", - `LIMIT ${hardCap ?? 100000}`, + `LIMIT ${cap}`, ] .filter(Boolean) .join(" "); - const rows = db.prepare(sql).all(params); - const decoded: Array> = []; - for (const r of rows) { + const qNorm = Math.sqrt(norm2(query)); + if (qNorm === 0) return []; + + // Build meta once per row — hoisted out of the loop so the meta + // projection is *not* re-allocated as a closure per iteration. The + // two heap-push branches below both need this when the row is kept, + // and rebuilding the closure in each iteration was pure GC pressure + // for hot searches. Only invoked for the heap-push branches, so + // filtered-out rows pay nothing. + const buildMeta = + selectExtra.length > 0 + ? (row: ScanRow): TMeta => + Object.fromEntries(selectExtra.map((c) => [c, row[c]])) as TMeta + : (_row: ScanRow): TMeta => undefined as TMeta; + + // Streaming top-K: min-heap of size k. We *never* build the full + // candidate array — one BLOB lives on the JS heap at a time (plus + // the k already-selected vectors we're comparing against). + const heap: Array> = []; + const stmt = db.prepare(sql); + const iter = params === undefined ? stmt.iterate() : stmt.iterate(params); + for (const r of iter) { const vec = decodeVector(r[vecColumn] as Buffer | null); if (!vec) continue; - const meta = selectExtra.length > 0 - ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) - : (undefined as TMeta); - decoded.push({ - id: String(r["id"]), - vec, - norm2: norm2Column ? ((r[norm2Column] as number | null) ?? undefined) : undefined, - meta, - }); + if (vec.length === 0 || vec.length !== query.length) { + // Non-empty vectors whose length disagrees with the query are the + // canonical "schema drift" signal — usually a re-embedding pass + // switched model dimensions and stale rows lingered. The old + // `topKCosine` path logged this exact case (see `topKCosine` + // above); the streaming rewrite must preserve the signal so + // operators can still detect silent under-recall in production. + if (vec.length !== 0) { + log.warn("search.dim_mismatch", { + expected: query.length, + got: vec.length, + rowId: String(r["id"]), + }); + } + continue; + } + const rowNorm2 = norm2Column + ? (r[norm2Column] as number | null | undefined) ?? norm2(vec) + : norm2(vec); + const score = cosinePrenormed(query, qNorm, vec, rowNorm2); + if (heap.length < k) { + heap.push({ id: String(r["id"]), score, meta: buildMeta(r) }); + siftUp(heap, heap.length - 1); + } else if (score > heap[0]!.score) { + heap[0] = { id: String(r["id"]), score, meta: buildMeta(r) }; + siftDown(heap, 0); + } } - return topKCosine(query, decoded, k); + // `heap` is a min-heap on score; caller wants DESC. + heap.sort((a, b) => b.score - a.score); + return heap; } diff --git a/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts new file mode 100644 index 000000000..7d0e3839f --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts @@ -0,0 +1,264 @@ +/** + * Regression tests for issue #2076 — dedup pagination cap causing unbounded + * re-insertion. + * + * Root cause: capture.ts dedup callers used `tracesRepo.list({ episodeId })`, + * which is paginated and silently truncates to 500 rows. Any episode with + * more than 500 rows would see the older rows re-inserted as "novel" every + * runLite/runReflect cycle. The fix is uncapped episode reconciliation reads + * (`listAllForEpisode(episodeId)` and the legacy `list({ episodeId })` path + * when no explicit page was requested). + * + * These tests pin the new contract: give me EVERY row for the episode, no + * matter how large it grew, so callers can compute the full "already seen" + * signature set. + */ + +import { describe, it, expect, beforeEach } from "vitest"; +import Database from "better-sqlite3"; +import { makeTracesRepo } from "../../../core/storage/repos/traces.js"; +import type { TraceRow } from "../../../agent-contract/dto.js"; + +describe("traces.listAllForEpisode — uncapped episode fetch (#2076)", () => { + let db: Database.Database; + let repo: ReturnType; + + beforeEach(() => { + db = new Database(":memory:"); + db.exec(` + CREATE TABLE traces ( + id TEXT PRIMARY KEY, + episode_id TEXT, + session_id TEXT NOT NULL, + owner_agent_kind TEXT, + owner_profile_id TEXT, + owner_workspace_id TEXT, + ts INTEGER NOT NULL, + user_text TEXT, + agent_text TEXT, + summary TEXT, + tool_calls_json TEXT, + reflection TEXT, + agent_thinking TEXT, + value REAL NOT NULL DEFAULT 0, + alpha REAL NOT NULL DEFAULT 0, + r_human REAL, + priority REAL NOT NULL DEFAULT 0, + tags_json TEXT, + error_signatures_json TEXT, + vec_summary BLOB, + vec_action BLOB, + share_scope TEXT, + share_target TEXT, + shared_at INTEGER, + turn_id INTEGER NOT NULL DEFAULT 0, + schema_version INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX idx_traces_episode_ts ON traces(episode_id, ts); + `); + // Cast around the type check — Database.Database duck-types StorageDb enough + // for these repo methods (prepare/all/get/run/iterate). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + repo = makeTracesRepo(db as any); + }); + + function makeTrace(overrides: Partial & { id: string; ts: number; turnId: number }): TraceRow { + return { + id: overrides.id, + episodeId: overrides.episodeId ?? "ep-huge", + sessionId: overrides.sessionId ?? "sess-1", + ts: overrides.ts, + userText: overrides.userText ?? `user ${overrides.id}`, + agentText: overrides.agentText ?? `agent ${overrides.id}`, + summary: overrides.summary ?? null, + toolCalls: overrides.toolCalls ?? [], + value: overrides.value ?? 0, + alpha: overrides.alpha ?? 0, + priority: overrides.priority ?? 0, + tags: overrides.tags ?? [], + errorSignatures: overrides.errorSignatures ?? [], + turnId: overrides.turnId, + schemaVersion: overrides.schemaVersion ?? 1, + }; + } + + it("returns EVERY trace for an episode, even when count exceeds the 500 pagination cap", () => { + // Insert 750 traces in one episode — more than the paginated 500 default + // that `list({ episodeId })` silently truncated to. + const N = 750; + for (let i = 0; i < N; i++) { + repo.insert(makeTrace({ id: `t-${i}`, ts: 1_000 + i, turnId: i })); + } + + const all = repo.listAllForEpisode("ep-huge"); + expect(all.length).toBe(N); + + // Also cross-check the ts set spans the full inserted range, so a caller + // computing `new Set(all.map(t => t.ts))` sees every timestamp. + const tsSet = new Set(all.map((t) => t.ts)); + expect(tsSet.has(1_000)).toBe(true); + expect(tsSet.has(1_000 + N - 1)).toBe(true); + }); + + it("keeps legacy episode-only list() calls uncapped unless the caller asks for a page", () => { + const N = 750; + for (let i = 0; i < N; i++) { + repo.insert(makeTrace({ id: `t-${i}`, ts: 1_000 + i, turnId: i })); + } + + const unpaged = repo.list({ episodeId: "ep-huge" }); + expect(unpaged.length).toBe(N); + + const paged = repo.list({ episodeId: "ep-huge", limit: 25 }); + expect(paged.length).toBe(25); + }); + + it("scopes strictly by episodeId — other episodes' rows are excluded", () => { + repo.insert(makeTrace({ id: "a1", episodeId: "ep-a", ts: 100, turnId: 1 })); + repo.insert(makeTrace({ id: "a2", episodeId: "ep-a", ts: 101, turnId: 2 })); + repo.insert(makeTrace({ id: "b1", episodeId: "ep-b", ts: 200, turnId: 1 })); + + const a = repo.listAllForEpisode("ep-a"); + expect(a.map((r) => r.id).sort()).toEqual(["a1", "a2"]); + const b = repo.listAllForEpisode("ep-b"); + expect(b.map((r) => r.id)).toEqual(["b1"]); + }); + + it("returns [] for an episode with no traces", () => { + expect(repo.listAllForEpisode("ep-missing")).toEqual([]); + }); + + it("orders rows by ts ascending — so callers see the causal chain the same way runLite / runReflect built it", () => { + // Insert out-of-order to defeat any implicit rowid order. + repo.insert(makeTrace({ id: "t-third", ts: 300, turnId: 3 })); + repo.insert(makeTrace({ id: "t-first", ts: 100, turnId: 1 })); + repo.insert(makeTrace({ id: "t-second", ts: 200, turnId: 2 })); + + const rows = repo.listAllForEpisode("ep-huge"); + expect(rows.map((r) => r.ts)).toEqual([100, 200, 300]); + }); +}); + +describe("traces.listDedupRowsForEpisode — narrow-projection dedup helper (#2077 OCR)", () => { + let db: Database.Database; + let repo: ReturnType; + + beforeEach(() => { + db = new Database(":memory:"); + db.exec(` + CREATE TABLE traces ( + id TEXT PRIMARY KEY, + episode_id TEXT, + session_id TEXT NOT NULL, + owner_agent_kind TEXT, + owner_profile_id TEXT, + owner_workspace_id TEXT, + ts INTEGER NOT NULL, + user_text TEXT, + agent_text TEXT, + summary TEXT, + tool_calls_json TEXT, + reflection TEXT, + agent_thinking TEXT, + value REAL NOT NULL DEFAULT 0, + alpha REAL NOT NULL DEFAULT 0, + r_human REAL, + priority REAL NOT NULL DEFAULT 0, + tags_json TEXT, + error_signatures_json TEXT, + vec_summary BLOB, + vec_action BLOB, + share_scope TEXT, + share_target TEXT, + shared_at INTEGER, + turn_id INTEGER NOT NULL DEFAULT 0, + schema_version INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX idx_traces_episode_ts_dedup ON traces(episode_id, ts); + `); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + repo = makeTracesRepo(db as any); + }); + + function makeTrace( + overrides: Partial & { id: string; ts: number; turnId: number }, + ): TraceRow { + return { + id: overrides.id, + episodeId: overrides.episodeId ?? "ep-dedup", + sessionId: overrides.sessionId ?? "sess-1", + ts: overrides.ts, + userText: overrides.userText ?? `user ${overrides.id}`, + agentText: overrides.agentText ?? `agent ${overrides.id}`, + summary: overrides.summary ?? null, + toolCalls: overrides.toolCalls ?? [], + value: overrides.value ?? 0, + alpha: overrides.alpha ?? 0, + priority: overrides.priority ?? 0, + tags: overrides.tags ?? [], + errorSignatures: overrides.errorSignatures ?? [], + turnId: overrides.turnId, + schemaVersion: overrides.schemaVersion ?? 1, + }; + } + + it("returns only the five dedup fields — no vecSummary / vecAction / tags / summary", () => { + // Populate an artificial 8 KB BLOB into vec_summary / vec_action so + // any accidental full-column projection would show up in memory tests. + const bigVec = new Float32Array(1024).fill(0.5); + repo.insert( + makeTrace({ + id: "t-1", + ts: 100, + turnId: 1, + toolCalls: [{ name: "get", input: { url: "https://example.com" } }], + }), + ); + // Poke a vector directly so the row has a heavy payload we can prove + // the narrow helper does NOT hydrate. + repo.updateVector("t-1", "vecSummary", bigVec); + repo.updateVector("t-1", "vecAction", bigVec); + + const rows = repo.listDedupRowsForEpisode("ep-dedup"); + expect(rows.length).toBe(1); + const [row] = rows; + // Just the five dedup fields — nothing more. + expect(Object.keys(row!).sort()).toEqual( + ["agentText", "toolCalls", "ts", "turnId", "userText"].sort(), + ); + expect(row!.toolCalls).toEqual([ + { name: "get", input: { url: "https://example.com" } }, + ]); + }); + + it("returns [] for empty episode / missing episodeId", () => { + expect(repo.listDedupRowsForEpisode("nope")).toEqual([]); + expect(repo.listDedupRowsForEpisode("")).toEqual([]); + }); + + it("streams every trace for an episode past the 500-row paginated cap", () => { + // Same regression guard as `listAllForEpisode` — the narrow path + // must also be uncapped, or callers get silent under-dedup. + const N = 750; + for (let i = 0; i < N; i++) { + repo.insert(makeTrace({ id: `t-${i}`, ts: 1_000 + i, turnId: i })); + } + const rows = repo.listDedupRowsForEpisode("ep-dedup"); + expect(rows.length).toBe(N); + expect(rows[0]!.ts).toBe(1_000); + expect(rows[N - 1]!.ts).toBe(1_000 + N - 1); + }); + + it("scopes strictly by episode and orders by ts ASC — matches listAllForEpisode contract", () => { + repo.insert(makeTrace({ id: "b1", episodeId: "ep-b", ts: 200, turnId: 1 })); + repo.insert(makeTrace({ id: "a-third", ts: 300, turnId: 3 })); + repo.insert(makeTrace({ id: "a-first", ts: 100, turnId: 1 })); + repo.insert(makeTrace({ id: "a-second", ts: 200, turnId: 2 })); + + const a = repo.listDedupRowsForEpisode("ep-dedup"); + expect(a.map((r) => r.ts)).toEqual([100, 200, 300]); + const b = repo.listDedupRowsForEpisode("ep-b"); + expect(b.length).toBe(1); + expect(b[0]!.ts).toBe(200); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts b/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts new file mode 100644 index 000000000..590691c23 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts @@ -0,0 +1,229 @@ +/** + * Regression tests for issue #2076 — scanAndTopK loading vectors synchronously. + * + * Root cause: `scanAndTopK` used `db.prepare(sql).all(params)`, which + * materialised up to `hardCap` (default 100_000) rows in one synchronous + * call, each carrying a multi-KB vector BLOB. On the reporter's DB the + * main thread pinned one core at 100 % with 4.2 GB RSS for 40 minutes. + * + * The fix: stream via `db.prepare(sql).iterate(params)` so at most one + * BLOB is decoded at a time; the top-K min-heap keeps only k vectors + * of state. Also drop the default `hardCap` to 5_000 so a caller that + * forgets to pass one doesn't accidentally scan the whole table. + * + * These tests pin: + * 1. correctness — streaming path returns identical top-K to the + * original brute-force behaviour on a small live DB. + * 2. safety — an unsupplied hardCap uses the reduced 5_000 default, + * not the old 100_000 default. + */ + +import { describe, it, expect, vi } from "vitest"; +import Database from "better-sqlite3"; +import { encodeVector, scanAndTopK } from "../../../core/storage/index.js"; +import { memoryBuffer, rootLogger } from "../../../core/logger/index.js"; + +function vec(arr: number[]): Float32Array { + return new Float32Array(arr); +} + +function openTinyVecDb(): Database.Database { + const db = new Database(":memory:"); + db.exec(`CREATE TABLE bench (id TEXT PRIMARY KEY, vec BLOB, extra TEXT);`); + return db; +} + +describe("scanAndTopK — streaming rewrite (#2076)", () => { + it("returns the same top-K as the naive brute-force on a small live DB", () => { + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + const target = vec([1, 0, 0]); + // Row exactly matching the query (score = 1) + insert.run("hit", encodeVector(target), "match"); + // Orthogonal row (score = 0) + insert.run("orth", encodeVector(vec([0, 1, 0])), "orth"); + // Opposite row (score = -1) + insert.run("opp", encodeVector(vec([-1, 0, 0])), "opp"); + + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + ["extra"], + target, + 2, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + expect(hits.length).toBe(2); + expect(hits[0]!.id).toBe("hit"); + expect(hits[0]!.score).toBeCloseTo(1); + expect(hits[1]!.id).toBe("orth"); + } finally { + db.close(); + } + }); + + it("survives more rows than the default hardCap keeping only the top-K in memory", () => { + // Sanity: on a table smaller than the cap, we should still see every + // row considered — the streaming rewrite must NOT truncate below the cap. + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + const query = vec([1, 0]); + for (let i = 0; i < 250; i++) { + // Vary the vector so scores span (0, 1); best is i=0 (aligned). + const v = vec([1 - i * 1e-3, i * 1e-4]); + insert.run(`r-${i}`, encodeVector(v), `x${i}`); + } + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 3, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + expect(hits.length).toBe(3); + // Highest scoring is r-0 (best-aligned), and the top-3 must be + // strictly-descending. + expect(hits[0]!.id).toBe("r-0"); + expect(hits[0]!.score).toBeGreaterThanOrEqual(hits[1]!.score); + expect(hits[1]!.score).toBeGreaterThanOrEqual(hits[2]!.score); + } finally { + db.close(); + } + }); + + it("respects an explicit hardCap smaller than the table size", () => { + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + // Row-0 is the "would-be top" hit; place it at row 10 so a cap of 5 + // demonstrably excludes it. The remaining rows have lower scores. + const query = vec([1, 0]); + for (let i = 0; i < 20; i++) { + // Order rows so best-aligned is at insert index 10. + const v = i === 10 ? vec([1, 0]) : vec([0.5 - i * 1e-3, 0.5]); + insert.run(`r-${i}`, encodeVector(v), null); + } + const withCap = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 1, + { vecColumn: "vec", where: "vec IS NOT NULL", hardCap: 5 }, + ); + // Only 5 rows were considered — none of which is r-10 (index 10 > 4), + // so the top hit is NOT the perfect match. + expect(withCap.length).toBe(1); + expect(withCap[0]!.id).not.toBe("r-10"); + + // With a cap large enough to reach r-10, the perfect match wins. + const noCap = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 1, + { vecColumn: "vec", where: "vec IS NOT NULL", hardCap: 20 }, + ); + expect(noCap[0]!.id).toBe("r-10"); + } finally { + db.close(); + } + }); + + it("logs `search.dim_mismatch` on a non-empty vector whose dimension disagrees with the query (#2077 OCR)", () => { + // Regression guard for the OCR observation: the streaming rewrite + // originally dropped mismatched rows silently, whereas the old + // topKCosine path emitted `search.dim_mismatch`. Losing that signal + // means operators can't detect schema drift (re-embedding at a new + // model dimension) — the function just returns fewer results with + // no warning. + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + // r-good matches the 3-dim query; r-drift is 4-dim (schema drift). + insert.run("r-good", encodeVector(vec([1, 0, 0])), null); + insert.run("r-drift", encodeVector(vec([0.1, 0.2, 0.3, 0.4])), null); + + const warnSpy = vi.spyOn(rootLogger, "warn"); + const query = vec([1, 0, 0]); + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 5, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + + // Only the aligned row is returned; the drift row is dropped. + expect(hits.map((h) => h.id)).toEqual(["r-good"]); + + // The observable operator signal — either a direct warn on the + // root logger or a warn record in the memory buffer — must fire + // for the drift row. + const bufWarn = memoryBuffer() + .tail({ limit: 100 }) + .find( + (r) => + r.msg === "search.dim_mismatch" && + (r.data as Record | undefined)?.rowId === "r-drift", + ); + const spyWarn = warnSpy.mock.calls.some( + ([msg, data]) => + msg === "search.dim_mismatch" && + (data as Record | undefined)?.rowId === "r-drift", + ); + expect(Boolean(bufWarn) || spyWarn).toBe(true); + warnSpy.mockRestore(); + } finally { + db.close(); + } + }); + + it("silently skips zero-length vector rows (empty embeddings are legitimate — not schema drift)", () => { + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + insert.run("r-good", encodeVector(vec([1, 0, 0])), null); + insert.run("r-empty", encodeVector(vec([])), null); + + const warnSpy = vi.spyOn(rootLogger, "warn"); + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + vec([1, 0, 0]), + 5, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + expect(hits.map((h) => h.id)).toEqual(["r-good"]); + // Zero-length rows are drop-without-warn — see vector.ts comment. + const spyDrift = warnSpy.mock.calls.some( + ([msg, data]) => + msg === "search.dim_mismatch" && + (data as Record | undefined)?.rowId === "r-empty", + ); + const bufDrift = memoryBuffer() + .tail({ limit: 100 }) + .some( + (r) => + r.msg === "search.dim_mismatch" && + (r.data as Record | undefined)?.rowId === "r-empty", + ); + expect(spyDrift || bufDrift).toBe(false); + warnSpy.mockRestore(); + } finally { + db.close(); + } + }); +});