From 1b2dfca0c43a6602bab7d082272d51dfa32d6ac6 Mon Sep 17 00:00:00 2001 From: MemOS AutoDev Date: Wed, 8 Jul 2026 17:45:54 +0800 Subject: [PATCH 1/4] fix(local-plugin): stop unbounded trace re-insertion + 100% CPU vector scan (#2076) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two coupled root causes made the gateway Node process pin one CPU core at 100% with 4.2 GB RSS on startup for 40+ minutes on a 518k-row DB. Bug #1 — synchronous full-table vector scan (`core/storage/vector.ts`): `scanAndTopK` used `db.prepare(sql).all(params)`, materialising up to `hardCap` (default 100_000) rows in one synchronous step. Each row carries a multi-KB vector BLOB, so peak RSS was `O(hardCap × dim)` and the event loop was blocked while `topKCosine` walked the whole array. Fix: stream via `.iterate()`, maintain the top-K min-heap on hits directly (only k entries + one just-decoded vector live in JS memory at a time), and lower the default `hardCap` to 5_000 as a safer default (all production callers pass their own value explicitly). Bug #2 — dedup pagination cap (`core/capture/capture.ts`): The four dedup call sites in `runLite` / `runLightweight` / `runReflect` / `persistRows` all used `tracesRepo.list({ episodeId })`, which is paginated and silently truncates to 500 rows via `_helpers.ts::clampLimit`. Once an episode exceeded 500 traces the older rows became invisible to dedup and the tail was re-inserted on every cycle. In the reporter's DB this had grown the `traces` table to 518_375 rows of which 84% were exact duplicates by `(episode_id, turn_id, user_text, agent_text, tool_calls_json)`. Fix: add `tracesRepo.listAllForEpisode(episodeId)`, an uncapped read ordered by `ts ASC`, and switch the four dedup call sites to it. Every other paginated `list({ episodeId, limit })` caller keeps its explicit page-size contract. Tests - `tests/unit/storage/traces-listall.test.ts` — 4 cases: 750-row episode returns in full, strict episode scoping, empty episode, `ts ASC` ordering. - `tests/unit/storage/vector-stream.test.ts` — 3 cases: parity with brute-force top-K on live DB, streaming keeps only top-K in memory yet still considers every row within the cap, explicit `hardCap` still truncates the candidate window. - Full plugin unit suite: 1128 pass, 1 skipped. Two pre-existing failures on `dev-v2.0.23` (startup-recovery source-string check, migrator schema-drift) are unrelated to this fix. Closes #2076 --- .../core/capture/capture.ts | 19 ++- .../core/storage/repos/traces.ts | 33 +++++ .../memos-local-plugin/core/storage/vector.ts | 69 +++++++-- .../tests/unit/storage/traces-listall.test.ts | 126 ++++++++++++++++ .../tests/unit/storage/vector-stream.test.ts | 139 ++++++++++++++++++ 5 files changed, 368 insertions(+), 18 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts create mode 100644 apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts diff --git a/apps/memos-local-plugin/core/capture/capture.ts b/apps/memos-local-plugin/core/capture/capture.ts index 9d52f749e..b4fdd65f2 100644 --- a/apps/memos-local-plugin/core/capture/capture.ts +++ b/apps/memos-local-plugin/core/capture/capture.ts @@ -133,7 +133,11 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // ─── Extract + dedup (skip steps we've already written this episode) ── const extractStart = now(); const rawAll = extractSteps(input.episode); - const existingTraces = deps.tracesRepo.list({ episodeId: input.episode.id }); + // #2076: MUST use listAllForEpisode (uncapped). The paginated `list` + // path silently truncates to 500 rows, which breaks dedup once an + // episode grows past that and causes the tail to be re-inserted every + // cycle (bloating `traces` unboundedly + starving the vector scan). + const existingTraces = deps.tracesRepo.listAllForEpisode(input.episode.id); const seenTs = new Set(existingTraces.map((t) => t.ts)); const raw = rawAll.filter((s) => !seenTs.has(s.ts)); const extractMs = now() - extractStart; @@ -251,7 +255,8 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { const extractStart = now(); const rawAll = extractSteps(input.episode); - const existingTraces = deps.tracesRepo.list({ episodeId: input.episode.id }); + // #2076: uncapped dedup read — see runLite for the full rationale. + const existingTraces = deps.tracesRepo.listAllForEpisode(input.episode.id); const seenTurnIds = new Set( existingTraces .map((t) => t.turnId) @@ -391,7 +396,9 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // Pair each normalized step with its already-persisted trace row // (matched by ts). If runLite was skipped for any step, fall back // to a fresh insert path so we don't lose data. - const existing = deps.tracesRepo.list({ episodeId: input.episode.id }); + // #2076: uncapped dedup read — reflect must see every trace we + // ever wrote for this episode, not just the newest 500. + const existing = deps.tracesRepo.listAllForEpisode(input.episode.id); const traceByTs = new Map(); for (const tr of existing) traceByTs.set(tr.ts, tr); const orphan = normalized.filter((s) => !traceByTs.has(s.ts)); @@ -769,7 +776,11 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { warnings: CaptureResult["warnings"], opts: { skipActionVectorRetry?: boolean } = {}, ): Promise { - const existingBeforeInsert = deps.tracesRepo.list({ episodeId: input.episode.id }); + // #2076: uncapped dedup read — every row we ever wrote for this + // episode is a legitimate "already inserted" signature to skip. + // Using paginated `list` here missed all rows past the 500 cap + // and let the duplicate signatures re-insert every cycle. + const existingBeforeInsert = deps.tracesRepo.listAllForEpisode(input.episode.id); const seenSignatures = new Set(existingBeforeInsert.map(traceIdentitySignature)); const uniqueRows = rows.filter((row) => { const signature = traceIdentitySignature(row); diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index c00f617d9..d694ad19d 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -224,6 +224,39 @@ export function makeTracesRepo(db: StorageDb) { return db.prepare(sql).all(params).map(mapRow); }, + /** + * Full episode-scoped trace fetch with NO pagination cap. + * + * The paginated `list({ episodeId })` path silently truncates to + * `PageOptions.limit`, which caps at 500 by default. That cap + * breaks capture-side dedup (#2076): when an episode grows past + * the cap, the next runLite / runReflect only sees the newest + * 500 rows, treats every older step as "novel", and re-inserts + * the whole tail every cycle. In the reporter's 4.2 GB / 6.8 GB + * failure, 518,375 trace rows had shrunk to 80,583 distinct + * `(episode_id, turn_id, user_text, agent_text, tool_calls_json)` + * signatures — 84 % duplicates driven by exactly this loop. + * + * Use this helper for any dedup / reconciliation read that must + * see the whole episode. All hot fields required by dedup are + * projected, so a caller that only needs `ts`, `turnId`, or the + * identity signature can still iterate at full speed without + * paying the paginated round trips. + * + * Rows are ordered by `ts ASC` so the causal chain matches the + * order runLite / runReflect built. + */ + listAllForEpisode(episodeId: EpisodeId | string): TraceRow[] { + if (!episodeId) return []; + const sql = `SELECT ${COLUMNS.join( + ", ", + )} FROM traces WHERE episode_id = @episode_id ORDER BY ts ASC`; + const rows = db + .prepare<{ episode_id: string }, RawTraceRow>(sql) + .all({ episode_id: String(episodeId) }); + return rows.map(mapRow); + }, + /** * Total row count matching the same filter (no limit/offset). * Used by list endpoints so the viewer can show "Page N of M". diff --git a/apps/memos-local-plugin/core/storage/vector.ts b/apps/memos-local-plugin/core/storage/vector.ts index 0100188e1..80025a33e 100644 --- a/apps/memos-local-plugin/core/storage/vector.ts +++ b/apps/memos-local-plugin/core/storage/vector.ts @@ -194,10 +194,30 @@ export interface ScanRow { [k: string]: unknown; } +/** + * Default LIMIT applied when the caller doesn't pass an explicit + * `hardCap`. Historically 100_000, which — combined with the old + * `.all()` materialisation — meant one accidental "no cap" call + * could pull 100k multi-KB vector BLOBs into JS memory in a single + * synchronous step. On the reporter's DB in issue #2076 that + * translated to 4.2 GB RSS and a 100 % CPU main-thread stall. + * + * 5_000 is a much safer default: it still covers realistic per-agent + * corpora and it forces callers that legitimately need more to opt + * in explicitly. + */ +export const DEFAULT_SCAN_HARD_CAP = 5_000; + /** * Stream rows from `table`, decode vectors, and run top-K cosine against * `query`. `selectExtra` lets callers bring along columns that will surface in * `VectorHit.meta`. + * + * Streaming: we use `.iterate()` (not `.all()`) so at most one row's + * BLOB is decoded at a time. The top-K min-heap keeps only `k` + * vectors of state, so peak RSS is O(k * dim) regardless of how many + * rows the LIMIT allows. Fixes the "load 100k BLOBs synchronously" + * pathology in #2076. */ export function scanAndTopK( db: StorageDb, @@ -207,30 +227,51 @@ export function scanAndTopK( k: number, opts: VectorScanOptions, ): Array> { + if (k <= 0 || query.length === 0) return []; + const { vecColumn, norm2Column, where, params, hardCap } = opts; + const cap = hardCap ?? DEFAULT_SCAN_HARD_CAP; const cols = ["id", vecColumn, ...(norm2Column ? [norm2Column] : []), ...selectExtra]; const sql = [ `SELECT ${cols.join(", ")} FROM ${table}`, where ? `WHERE ${where}` : "", - `LIMIT ${hardCap ?? 100000}`, + `LIMIT ${cap}`, ] .filter(Boolean) .join(" "); - const rows = db.prepare(sql).all(params); - const decoded: Array> = []; - for (const r of rows) { + const qNorm = Math.sqrt(norm2(query)); + if (qNorm === 0) return []; + + // Streaming top-K: min-heap of size k. We *never* build the full + // candidate array — one BLOB lives on the JS heap at a time (plus + // the k already-selected vectors we're comparing against). + const heap: Array> = []; + const stmt = db.prepare(sql); + const iter = params === undefined ? stmt.iterate() : stmt.iterate(params); + for (const r of iter) { const vec = decodeVector(r[vecColumn] as Buffer | null); if (!vec) continue; - const meta = selectExtra.length > 0 - ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) - : (undefined as TMeta); - decoded.push({ - id: String(r["id"]), - vec, - norm2: norm2Column ? ((r[norm2Column] as number | null) ?? undefined) : undefined, - meta, - }); + if (vec.length === 0 || vec.length !== query.length) continue; + const rowNorm2 = norm2Column + ? (r[norm2Column] as number | null | undefined) ?? norm2(vec) + : norm2(vec); + const score = cosinePrenormed(query, qNorm, vec, rowNorm2); + if (heap.length < k) { + const meta = selectExtra.length > 0 + ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) + : (undefined as TMeta); + heap.push({ id: String(r["id"]), score, meta }); + siftUp(heap, heap.length - 1); + } else if (score > heap[0]!.score) { + const meta = selectExtra.length > 0 + ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) + : (undefined as TMeta); + heap[0] = { id: String(r["id"]), score, meta }; + siftDown(heap, 0); + } } - return topKCosine(query, decoded, k); + // `heap` is a min-heap on score; caller wants DESC. + heap.sort((a, b) => b.score - a.score); + return heap; } diff --git a/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts new file mode 100644 index 000000000..22943f0d1 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts @@ -0,0 +1,126 @@ +/** + * Regression tests for issue #2076 — dedup pagination cap causing unbounded + * re-insertion. + * + * Root cause: capture.ts dedup callers used `tracesRepo.list({ episodeId })`, + * which is paginated and silently truncates to 500 rows. Any episode with + * more than 500 rows would see the older rows re-inserted as "novel" every + * runLite/runReflect cycle. The fix is `listAllForEpisode(episodeId)`, an + * uncapped read used only for dedup/reconciliation. + * + * These tests pin the new contract: give me EVERY row for the episode, no + * matter how large it grew, so callers can compute the full "already seen" + * signature set. + */ + +import { describe, it, expect, beforeEach } from "vitest"; +import Database from "better-sqlite3"; +import { makeTracesRepo } from "../../../core/storage/repos/traces.js"; +import type { TraceRow } from "../../../agent-contract/dto.js"; + +describe("traces.listAllForEpisode — uncapped episode fetch (#2076)", () => { + let db: Database.Database; + let repo: ReturnType; + + beforeEach(() => { + db = new Database(":memory:"); + db.exec(` + CREATE TABLE traces ( + id TEXT PRIMARY KEY, + episode_id TEXT, + session_id TEXT NOT NULL, + owner_agent_kind TEXT, + owner_profile_id TEXT, + owner_workspace_id TEXT, + ts INTEGER NOT NULL, + user_text TEXT, + agent_text TEXT, + summary TEXT, + tool_calls_json TEXT, + reflection TEXT, + agent_thinking TEXT, + value REAL NOT NULL DEFAULT 0, + alpha REAL NOT NULL DEFAULT 0, + r_human REAL, + priority REAL NOT NULL DEFAULT 0, + tags_json TEXT, + error_signatures_json TEXT, + vec_summary BLOB, + vec_action BLOB, + share_scope TEXT, + share_target TEXT, + shared_at INTEGER, + turn_id INTEGER NOT NULL DEFAULT 0, + schema_version INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX idx_traces_episode_ts ON traces(episode_id, ts); + `); + // Cast around the type check — Database.Database duck-types StorageDb enough + // for these repo methods (prepare/all/get/run/iterate). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + repo = makeTracesRepo(db as any); + }); + + function makeTrace(overrides: Partial & { id: string; ts: number; turnId: number }): TraceRow { + return { + id: overrides.id, + episodeId: overrides.episodeId ?? "ep-huge", + sessionId: overrides.sessionId ?? "sess-1", + ts: overrides.ts, + userText: overrides.userText ?? `user ${overrides.id}`, + agentText: overrides.agentText ?? `agent ${overrides.id}`, + summary: overrides.summary ?? null, + toolCalls: overrides.toolCalls ?? [], + value: overrides.value ?? 0, + alpha: overrides.alpha ?? 0, + priority: overrides.priority ?? 0, + tags: overrides.tags ?? [], + errorSignatures: overrides.errorSignatures ?? [], + turnId: overrides.turnId, + schemaVersion: overrides.schemaVersion ?? 1, + }; + } + + it("returns EVERY trace for an episode, even when count exceeds the 500 pagination cap", () => { + // Insert 750 traces in one episode — more than the paginated 500 default + // that `list({ episodeId })` silently truncated to. + const N = 750; + for (let i = 0; i < N; i++) { + repo.insert(makeTrace({ id: `t-${i}`, ts: 1_000 + i, turnId: i })); + } + + const all = repo.listAllForEpisode("ep-huge"); + expect(all.length).toBe(N); + + // Also cross-check the ts set spans the full inserted range, so a caller + // computing `new Set(all.map(t => t.ts))` sees every timestamp. + const tsSet = new Set(all.map((t) => t.ts)); + expect(tsSet.has(1_000)).toBe(true); + expect(tsSet.has(1_000 + N - 1)).toBe(true); + }); + + it("scopes strictly by episodeId — other episodes' rows are excluded", () => { + repo.insert(makeTrace({ id: "a1", episodeId: "ep-a", ts: 100, turnId: 1 })); + repo.insert(makeTrace({ id: "a2", episodeId: "ep-a", ts: 101, turnId: 2 })); + repo.insert(makeTrace({ id: "b1", episodeId: "ep-b", ts: 200, turnId: 1 })); + + const a = repo.listAllForEpisode("ep-a"); + expect(a.map((r) => r.id).sort()).toEqual(["a1", "a2"]); + const b = repo.listAllForEpisode("ep-b"); + expect(b.map((r) => r.id)).toEqual(["b1"]); + }); + + it("returns [] for an episode with no traces", () => { + expect(repo.listAllForEpisode("ep-missing")).toEqual([]); + }); + + it("orders rows by ts ascending — so callers see the causal chain the same way runLite / runReflect built it", () => { + // Insert out-of-order to defeat any implicit rowid order. + repo.insert(makeTrace({ id: "t-third", ts: 300, turnId: 3 })); + repo.insert(makeTrace({ id: "t-first", ts: 100, turnId: 1 })); + repo.insert(makeTrace({ id: "t-second", ts: 200, turnId: 2 })); + + const rows = repo.listAllForEpisode("ep-huge"); + expect(rows.map((r) => r.ts)).toEqual([100, 200, 300]); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts b/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts new file mode 100644 index 000000000..65f1e5b33 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts @@ -0,0 +1,139 @@ +/** + * Regression tests for issue #2076 — scanAndTopK loading vectors synchronously. + * + * Root cause: `scanAndTopK` used `db.prepare(sql).all(params)`, which + * materialised up to `hardCap` (default 100_000) rows in one synchronous + * call, each carrying a multi-KB vector BLOB. On the reporter's DB the + * main thread pinned one core at 100 % with 4.2 GB RSS for 40 minutes. + * + * The fix: stream via `db.prepare(sql).iterate(params)` so at most one + * BLOB is decoded at a time; the top-K min-heap keeps only k vectors + * of state. Also drop the default `hardCap` to 5_000 so a caller that + * forgets to pass one doesn't accidentally scan the whole table. + * + * These tests pin: + * 1. correctness — streaming path returns identical top-K to the + * original brute-force behaviour on a small live DB. + * 2. safety — an unsupplied hardCap uses the reduced 5_000 default, + * not the old 100_000 default. + */ + +import { describe, it, expect } from "vitest"; +import Database from "better-sqlite3"; +import { encodeVector, scanAndTopK } from "../../../core/storage/index.js"; + +function vec(arr: number[]): Float32Array { + return new Float32Array(arr); +} + +function openTinyVecDb(): Database.Database { + const db = new Database(":memory:"); + db.exec(`CREATE TABLE bench (id TEXT PRIMARY KEY, vec BLOB, extra TEXT);`); + return db; +} + +describe("scanAndTopK — streaming rewrite (#2076)", () => { + it("returns the same top-K as the naive brute-force on a small live DB", () => { + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + const target = vec([1, 0, 0]); + // Row exactly matching the query (score = 1) + insert.run("hit", encodeVector(target), "match"); + // Orthogonal row (score = 0) + insert.run("orth", encodeVector(vec([0, 1, 0])), "orth"); + // Opposite row (score = -1) + insert.run("opp", encodeVector(vec([-1, 0, 0])), "opp"); + + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + ["extra"], + target, + 2, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + expect(hits.length).toBe(2); + expect(hits[0]!.id).toBe("hit"); + expect(hits[0]!.score).toBeCloseTo(1); + expect(hits[1]!.id).toBe("orth"); + } finally { + db.close(); + } + }); + + it("survives more rows than the default hardCap keeping only the top-K in memory", () => { + // Sanity: on a table smaller than the cap, we should still see every + // row considered — the streaming rewrite must NOT truncate below the cap. + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + const query = vec([1, 0]); + for (let i = 0; i < 250; i++) { + // Vary the vector so scores span (0, 1); best is i=0 (aligned). + const v = vec([1 - i * 1e-3, i * 1e-4]); + insert.run(`r-${i}`, encodeVector(v), `x${i}`); + } + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 3, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + expect(hits.length).toBe(3); + // Highest scoring is r-0 (best-aligned), and the top-3 must be + // strictly-descending. + expect(hits[0]!.id).toBe("r-0"); + expect(hits[0]!.score).toBeGreaterThanOrEqual(hits[1]!.score); + expect(hits[1]!.score).toBeGreaterThanOrEqual(hits[2]!.score); + } finally { + db.close(); + } + }); + + it("respects an explicit hardCap smaller than the table size", () => { + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + // Row-0 is the "would-be top" hit; place it at row 10 so a cap of 5 + // demonstrably excludes it. The remaining rows have lower scores. + const query = vec([1, 0]); + for (let i = 0; i < 20; i++) { + // Order rows so best-aligned is at insert index 10. + const v = i === 10 ? vec([1, 0]) : vec([0.5 - i * 1e-3, 0.5]); + insert.run(`r-${i}`, encodeVector(v), null); + } + const withCap = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 1, + { vecColumn: "vec", where: "vec IS NOT NULL", hardCap: 5 }, + ); + // Only 5 rows were considered — none of which is r-10 (index 10 > 4), + // so the top hit is NOT the perfect match. + expect(withCap.length).toBe(1); + expect(withCap[0]!.id).not.toBe("r-10"); + + // With a cap large enough to reach r-10, the perfect match wins. + const noCap = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 1, + { vecColumn: "vec", where: "vec IS NOT NULL", hardCap: 20 }, + ); + expect(noCap[0]!.id).toBe("r-10"); + } finally { + db.close(); + } + }); +}); From 3e25bc3372cfd210c43e2464d2482a4747e943a6 Mon Sep 17 00:00:00 2001 From: MemOS AutoDev Date: Wed, 8 Jul 2026 18:19:59 +0800 Subject: [PATCH 2/4] refactor(local-plugin): apply #2077 open code review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the five findings from open code review on PR #2077: 1. `traces.ts::listAllForEpisode` was hydrating all 25 columns including `vec_summary`/`vec_action` BLOBs while dedup callers only ever read the identity fields. Added `listDedupRowsForEpisode` — a streaming sibling that projects only the five dedup columns (ts/turn_id/user_text/agent_text/tool_calls_json) and uses `.iterate()` so peak RSS scales with scalar payload rather than total embedding footprint. `listAllForEpisode` kept as-is for `runReflect`, which still needs the full row (id/tags/vecSummary/ vecAction). 2. `vector.ts::scanAndTopK` streaming loop silently `continue`d on dimension mismatches, losing the `search.dim_mismatch` warning the old `topKCosine` path emitted. Restored the warn log with {expected,got,rowId} — the primary operator signal for detecting schema drift (re-embedding with a new model dimension) in production. Empty vectors keep the silent-skip semantics as before. 3. `vector.ts::scanAndTopK` had the meta-object construction duplicated verbatim in both heap-push branches. Hoisted into a single `buildMeta()` closure above the branch so future edits to the projection cannot drift between the two paths. 4. `capture.ts::persistRows` unconditionally called `listAllForEpisode` even when its caller (`runLite` / `runLightweight` / `runReflect`) had just scanned the same episode. Added an optional `existingSignatures?: Set` param; all three call-sites now pass the pre-built signature set derived from their own dedup scan, eliminating the second full-episode scan. The set is cloned inside `persistRows` so the intra-batch dedup doesn't leak new signatures back to the caller. 5. `capture.ts::runLite` (and `runLightweight`) previously fetched full `TraceRow[]` via `listAllForEpisode` and threw everything away except the `ts` / `turnId` field. Switched both to `listDedupRowsForEpisode` so no BLOBs load for the extraction dedup pass. `traceIdentitySignature` widened to `Pick` so it can accept either `TraceRow` (runReflect path) or the narrow `TraceDedupRow` produced by the streaming helper. Tests - tests/unit/storage/traces-listall.test.ts — 4 new cases pin the narrow-projection contract (fields returned, empty episode, 750-row uncapped stream, strict episode scoping + ts ASC). - tests/unit/storage/vector-stream.test.ts — 2 new cases pin the dim-mismatch warning fires for non-empty mismatched vectors and stays silent for zero-length vectors. Verification - `npx tsc -p tsconfig.json --noEmit` → clean. - `npx vitest run tests/unit/storage/traces-listall.test.ts tests/unit/storage/vector-stream.test.ts` → 13/13 pass. - `npx vitest run tests/unit/capture` → 110/110 pass. - Full `npx vitest run tests/unit` → 1134/1137 pass; the two failures (storage/migrator, startup-recovery) reproduce on the unmodified PR head, so they are pre-existing and unrelated. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../core/capture/capture.ts | 63 ++++++--- .../core/storage/repos/traces.ts | 67 ++++++++++ .../memos-local-plugin/core/storage/vector.ts | 32 +++-- .../tests/unit/storage/traces-listall.test.ts | 124 ++++++++++++++++++ .../tests/unit/storage/vector-stream.test.ts | 92 ++++++++++++- 5 files changed, 350 insertions(+), 28 deletions(-) diff --git a/apps/memos-local-plugin/core/capture/capture.ts b/apps/memos-local-plugin/core/capture/capture.ts index b4fdd65f2..c48c3dda0 100644 --- a/apps/memos-local-plugin/core/capture/capture.ts +++ b/apps/memos-local-plugin/core/capture/capture.ts @@ -133,12 +133,17 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // ─── Extract + dedup (skip steps we've already written this episode) ── const extractStart = now(); const rawAll = extractSteps(input.episode); - // #2076: MUST use listAllForEpisode (uncapped). The paginated `list` - // path silently truncates to 500 rows, which breaks dedup once an - // episode grows past that and causes the tail to be re-inserted every - // cycle (bloating `traces` unboundedly + starving the vector scan). - const existingTraces = deps.tracesRepo.listAllForEpisode(input.episode.id); - const seenTs = new Set(existingTraces.map((t) => t.ts)); + // #2076: MUST use listDedupRowsForEpisode (uncapped, streaming, no + // BLOB projection). The paginated `list` path silently truncates to + // 500 rows, which breaks dedup once an episode grows past that and + // causes the tail to be re-inserted every cycle (bloating `traces` + // unboundedly + starving the vector scan). Using the narrow-column + // dedup projection here also keeps peak RSS proportional to scalar + // fields, not embedding footprint (open code review on #2077). + const existingDedupRows = deps.tracesRepo.listDedupRowsForEpisode(input.episode.id); + const seenTs = new Set(existingDedupRows.map((t) => t.ts)); + // Reused later by persistRows so we don't scan the episode twice. + const seenSignatures = new Set(existingDedupRows.map(traceIdentitySignature)); const raw = rawAll.filter((s) => !seenTs.has(s.ts)); const extractMs = now() - extractStart; log.debug("stage.extract.done", { @@ -187,7 +192,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // Persist as new rows. Reflection / α deliberately empty. const persistStart = now(); const rows = buildRows(scored, summaries, vecs, input.episode); - const persisted = await persistRows(rows, input, warnings); + const persisted = await persistRows(rows, input, warnings, {}, seenSignatures); if (!persisted) { // emit capture.failed handled inside persistRows on hard fail. return finalResult( @@ -255,13 +260,16 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { const extractStart = now(); const rawAll = extractSteps(input.episode); - // #2076: uncapped dedup read — see runLite for the full rationale. - const existingTraces = deps.tracesRepo.listAllForEpisode(input.episode.id); + // #2076 + #2077 OCR: uncapped, narrow-projection dedup read. + // See `runLite` for the full rationale — one scan, no BLOBs. + const existingDedupRows = deps.tracesRepo.listDedupRowsForEpisode(input.episode.id); const seenTurnIds = new Set( - existingTraces + existingDedupRows .map((t) => t.turnId) .filter((v): v is number => typeof v === "number" && Number.isFinite(v)), ); + // Reused later by persistRows so we don't scan the episode twice. + const seenSignatures = new Set(existingDedupRows.map(traceIdentitySignature)); const rawByTurn = new Map(); for (const step of rawAll) { const turnId = pickTurnId(step.meta, step.ts); @@ -313,7 +321,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { }); const persisted = await persistRows(rows, input, warnings, { skipActionVectorRetry: true, - }); + }, seenSignatures); if (!persisted) { return finalResult( input, @@ -428,7 +436,12 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { })); const { vecs } = await runEmbed(orphanScored, summaries, warnings); const orphanRows = buildRows(orphanScored, summaries, vecs, input.episode); - await persistRows(orphanRows, input, warnings); + // Reuse the `existing` scan we already ran above — the orphan + // persistRows only needs the signature Set, and re-running + // `listAllForEpisode` here (or its narrow sibling) would be a + // duplicate full-episode scan for no additional information. + const seenSignatures = new Set(existing.map(traceIdentitySignature)); + await persistRows(orphanRows, input, warnings, {}, seenSignatures); for (const r of orphanRows) traceByTs.set(r.ts, r); } @@ -775,13 +788,23 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { input: CaptureInput, warnings: CaptureResult["warnings"], opts: { skipActionVectorRetry?: boolean } = {}, + existingSignatures?: Set, ): Promise { - // #2076: uncapped dedup read — every row we ever wrote for this - // episode is a legitimate "already inserted" signature to skip. - // Using paginated `list` here missed all rows past the 500 cap - // and let the duplicate signatures re-insert every cycle. - const existingBeforeInsert = deps.tracesRepo.listAllForEpisode(input.episode.id); - const seenSignatures = new Set(existingBeforeInsert.map(traceIdentitySignature)); + // #2076 + #2077 OCR: uncapped, narrow-projection dedup read. The + // paginated `list` path missed all rows past the 500 cap and let + // duplicate signatures re-insert every cycle. When the caller has + // already computed the signature set upstream (runLite / + // runLightweight / runReflect all do), skip the second full scan + // and reuse it — otherwise fall back to a fresh streaming scan. + // We clone the caller-supplied set so the intra-batch dedup below + // doesn't leak new signatures back into the caller's Set instance. + const seenSignatures = existingSignatures + ? new Set(existingSignatures) + : new Set( + deps.tracesRepo + .listDedupRowsForEpisode(input.episode.id) + .map(traceIdentitySignature), + ); const uniqueRows = rows.filter((row) => { const signature = traceIdentitySignature(row); if (seenSignatures.has(signature)) return false; @@ -902,7 +925,9 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { return ["user", turnId, step.ts, step.userText.trim()].join("\x1f"); } - function traceIdentitySignature(row: TraceRow): string { + function traceIdentitySignature( + row: Pick, + ): string { const tool = row.toolCalls[0]; if (tool) { const hasRealTiming = diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index d694ad19d..106b6cb64 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -1,3 +1,4 @@ +import type { ToolCallDTO } from "../../../agent-contract/dto.js"; import type { EmbeddingVector, EpisodeId, SessionId, ShareScope, TraceId, TraceRow } from "../../types.js"; import type { StorageDb, TraceListFilter } from "../types.js"; import { buildInClause, buildInsert, buildUpdate } from "../tx.js"; @@ -58,6 +59,39 @@ export type TraceSearchMeta = { error_signatures_json?: string; }; +/** + * Narrow row shape returned by {@link listDedupRowsForEpisode}. Includes + * exactly the fields the capture-side dedup path needs (see + * `traceIdentitySignature` / `runLite` / `runLightweight` in + * `core/capture/capture.ts`). Deliberately excludes the two big BLOB + * columns (`vec_summary`, `vec_action`) so an episode with 500k rows + * can be scanned without pulling ~4 GB of embeddings into JS memory — + * the root pathology in #2076. + */ +export interface TraceDedupRow { + ts: number; + turnId: number; + userText: string; + agentText: string; + toolCalls: ToolCallDTO[]; +} + +interface RawDedupRow { + ts: number; + turn_id: number; + user_text: string; + agent_text: string; + tool_calls_json: string; +} + +const DEDUP_COLUMNS = [ + "ts", + "turn_id", + "user_text", + "agent_text", + "tool_calls_json", +] as const; + export function makeTracesRepo(db: StorageDb) { const insert = db.prepare(buildInsert({ table: "traces", columns: COLUMNS })); const upsert = db.prepare( @@ -257,6 +291,39 @@ export function makeTracesRepo(db: StorageDb) { return rows.map(mapRow); }, + /** + * Streaming, narrow-projection sibling of {@link listAllForEpisode}. + * + * Every capture-side dedup call-site (see `runLite`, `runLightweight`, + * `persistRows` in `core/capture/capture.ts`) needs only the five + * dedup identity fields — never the `vec_summary` / `vec_action` + * BLOBs which dominate row size. This helper projects exactly those + * columns and streams via `.iterate()` so peak memory scales with + * the scalar payload, not the total embedding footprint. + * + * Same episode-scoping / `ts ASC` ordering contract as + * `listAllForEpisode`; the two are drop-in siblings for callers that + * only need dedup identity. + */ + listDedupRowsForEpisode(episodeId: EpisodeId | string): TraceDedupRow[] { + if (!episodeId) return []; + const sql = `SELECT ${DEDUP_COLUMNS.join( + ", ", + )} FROM traces WHERE episode_id = @episode_id ORDER BY ts ASC`; + const stmt = db.prepare<{ episode_id: string }, RawDedupRow>(sql); + const rows: TraceDedupRow[] = []; + for (const r of stmt.iterate({ episode_id: String(episodeId) })) { + rows.push({ + ts: r.ts, + turnId: r.turn_id, + userText: r.user_text, + agentText: r.agent_text, + toolCalls: fromJsonText(r.tool_calls_json, []), + }); + } + return rows; + }, + /** * Total row count matching the same filter (no limit/offset). * Used by list endpoints so the viewer can show "Page N of M". diff --git a/apps/memos-local-plugin/core/storage/vector.ts b/apps/memos-local-plugin/core/storage/vector.ts index 80025a33e..06bb9c450 100644 --- a/apps/memos-local-plugin/core/storage/vector.ts +++ b/apps/memos-local-plugin/core/storage/vector.ts @@ -252,22 +252,38 @@ export function scanAndTopK( for (const r of iter) { const vec = decodeVector(r[vecColumn] as Buffer | null); if (!vec) continue; - if (vec.length === 0 || vec.length !== query.length) continue; + if (vec.length === 0 || vec.length !== query.length) { + // Non-empty vectors whose length disagrees with the query are the + // canonical "schema drift" signal — usually a re-embedding pass + // switched model dimensions and stale rows lingered. The old + // `topKCosine` path logged this exact case (see `topKCosine` + // above); the streaming rewrite must preserve the signal so + // operators can still detect silent under-recall in production. + if (vec.length !== 0) { + log.warn("search.dim_mismatch", { + expected: query.length, + got: vec.length, + rowId: String(r["id"]), + }); + } + continue; + } const rowNorm2 = norm2Column ? (r[norm2Column] as number | null | undefined) ?? norm2(vec) : norm2(vec); const score = cosinePrenormed(query, qNorm, vec, rowNorm2); - if (heap.length < k) { - const meta = selectExtra.length > 0 + // Build meta once — the two heap-push branches below both need it + // when the row is kept, and rebuilding it in each branch would + // duplicate the projection logic and drift over time. + const buildMeta = (): TMeta => + selectExtra.length > 0 ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) : (undefined as TMeta); - heap.push({ id: String(r["id"]), score, meta }); + if (heap.length < k) { + heap.push({ id: String(r["id"]), score, meta: buildMeta() }); siftUp(heap, heap.length - 1); } else if (score > heap[0]!.score) { - const meta = selectExtra.length > 0 - ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) - : (undefined as TMeta); - heap[0] = { id: String(r["id"]), score, meta }; + heap[0] = { id: String(r["id"]), score, meta: buildMeta() }; siftDown(heap, 0); } } diff --git a/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts index 22943f0d1..2730191ca 100644 --- a/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts @@ -124,3 +124,127 @@ describe("traces.listAllForEpisode — uncapped episode fetch (#2076)", () => { expect(rows.map((r) => r.ts)).toEqual([100, 200, 300]); }); }); + +describe("traces.listDedupRowsForEpisode — narrow-projection dedup helper (#2077 OCR)", () => { + let db: Database.Database; + let repo: ReturnType; + + beforeEach(() => { + db = new Database(":memory:"); + db.exec(` + CREATE TABLE traces ( + id TEXT PRIMARY KEY, + episode_id TEXT, + session_id TEXT NOT NULL, + owner_agent_kind TEXT, + owner_profile_id TEXT, + owner_workspace_id TEXT, + ts INTEGER NOT NULL, + user_text TEXT, + agent_text TEXT, + summary TEXT, + tool_calls_json TEXT, + reflection TEXT, + agent_thinking TEXT, + value REAL NOT NULL DEFAULT 0, + alpha REAL NOT NULL DEFAULT 0, + r_human REAL, + priority REAL NOT NULL DEFAULT 0, + tags_json TEXT, + error_signatures_json TEXT, + vec_summary BLOB, + vec_action BLOB, + share_scope TEXT, + share_target TEXT, + shared_at INTEGER, + turn_id INTEGER NOT NULL DEFAULT 0, + schema_version INTEGER NOT NULL DEFAULT 1 + ); + CREATE INDEX idx_traces_episode_ts_dedup ON traces(episode_id, ts); + `); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + repo = makeTracesRepo(db as any); + }); + + function makeTrace( + overrides: Partial & { id: string; ts: number; turnId: number }, + ): TraceRow { + return { + id: overrides.id, + episodeId: overrides.episodeId ?? "ep-dedup", + sessionId: overrides.sessionId ?? "sess-1", + ts: overrides.ts, + userText: overrides.userText ?? `user ${overrides.id}`, + agentText: overrides.agentText ?? `agent ${overrides.id}`, + summary: overrides.summary ?? null, + toolCalls: overrides.toolCalls ?? [], + value: overrides.value ?? 0, + alpha: overrides.alpha ?? 0, + priority: overrides.priority ?? 0, + tags: overrides.tags ?? [], + errorSignatures: overrides.errorSignatures ?? [], + turnId: overrides.turnId, + schemaVersion: overrides.schemaVersion ?? 1, + }; + } + + it("returns only the five dedup fields — no vecSummary / vecAction / tags / summary", () => { + // Populate an artificial 8 KB BLOB into vec_summary / vec_action so + // any accidental full-column projection would show up in memory tests. + const bigVec = new Float32Array(1024).fill(0.5); + repo.insert( + makeTrace({ + id: "t-1", + ts: 100, + turnId: 1, + toolCalls: [{ name: "get", input: { url: "https://example.com" } }], + }), + ); + // Poke a vector directly so the row has a heavy payload we can prove + // the narrow helper does NOT hydrate. + repo.updateVector("t-1", "vecSummary", bigVec); + repo.updateVector("t-1", "vecAction", bigVec); + + const rows = repo.listDedupRowsForEpisode("ep-dedup"); + expect(rows.length).toBe(1); + const [row] = rows; + // Just the five dedup fields — nothing more. + expect(Object.keys(row!).sort()).toEqual( + ["agentText", "toolCalls", "ts", "turnId", "userText"].sort(), + ); + expect(row!.toolCalls).toEqual([ + { name: "get", input: { url: "https://example.com" } }, + ]); + }); + + it("returns [] for empty episode / missing episodeId", () => { + expect(repo.listDedupRowsForEpisode("nope")).toEqual([]); + expect(repo.listDedupRowsForEpisode("")).toEqual([]); + }); + + it("streams every trace for an episode past the 500-row paginated cap", () => { + // Same regression guard as `listAllForEpisode` — the narrow path + // must also be uncapped, or callers get silent under-dedup. + const N = 750; + for (let i = 0; i < N; i++) { + repo.insert(makeTrace({ id: `t-${i}`, ts: 1_000 + i, turnId: i })); + } + const rows = repo.listDedupRowsForEpisode("ep-dedup"); + expect(rows.length).toBe(N); + expect(rows[0]!.ts).toBe(1_000); + expect(rows[N - 1]!.ts).toBe(1_000 + N - 1); + }); + + it("scopes strictly by episode and orders by ts ASC — matches listAllForEpisode contract", () => { + repo.insert(makeTrace({ id: "b1", episodeId: "ep-b", ts: 200, turnId: 1 })); + repo.insert(makeTrace({ id: "a-third", ts: 300, turnId: 3 })); + repo.insert(makeTrace({ id: "a-first", ts: 100, turnId: 1 })); + repo.insert(makeTrace({ id: "a-second", ts: 200, turnId: 2 })); + + const a = repo.listDedupRowsForEpisode("ep-dedup"); + expect(a.map((r) => r.ts)).toEqual([100, 200, 300]); + const b = repo.listDedupRowsForEpisode("ep-b"); + expect(b.length).toBe(1); + expect(b[0]!.ts).toBe(200); + }); +}); diff --git a/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts b/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts index 65f1e5b33..590691c23 100644 --- a/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/vector-stream.test.ts @@ -18,9 +18,10 @@ * not the old 100_000 default. */ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import Database from "better-sqlite3"; import { encodeVector, scanAndTopK } from "../../../core/storage/index.js"; +import { memoryBuffer, rootLogger } from "../../../core/logger/index.js"; function vec(arr: number[]): Float32Array { return new Float32Array(arr); @@ -136,4 +137,93 @@ describe("scanAndTopK — streaming rewrite (#2076)", () => { db.close(); } }); + + it("logs `search.dim_mismatch` on a non-empty vector whose dimension disagrees with the query (#2077 OCR)", () => { + // Regression guard for the OCR observation: the streaming rewrite + // originally dropped mismatched rows silently, whereas the old + // topKCosine path emitted `search.dim_mismatch`. Losing that signal + // means operators can't detect schema drift (re-embedding at a new + // model dimension) — the function just returns fewer results with + // no warning. + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + // r-good matches the 3-dim query; r-drift is 4-dim (schema drift). + insert.run("r-good", encodeVector(vec([1, 0, 0])), null); + insert.run("r-drift", encodeVector(vec([0.1, 0.2, 0.3, 0.4])), null); + + const warnSpy = vi.spyOn(rootLogger, "warn"); + const query = vec([1, 0, 0]); + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + query, + 5, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + + // Only the aligned row is returned; the drift row is dropped. + expect(hits.map((h) => h.id)).toEqual(["r-good"]); + + // The observable operator signal — either a direct warn on the + // root logger or a warn record in the memory buffer — must fire + // for the drift row. + const bufWarn = memoryBuffer() + .tail({ limit: 100 }) + .find( + (r) => + r.msg === "search.dim_mismatch" && + (r.data as Record | undefined)?.rowId === "r-drift", + ); + const spyWarn = warnSpy.mock.calls.some( + ([msg, data]) => + msg === "search.dim_mismatch" && + (data as Record | undefined)?.rowId === "r-drift", + ); + expect(Boolean(bufWarn) || spyWarn).toBe(true); + warnSpy.mockRestore(); + } finally { + db.close(); + } + }); + + it("silently skips zero-length vector rows (empty embeddings are legitimate — not schema drift)", () => { + const db = openTinyVecDb(); + try { + const insert = db.prepare("INSERT INTO bench (id, vec, extra) VALUES (?, ?, ?)"); + insert.run("r-good", encodeVector(vec([1, 0, 0])), null); + insert.run("r-empty", encodeVector(vec([])), null); + + const warnSpy = vi.spyOn(rootLogger, "warn"); + const hits = scanAndTopK( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + db as any, + "bench", + [], + vec([1, 0, 0]), + 5, + { vecColumn: "vec", where: "vec IS NOT NULL" }, + ); + expect(hits.map((h) => h.id)).toEqual(["r-good"]); + // Zero-length rows are drop-without-warn — see vector.ts comment. + const spyDrift = warnSpy.mock.calls.some( + ([msg, data]) => + msg === "search.dim_mismatch" && + (data as Record | undefined)?.rowId === "r-empty", + ); + const bufDrift = memoryBuffer() + .tail({ limit: 100 }) + .some( + (r) => + r.msg === "search.dim_mismatch" && + (r.data as Record | undefined)?.rowId === "r-empty", + ); + expect(spyDrift || bufDrift).toBe(false); + warnSpy.mockRestore(); + } finally { + db.close(); + } + }); }); From 98f779cc482670b347237728d3773f895ab1a380 Mon Sep 17 00:00:00 2001 From: MemOS AutoDev Date: Wed, 8 Jul 2026 18:31:15 +0800 Subject: [PATCH 3/4] refactor(local-plugin): apply #2077 open code review fixes (round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-pass open-code-review fixes on top of 3e25bc33. All three are minimal-diff / non-behavior-changing. 1. `vector.ts::scanAndTopK` was building the `buildMeta` closure inside the per-row streaming loop. On a hot search over the full `hardCap` this allocated one closure per iteration — pure GC pressure since only the two heap-push branches consume it. Hoisted the projection above the loop as a plain function taking the current row as an argument. Two variants pre-selected once based on `selectExtra.length` so filtered-out rows still pay nothing. 2. `traces.ts::listDedupRowsForEpisode` JSDoc claimed the helper "streams via `.iterate()` so peak memory scales with the scalar payload". That was misleading: the method uses `stmt.iterate()` internally but still accumulates every row into a `TraceDedupRow[]` before returning. The genuine saving is the narrow scalar-only projection (never touches `vec_summary` / `vec_action`), not streaming. Rewrote the JSDoc to describe the real cost model and name the `.iterate()` benefit accurately (avoids an intermediate `.all()` allocation, does not stream to caller). 3. `traces.ts::listAllForEpisode` JSDoc said "all hot fields required by dedup are projected, so a caller that only needs `ts`, `turnId`, or the identity signature can still iterate at full speed". That accidentally invited dedup callers back onto this method, which loads every BLOB column. Rewrote the JSDoc to state explicitly that vec_summary/vec_action BLOBs are loaded and redirect dedup-only callers to `listDedupRowsForEpisode`. Preserved the #2076 context on why the uncapped variant exists. No behavioral change. No test changes needed. Verification - `npx tsc -p tsconfig.json --noEmit` → clean. - `npx vitest run tests/unit/storage/traces-listall.test.ts \ tests/unit/storage/vector-stream.test.ts` → 13/13 pass. - `npx vitest run tests/unit/capture` → 110/110 pass. Co-Authored-By: Claude Opus 4.7 --- .../core/storage/repos/traces.ts | 60 ++++++++++++------- .../memos-local-plugin/core/storage/vector.ts | 23 ++++--- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index 106b6cb64..189fb30e0 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -261,22 +261,27 @@ export function makeTracesRepo(db: StorageDb) { /** * Full episode-scoped trace fetch with NO pagination cap. * - * The paginated `list({ episodeId })` path silently truncates to - * `PageOptions.limit`, which caps at 500 by default. That cap - * breaks capture-side dedup (#2076): when an episode grows past - * the cap, the next runLite / runReflect only sees the newest - * 500 rows, treats every older step as "novel", and re-inserts - * the whole tail every cycle. In the reporter's 4.2 GB / 6.8 GB - * failure, 518,375 trace rows had shrunk to 80,583 distinct + * Fetches ALL columns including the large `vec_summary` and + * `vec_action` BLOB columns, mapped into full `TraceRow` shape. + * Use this only when the caller genuinely needs those BLOBs + * (e.g. `runReflect`, which re-embeds and rewrites every field). + * + * **For dedup-only reads** — where only `ts`, `turnId`, + * `userText`, `agentText`, and `toolCalls` are needed — use + * {@link listDedupRowsForEpisode} instead to avoid loading + * multi-GB of embeddings into JS memory. + * + * Why an uncapped read exists at all: the paginated + * `list({ episodeId })` path silently truncates to + * `PageOptions.limit` (default 500). That cap breaks capture-side + * dedup (#2076): when an episode grows past the cap, the next + * runLite / runReflect only sees the newest 500 rows, treats + * every older step as "novel", and re-inserts the whole tail + * every cycle. In the reporter's 4.2 GB / 6.8 GB failure, + * 518,375 trace rows had shrunk to 80,583 distinct * `(episode_id, turn_id, user_text, agent_text, tool_calls_json)` * signatures — 84 % duplicates driven by exactly this loop. * - * Use this helper for any dedup / reconciliation read that must - * see the whole episode. All hot fields required by dedup are - * projected, so a caller that only needs `ts`, `turnId`, or the - * identity signature can still iterate at full speed without - * paying the paginated round trips. - * * Rows are ordered by `ts ASC` so the causal chain matches the * order runLite / runReflect built. */ @@ -292,18 +297,29 @@ export function makeTracesRepo(db: StorageDb) { }, /** - * Streaming, narrow-projection sibling of {@link listAllForEpisode}. + * Narrow-projection episode fetch for capture-side dedup. + * + * Sibling to {@link listAllForEpisode}, but projects only the five + * scalar dedup-identity columns + * (`ts` / `turn_id` / `user_text` / `agent_text` / `tool_calls_json`) + * and NEVER touches `vec_summary` / `vec_action`. That is the real + * saving: on the reporter's DB in #2076 the two BLOB columns + * dominated row size, so skipping them cuts per-row bytes by + * ~1000× regardless of how many rows the episode contains. + * + * Uses `stmt.iterate()` internally to avoid better-sqlite3's + * intermediate `.all()` allocation, but the result is still + * materialised into a `TraceDedupRow[]` and returned to the + * caller — this is not a streaming API. Peak JS memory therefore + * scales linearly with row count × the small scalar payload, + * which is what capture-side dedup actually needs. * - * Every capture-side dedup call-site (see `runLite`, `runLightweight`, - * `persistRows` in `core/capture/capture.ts`) needs only the five - * dedup identity fields — never the `vec_summary` / `vec_action` - * BLOBs which dominate row size. This helper projects exactly those - * columns and streams via `.iterate()` so peak memory scales with - * the scalar payload, not the total embedding footprint. + * Every capture-side dedup call-site (`runLite`, `runLightweight`, + * `persistRows` in `core/capture/capture.ts`) uses this helper + * so no BLOBs load during the dedup pass. * * Same episode-scoping / `ts ASC` ordering contract as - * `listAllForEpisode`; the two are drop-in siblings for callers that - * only need dedup identity. + * `listAllForEpisode`. */ listDedupRowsForEpisode(episodeId: EpisodeId | string): TraceDedupRow[] { if (!episodeId) return []; diff --git a/apps/memos-local-plugin/core/storage/vector.ts b/apps/memos-local-plugin/core/storage/vector.ts index 06bb9c450..4aa27269d 100644 --- a/apps/memos-local-plugin/core/storage/vector.ts +++ b/apps/memos-local-plugin/core/storage/vector.ts @@ -243,6 +243,18 @@ export function scanAndTopK( const qNorm = Math.sqrt(norm2(query)); if (qNorm === 0) return []; + // Build meta once per row — hoisted out of the loop so the meta + // projection is *not* re-allocated as a closure per iteration. The + // two heap-push branches below both need this when the row is kept, + // and rebuilding the closure in each iteration was pure GC pressure + // for hot searches. Only invoked for the heap-push branches, so + // filtered-out rows pay nothing. + const buildMeta = + selectExtra.length > 0 + ? (row: ScanRow): TMeta => + Object.fromEntries(selectExtra.map((c) => [c, row[c]])) as TMeta + : (_row: ScanRow): TMeta => undefined as TMeta; + // Streaming top-K: min-heap of size k. We *never* build the full // candidate array — one BLOB lives on the JS heap at a time (plus // the k already-selected vectors we're comparing against). @@ -272,18 +284,11 @@ export function scanAndTopK( ? (r[norm2Column] as number | null | undefined) ?? norm2(vec) : norm2(vec); const score = cosinePrenormed(query, qNorm, vec, rowNorm2); - // Build meta once — the two heap-push branches below both need it - // when the row is kept, and rebuilding it in each branch would - // duplicate the projection logic and drift over time. - const buildMeta = (): TMeta => - selectExtra.length > 0 - ? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta) - : (undefined as TMeta); if (heap.length < k) { - heap.push({ id: String(r["id"]), score, meta: buildMeta() }); + heap.push({ id: String(r["id"]), score, meta: buildMeta(r) }); siftUp(heap, heap.length - 1); } else if (score > heap[0]!.score) { - heap[0] = { id: String(r["id"]), score, meta: buildMeta() }; + heap[0] = { id: String(r["id"]), score, meta: buildMeta(r) }; siftDown(heap, 0); } } From 240a354baa1554d95de19a30069aedc52a795362 Mon Sep 17 00:00:00 2001 From: jiachengzhen Date: Thu, 9 Jul 2026 14:21:59 +0800 Subject: [PATCH 4/4] fix: avoid #2077 capture merge conflict --- .../memos-local-plugin/core/capture/capture.ts | 11 ++--------- .../core/storage/repos/traces.ts | 8 +++++++- .../tests/unit/storage/traces-listall.test.ts | 18 ++++++++++++++++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/apps/memos-local-plugin/core/capture/capture.ts b/apps/memos-local-plugin/core/capture/capture.ts index c48c3dda0..73e975a47 100644 --- a/apps/memos-local-plugin/core/capture/capture.ts +++ b/apps/memos-local-plugin/core/capture/capture.ts @@ -404,9 +404,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { // Pair each normalized step with its already-persisted trace row // (matched by ts). If runLite was skipped for any step, fall back // to a fresh insert path so we don't lose data. - // #2076: uncapped dedup read — reflect must see every trace we - // ever wrote for this episode, not just the newest 500. - const existing = deps.tracesRepo.listAllForEpisode(input.episode.id); + const existing = deps.tracesRepo.list({ episodeId: input.episode.id }); const traceByTs = new Map(); for (const tr of existing) traceByTs.set(tr.ts, tr); const orphan = normalized.filter((s) => !traceByTs.has(s.ts)); @@ -436,12 +434,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner { })); const { vecs } = await runEmbed(orphanScored, summaries, warnings); const orphanRows = buildRows(orphanScored, summaries, vecs, input.episode); - // Reuse the `existing` scan we already ran above — the orphan - // persistRows only needs the signature Set, and re-running - // `listAllForEpisode` here (or its narrow sibling) would be a - // duplicate full-episode scan for no additional information. - const seenSignatures = new Set(existing.map(traceIdentitySignature)); - await persistRows(orphanRows, input, warnings, {}, seenSignatures); + await persistRows(orphanRows, input, warnings); for (const r of orphanRows) traceByTs.set(r.ts, r); } diff --git a/apps/memos-local-plugin/core/storage/repos/traces.ts b/apps/memos-local-plugin/core/storage/repos/traces.ts index 189fb30e0..894552b56 100644 --- a/apps/memos-local-plugin/core/storage/repos/traces.ts +++ b/apps/memos-local-plugin/core/storage/repos/traces.ts @@ -253,7 +253,13 @@ export function makeTracesRepo(db: StorageDb) { } if (tr.sql) fragments.push(tr.sql); const where = joinWhere(fragments); - const page = buildPageClauses(filter, "ts"); + const shouldUseUncappedEpisodeScan = + Boolean(filter.episodeId) && + filter.limit === undefined && + filter.offset === undefined; + const page = shouldUseUncappedEpisodeScan + ? `ORDER BY ts ${filter.newestFirst === false ? "ASC" : "DESC"}` + : buildPageClauses(filter, "ts"); const sql = `SELECT ${COLUMNS.join(", ")} FROM traces ${where} ${page}`; return db.prepare(sql).all(params).map(mapRow); }, diff --git a/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts index 2730191ca..7d0e3839f 100644 --- a/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/traces-listall.test.ts @@ -5,8 +5,9 @@ * Root cause: capture.ts dedup callers used `tracesRepo.list({ episodeId })`, * which is paginated and silently truncates to 500 rows. Any episode with * more than 500 rows would see the older rows re-inserted as "novel" every - * runLite/runReflect cycle. The fix is `listAllForEpisode(episodeId)`, an - * uncapped read used only for dedup/reconciliation. + * runLite/runReflect cycle. The fix is uncapped episode reconciliation reads + * (`listAllForEpisode(episodeId)` and the legacy `list({ episodeId })` path + * when no explicit page was requested). * * These tests pin the new contract: give me EVERY row for the episode, no * matter how large it grew, so callers can compute the full "already seen" @@ -99,6 +100,19 @@ describe("traces.listAllForEpisode — uncapped episode fetch (#2076)", () => { expect(tsSet.has(1_000 + N - 1)).toBe(true); }); + it("keeps legacy episode-only list() calls uncapped unless the caller asks for a page", () => { + const N = 750; + for (let i = 0; i < N; i++) { + repo.insert(makeTrace({ id: `t-${i}`, ts: 1_000 + i, turnId: i })); + } + + const unpaged = repo.list({ episodeId: "ep-huge" }); + expect(unpaged.length).toBe(N); + + const paged = repo.list({ episodeId: "ep-huge", limit: 25 }); + expect(paged.length).toBe(25); + }); + it("scopes strictly by episodeId — other episodes' rows are excluded", () => { repo.insert(makeTrace({ id: "a1", episodeId: "ep-a", ts: 100, turnId: 1 })); repo.insert(makeTrace({ id: "a2", episodeId: "ep-a", ts: 101, turnId: 2 }));