Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 38 additions & 9 deletions apps/memos-local-plugin/core/capture/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,17 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner {
// ─── Extract + dedup (skip steps we've already written this episode) ──
const extractStart = now();
const rawAll = extractSteps(input.episode);
const existingTraces = deps.tracesRepo.list({ episodeId: input.episode.id });
const seenTs = new Set<number>(existingTraces.map((t) => t.ts));
// #2076: MUST use listDedupRowsForEpisode (uncapped, streaming, no
// BLOB projection). The paginated `list` path silently truncates to
// 500 rows, which breaks dedup once an episode grows past that and
// causes the tail to be re-inserted every cycle (bloating `traces`
// unboundedly + starving the vector scan). Using the narrow-column
// dedup projection here also keeps peak RSS proportional to scalar
// fields, not embedding footprint (open code review on #2077).
const existingDedupRows = deps.tracesRepo.listDedupRowsForEpisode(input.episode.id);
const seenTs = new Set<number>(existingDedupRows.map((t) => t.ts));
// Reused later by persistRows so we don't scan the episode twice.
const seenSignatures = new Set(existingDedupRows.map(traceIdentitySignature));
const raw = rawAll.filter((s) => !seenTs.has(s.ts));
const extractMs = now() - extractStart;
log.debug("stage.extract.done", {
Expand Down Expand Up @@ -183,7 +192,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner {
// Persist as new rows. Reflection / α deliberately empty.
const persistStart = now();
const rows = buildRows(scored, summaries, vecs, input.episode);
const persisted = await persistRows(rows, input, warnings);
const persisted = await persistRows(rows, input, warnings, {}, seenSignatures);
if (!persisted) {
// emit capture.failed handled inside persistRows on hard fail.
return finalResult(
Expand Down Expand Up @@ -251,12 +260,16 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner {

const extractStart = now();
const rawAll = extractSteps(input.episode);
const existingTraces = deps.tracesRepo.list({ episodeId: input.episode.id });
// #2076 + #2077 OCR: uncapped, narrow-projection dedup read.
// See `runLite` for the full rationale — one scan, no BLOBs.
const existingDedupRows = deps.tracesRepo.listDedupRowsForEpisode(input.episode.id);
const seenTurnIds = new Set(
existingTraces
existingDedupRows
.map((t) => t.turnId)
.filter((v): v is number => typeof v === "number" && Number.isFinite(v)),
);
// Reused later by persistRows so we don't scan the episode twice.
const seenSignatures = new Set(existingDedupRows.map(traceIdentitySignature));
const rawByTurn = new Map<number, StepCandidate[]>();
for (const step of rawAll) {
const turnId = pickTurnId(step.meta, step.ts);
Expand Down Expand Up @@ -308,7 +321,7 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner {
});
const persisted = await persistRows(rows, input, warnings, {
skipActionVectorRetry: true,
});
}, seenSignatures);
if (!persisted) {
return finalResult(
input,
Expand Down Expand Up @@ -768,9 +781,23 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner {
input: CaptureInput,
warnings: CaptureResult["warnings"],
opts: { skipActionVectorRetry?: boolean } = {},
existingSignatures?: Set<string>,
): Promise<boolean> {
const existingBeforeInsert = deps.tracesRepo.list({ episodeId: input.episode.id });
const seenSignatures = new Set(existingBeforeInsert.map(traceIdentitySignature));
// #2076 + #2077 OCR: uncapped, narrow-projection dedup read. The
// paginated `list` path missed all rows past the 500 cap and let
// duplicate signatures re-insert every cycle. When the caller has
// already computed the signature set upstream (runLite /
// runLightweight / runReflect all do), skip the second full scan
// and reuse it — otherwise fall back to a fresh streaming scan.
// We clone the caller-supplied set so the intra-batch dedup below
// doesn't leak new signatures back into the caller's Set instance.
const seenSignatures = existingSignatures
? new Set(existingSignatures)
: new Set(
deps.tracesRepo
.listDedupRowsForEpisode(input.episode.id)
.map(traceIdentitySignature),
);
const uniqueRows = rows.filter((row) => {
const signature = traceIdentitySignature(row);
if (seenSignatures.has(signature)) return false;
Expand Down Expand Up @@ -891,7 +918,9 @@ export function createCaptureRunner(deps: CaptureDeps): CaptureRunner {
return ["user", turnId, step.ts, step.userText.trim()].join("\x1f");
}

function traceIdentitySignature(row: TraceRow): string {
function traceIdentitySignature(
row: Pick<TraceRow, "toolCalls" | "turnId" | "ts" | "agentText" | "userText">,
): string {
const tool = row.toolCalls[0];
if (tool) {
const hasRealTiming =
Expand Down
124 changes: 123 additions & 1 deletion apps/memos-local-plugin/core/storage/repos/traces.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { ToolCallDTO } from "../../../agent-contract/dto.js";
import type { EmbeddingVector, EpisodeId, SessionId, ShareScope, TraceId, TraceRow } from "../../types.js";
import type { StorageDb, TraceListFilter } from "../types.js";
import { buildInClause, buildInsert, buildUpdate } from "../tx.js";
Expand Down Expand Up @@ -58,6 +59,39 @@ export type TraceSearchMeta = {
error_signatures_json?: string;
};

/**
* Narrow row shape returned by {@link listDedupRowsForEpisode}. Includes
* exactly the fields the capture-side dedup path needs (see
* `traceIdentitySignature` / `runLite` / `runLightweight` in
* `core/capture/capture.ts`). Deliberately excludes the two big BLOB
* columns (`vec_summary`, `vec_action`) so an episode with 500k rows
* can be scanned without pulling ~4 GB of embeddings into JS memory —
* the root pathology in #2076.
*/
export interface TraceDedupRow {
ts: number;
turnId: number;
userText: string;
agentText: string;
toolCalls: ToolCallDTO[];
}

interface RawDedupRow {
ts: number;
turn_id: number;
user_text: string;
agent_text: string;
tool_calls_json: string;
}

const DEDUP_COLUMNS = [
"ts",
"turn_id",
"user_text",
"agent_text",
"tool_calls_json",
] as const;

export function makeTracesRepo(db: StorageDb) {
const insert = db.prepare(buildInsert({ table: "traces", columns: COLUMNS }));
const upsert = db.prepare(
Expand Down Expand Up @@ -219,11 +253,99 @@ export function makeTracesRepo(db: StorageDb) {
}
if (tr.sql) fragments.push(tr.sql);
const where = joinWhere(fragments);
const page = buildPageClauses(filter, "ts");
const shouldUseUncappedEpisodeScan =
Boolean(filter.episodeId) &&
filter.limit === undefined &&
filter.offset === undefined;
const page = shouldUseUncappedEpisodeScan
? `ORDER BY ts ${filter.newestFirst === false ? "ASC" : "DESC"}`
: buildPageClauses(filter, "ts");
const sql = `SELECT ${COLUMNS.join(", ")} FROM traces ${where} ${page}`;
return db.prepare<typeof params, RawTraceRow>(sql).all(params).map(mapRow);
},

/**
* Full episode-scoped trace fetch with NO pagination cap.
*
* Fetches ALL columns including the large `vec_summary` and
* `vec_action` BLOB columns, mapped into full `TraceRow` shape.
* Use this only when the caller genuinely needs those BLOBs
* (e.g. `runReflect`, which re-embeds and rewrites every field).
*
* **For dedup-only reads** — where only `ts`, `turnId`,
* `userText`, `agentText`, and `toolCalls` are needed — use
* {@link listDedupRowsForEpisode} instead to avoid loading
* multi-GB of embeddings into JS memory.
*
* Why an uncapped read exists at all: the paginated
* `list({ episodeId })` path silently truncates to
* `PageOptions.limit` (default 500). That cap breaks capture-side
* dedup (#2076): when an episode grows past the cap, the next
* runLite / runReflect only sees the newest 500 rows, treats
* every older step as "novel", and re-inserts the whole tail
* every cycle. In the reporter's 4.2 GB / 6.8 GB failure,
* 518,375 trace rows had shrunk to 80,583 distinct
* `(episode_id, turn_id, user_text, agent_text, tool_calls_json)`
* signatures — 84 % duplicates driven by exactly this loop.
*
* Rows are ordered by `ts ASC` so the causal chain matches the
* order runLite / runReflect built.
*/
listAllForEpisode(episodeId: EpisodeId | string): TraceRow[] {
if (!episodeId) return [];
const sql = `SELECT ${COLUMNS.join(
", ",
)} FROM traces WHERE episode_id = @episode_id ORDER BY ts ASC`;
const rows = db
.prepare<{ episode_id: string }, RawTraceRow>(sql)
.all({ episode_id: String(episodeId) });
return rows.map(mapRow);
},

/**
* Narrow-projection episode fetch for capture-side dedup.
*
* Sibling to {@link listAllForEpisode}, but projects only the five
* scalar dedup-identity columns
* (`ts` / `turn_id` / `user_text` / `agent_text` / `tool_calls_json`)
* and NEVER touches `vec_summary` / `vec_action`. That is the real
* saving: on the reporter's DB in #2076 the two BLOB columns
* dominated row size, so skipping them cuts per-row bytes by
* ~1000× regardless of how many rows the episode contains.
*
* Uses `stmt.iterate()` internally to avoid better-sqlite3's
* intermediate `.all()` allocation, but the result is still
* materialised into a `TraceDedupRow[]` and returned to the
* caller — this is not a streaming API. Peak JS memory therefore
* scales linearly with row count × the small scalar payload,
* which is what capture-side dedup actually needs.
*
* Every capture-side dedup call-site (`runLite`, `runLightweight`,
* `persistRows` in `core/capture/capture.ts`) uses this helper
* so no BLOBs load during the dedup pass.
*
* Same episode-scoping / `ts ASC` ordering contract as
* `listAllForEpisode`.
*/
listDedupRowsForEpisode(episodeId: EpisodeId | string): TraceDedupRow[] {
if (!episodeId) return [];
const sql = `SELECT ${DEDUP_COLUMNS.join(
", ",
)} FROM traces WHERE episode_id = @episode_id ORDER BY ts ASC`;
const stmt = db.prepare<{ episode_id: string }, RawDedupRow>(sql);
const rows: TraceDedupRow[] = [];
for (const r of stmt.iterate({ episode_id: String(episodeId) })) {
rows.push({
ts: r.ts,
turnId: r.turn_id,
userText: r.user_text,
agentText: r.agent_text,
toolCalls: fromJsonText<ToolCallDTO[]>(r.tool_calls_json, []),
});
}
return rows;
},

/**
* Total row count matching the same filter (no limit/offset).
* Used by list endpoints so the viewer can show "Page N of M".
Expand Down
90 changes: 76 additions & 14 deletions apps/memos-local-plugin/core/storage/vector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,30 @@ export interface ScanRow {
[k: string]: unknown;
}

/**
* Default LIMIT applied when the caller doesn't pass an explicit
* `hardCap`. Historically 100_000, which — combined with the old
* `.all()` materialisation — meant one accidental "no cap" call
* could pull 100k multi-KB vector BLOBs into JS memory in a single
* synchronous step. On the reporter's DB in issue #2076 that
* translated to 4.2 GB RSS and a 100 % CPU main-thread stall.
*
* 5_000 is a much safer default: it still covers realistic per-agent
* corpora and it forces callers that legitimately need more to opt
* in explicitly.
*/
export const DEFAULT_SCAN_HARD_CAP = 5_000;

/**
* Stream rows from `table`, decode vectors, and run top-K cosine against
* `query`. `selectExtra` lets callers bring along columns that will surface in
* `VectorHit.meta`.
*
* Streaming: we use `.iterate()` (not `.all()`) so at most one row's
* BLOB is decoded at a time. The top-K min-heap keeps only `k`
* vectors of state, so peak RSS is O(k * dim) regardless of how many
* rows the LIMIT allows. Fixes the "load 100k BLOBs synchronously"
* pathology in #2076.
*/
export function scanAndTopK<TMeta = undefined>(
db: StorageDb,
Expand All @@ -207,30 +227,72 @@ export function scanAndTopK<TMeta = undefined>(
k: number,
opts: VectorScanOptions,
): Array<VectorHit<string, TMeta>> {
if (k <= 0 || query.length === 0) return [];

const { vecColumn, norm2Column, where, params, hardCap } = opts;
const cap = hardCap ?? DEFAULT_SCAN_HARD_CAP;
const cols = ["id", vecColumn, ...(norm2Column ? [norm2Column] : []), ...selectExtra];
const sql = [
`SELECT ${cols.join(", ")} FROM ${table}`,
where ? `WHERE ${where}` : "",
`LIMIT ${hardCap ?? 100000}`,
`LIMIT ${cap}`,
]
.filter(Boolean)
.join(" ");

const rows = db.prepare<typeof params, ScanRow>(sql).all(params);
const decoded: Array<VectorRow<string, TMeta>> = [];
for (const r of rows) {
const qNorm = Math.sqrt(norm2(query));
if (qNorm === 0) return [];

// Build meta once per row — hoisted out of the loop so the meta
// projection is *not* re-allocated as a closure per iteration. The
// two heap-push branches below both need this when the row is kept,
// and rebuilding the closure in each iteration was pure GC pressure
// for hot searches. Only invoked for the heap-push branches, so
// filtered-out rows pay nothing.
const buildMeta =
selectExtra.length > 0
? (row: ScanRow): TMeta =>
Object.fromEntries(selectExtra.map((c) => [c, row[c]])) as TMeta
: (_row: ScanRow): TMeta => undefined as TMeta;

// Streaming top-K: min-heap of size k. We *never* build the full
// candidate array — one BLOB lives on the JS heap at a time (plus
// the k already-selected vectors we're comparing against).
const heap: Array<VectorHit<string, TMeta>> = [];
const stmt = db.prepare<typeof params, ScanRow>(sql);
const iter = params === undefined ? stmt.iterate() : stmt.iterate(params);
for (const r of iter) {
const vec = decodeVector(r[vecColumn] as Buffer | null);
if (!vec) continue;
const meta = selectExtra.length > 0
? (Object.fromEntries(selectExtra.map((c) => [c, r[c]])) as TMeta)
: (undefined as TMeta);
decoded.push({
id: String(r["id"]),
vec,
norm2: norm2Column ? ((r[norm2Column] as number | null) ?? undefined) : undefined,
meta,
});
if (vec.length === 0 || vec.length !== query.length) {
// Non-empty vectors whose length disagrees with the query are the
// canonical "schema drift" signal — usually a re-embedding pass
// switched model dimensions and stale rows lingered. The old
// `topKCosine` path logged this exact case (see `topKCosine`
// above); the streaming rewrite must preserve the signal so
// operators can still detect silent under-recall in production.
if (vec.length !== 0) {
log.warn("search.dim_mismatch", {
expected: query.length,
got: vec.length,
rowId: String(r["id"]),
});
}
continue;
}
const rowNorm2 = norm2Column
? (r[norm2Column] as number | null | undefined) ?? norm2(vec)
: norm2(vec);
const score = cosinePrenormed(query, qNorm, vec, rowNorm2);
if (heap.length < k) {
heap.push({ id: String(r["id"]), score, meta: buildMeta(r) });
siftUp(heap, heap.length - 1);
} else if (score > heap[0]!.score) {
heap[0] = { id: String(r["id"]), score, meta: buildMeta(r) };
siftDown(heap, 0);
}
}
return topKCosine(query, decoded, k);
// `heap` is a min-heap on score; caller wants DESC.
heap.sort((a, b) => b.score - a.score);
return heap;
}
Loading
Loading