diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index 0db99c787..e51dc656b 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -80,10 +80,12 @@ import type { Logger } from "../logger/types.js"; import { openDb } from "../storage/connection.js"; import { runMigrations } from "../storage/migrator.js"; import { + makeRepos, embeddingMaintenanceCounts, inferStoredEmbeddingByteLen, - makeRepos, + FLOAT32_BYTES, } from "../storage/repos/index.js"; +import type { EmbeddingCountsBucket } from "../storage/repos/index.js"; import { createEmbedder } from "../embedding/embedder.js"; import { createLlmClient } from "../llm/client.js"; import { @@ -116,15 +118,6 @@ import type { UserFeedback } from "../reward/types.js"; const FINAL_HUB_LLM_FILTER_TIMEOUT_MS = 3_000; const IMPORT_WRITE_BATCH_SIZE = 500; -/** - * Float32 byte width. Stored vector BLOBs are little-endian Float32 - * arrays produced by `encodeVector(Float32Array)`; their byte length - * is `dimensions * FLOAT32_BYTES_PER_ELEMENT`. The SQL fast path of - * `computeEmbeddingMaintenanceStats` compares stored BLOB byte length - * against `configuredDimension * FLOAT32_BYTES_PER_ELEMENT`. - */ -const FLOAT32_BYTES_PER_ELEMENT = 4; - export interface BootstrapOptions { agent: AgentKind; namespace?: RuntimeNamespace; @@ -4079,17 +4072,19 @@ export function createMemoryCore( // header, never the payload — so we keep the same per-bucket // semantics without touching a single vector byte. const configuredDimension = handle.embedder?.dimensions ?? 0; - const inferredByteLen = configuredDimension > 0 - ? configuredDimension * FLOAT32_BYTES_PER_ELEMENT + const expectedByteLenFromEmbedder = configuredDimension > 0 + ? configuredDimension * FLOAT32_BYTES + : 0; + // When the embedder has not been probed yet, fall back to the most common + // stored BLOB byte length (mirrors the pre-fix `inferStoredEmbeddingDimension` + // path, but computed via SQL `GROUP BY LENGTH(vec_summary)` — never touches + // the BLOB bodies). + const expectedByteLen = expectedByteLenFromEmbedder > 0 + ? expectedByteLenFromEmbedder : inferStoredEmbeddingByteLen(handle.db); - const dimension = configuredDimension > 0 - ? configuredDimension - : Math.floor(inferredByteLen / FLOAT32_BYTES_PER_ELEMENT); - - const raw = embeddingMaintenanceCounts(handle.db, { - expectedByteLen: inferredByteLen, - }); + const dimension = expectedByteLen > 0 ? expectedByteLen / FLOAT32_BYTES : 0; + const raw = embeddingMaintenanceCounts(handle.db, { expectedByteLen }); const byKind: EmbeddingMaintenanceStats["byKind"] = { trace: addNeedsRepair(raw.trace), policy: addNeedsRepair(raw.policy), @@ -4112,17 +4107,11 @@ export function createMemoryCore( }; } - function addNeedsRepair(bucket: { - totalSlots: number; - ready: number; - missing: number; - dimMismatch: number; - }): EmbeddingMaintenanceStats["byKind"]["trace"] { + function addNeedsRepair( + bucket: EmbeddingCountsBucket, + ): EmbeddingCountsBucket & { needsRepair: number } { return { - totalSlots: bucket.totalSlots, - ready: bucket.ready, - missing: bucket.missing, - dimMismatch: bucket.dimMismatch, + ...bucket, needsRepair: bucket.missing + bucket.dimMismatch, }; } diff --git a/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts b/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts new file mode 100644 index 000000000..65632ff59 --- /dev/null +++ b/apps/memos-local-plugin/core/storage/repos/embedding_maintenance.ts @@ -0,0 +1,204 @@ +/** + * SQL-only embedding maintenance stats. + * + * Regression fix for issue #1929: `/api/v1/embeddings/maintenance` used to + * paginate every trace/policy/world_model/skill row through JS just to + * inspect vector byte lengths, hydrating hundreds of MB of BLOBs into the + * Node heap and blocking the event loop for minutes on production DBs + * (93K traces × 2 vectors × 1536 dims × 4 bytes ≈ 1.1 GB pread64 traffic). + * + * The strategy is a single `SELECT COUNT(*) + SUM(CASE WHEN ...)` per + * `(table, vec column)` pair, using `LENGTH(vec)` for the dimension + * comparison. SQLite's `LENGTH()` on a BLOB column returns the byte length + * from the row header and does not deserialise the buffer, so the maintenance + * call now stays in the same asymptotic ballpark as `SELECT COUNT(*) FROM t`. + * + * The two pre-fix semantic filters are preserved verbatim inside the WHERE + * clauses so per-bucket counts do not shift for already-installed users: + * + * - `shouldTraceHaveEmbeddings` (short-text traces skipped) → SQL + * `LENGTH(TRIM(user_text)) / LENGTH(TRIM(agent_text))` predicates. + * - `isLightweightMemoryTrace` (lightweight traces skip vec_action) → + * `instr(COALESCE(tags_json, ''), '"lightweight_memory"') = 0` + * predicate for the vec_action count only. + */ + +import type { StorageDb } from "../types.js"; + +/** Little-endian Float32 element size. Matches `core/storage/vector.ts`. */ +export const FLOAT32_BYTES = 4; + +export interface EmbeddingCountsBucket { + /** Number of `(row, vec column)` slots included in the bucket. */ + totalSlots: number; + /** + * Vec is non-NULL and either `expectedByteLen === 0` (dimension not + * probed yet) or `LENGTH(vec) === expectedByteLen`. + */ + ready: number; + /** Vec is SQL NULL. */ + missing: number; + /** + * Vec is non-NULL and its byte length ≠ `expectedByteLen` + * (only meaningful when `expectedByteLen > 0`). + */ + dimMismatch: number; +} + +export interface EmbeddingCounts { + trace: EmbeddingCountsBucket; + policy: EmbeddingCountsBucket; + world_model: EmbeddingCountsBucket; + skill: EmbeddingCountsBucket; +} + +interface CountRow { + total: number | null; + missing: number | null; + dim_mismatch: number | null; + ready: number | null; +} + +interface CountArgs { + expected_byte_len: number; +} + +/** + * SQL for the four `(table, vec column)` slots. Each query returns + * `(total, missing, dim_mismatch, ready)` for a single slot in one round-trip. + * + * `LENGTH(vec)` on a BLOB column returns the stored byte count without + * deserialising the BLOB into JS memory — this is the whole point of the + * fix (see #1929 evidence: 99.96 % of the pre-fix time was in `pread64`). + */ +const TRACE_QUALIFICATION = + "(LENGTH(TRIM(user_text)) >= 10 OR LENGTH(TRIM(agent_text)) >= 10) " + + "AND (LENGTH(TRIM(user_text)) + LENGTH(TRIM(agent_text)) >= 20)"; + +/** Build the count SQL for a single (table, column) slot. */ +function slotSql( + table: string, + column: string, + extraWhere: string = "", +): string { + const clauses: string[] = []; + if (table === "traces") clauses.push(TRACE_QUALIFICATION); + if (extraWhere) clauses.push(extraWhere); + const where = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : ""; + return ` + SELECT + COUNT(*) AS total, + SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END) AS missing, + SUM(CASE WHEN ${column} IS NOT NULL + AND @expected_byte_len > 0 + AND LENGTH(${column}) <> @expected_byte_len + THEN 1 ELSE 0 END) AS dim_mismatch, + SUM(CASE WHEN ${column} IS NOT NULL + AND (@expected_byte_len = 0 OR LENGTH(${column}) = @expected_byte_len) + THEN 1 ELSE 0 END) AS ready + FROM ${table} + ${where} + `; +} + +function normalizeRow(row: CountRow | undefined): EmbeddingCountsBucket { + if (!row) { + return { totalSlots: 0, ready: 0, missing: 0, dimMismatch: 0 }; + } + return { + totalSlots: row.total ?? 0, + ready: row.ready ?? 0, + missing: row.missing ?? 0, + dimMismatch: row.dim_mismatch ?? 0, + }; +} + +/** + * Count embedding slots per (table, vec column) purely with SQL. + * + * MUST NOT read or decode any BLOB into JS. Total wall-clock work is + * `O(rows)` SQL scan touching only BLOB header bytes. + * + * @param db - open storage handle (better-sqlite3 wrapper). + * @param opts.expectedByteLen - `dimensions * 4` for a known Float32 + * dimension, or `0` when the dimension has not been probed yet. In the + * `0` fallback every non-NULL vector counts as ready and dimMismatch + * is always 0 — matches the pre-fix "any non-null = ready" behaviour + * that `inferStoredEmbeddingDimension(slots)` used to fall back to. + */ +export function embeddingMaintenanceCounts( + db: StorageDb, + opts: { expectedByteLen: number }, +): EmbeddingCounts { + const args: CountArgs = { + expected_byte_len: Math.max(0, Math.floor(opts.expectedByteLen) || 0), + }; + + const traceSummary = db + .prepare(slotSql("traces", "vec_summary")) + .get(args); + const traceAction = db + .prepare( + slotSql( + "traces", + "vec_action", + "instr(COALESCE(tags_json, ''), '\"lightweight_memory\"') = 0", + ), + ) + .get(args); + const policy = db + .prepare(slotSql("policies", "vec")) + .get(args); + const worldModel = db + .prepare(slotSql("world_model", "vec")) + .get(args); + const skill = db + .prepare(slotSql("skills", "vec")) + .get(args); + + const summary = normalizeRow(traceSummary); + const action = normalizeRow(traceAction); + + return { + trace: { + totalSlots: summary.totalSlots + action.totalSlots, + ready: summary.ready + action.ready, + missing: summary.missing + action.missing, + dimMismatch: summary.dimMismatch + action.dimMismatch, + }, + policy: normalizeRow(policy), + world_model: normalizeRow(worldModel), + skill: normalizeRow(skill), + }; +} + +interface ModeRow { + byte_len: number; + n: number; +} + +/** + * Infer the dominant stored embedding byte length by GROUP BY the byte + * length of every non-NULL `traces.vec_summary` BLOB. Returns the byte + * length with the highest row count, or 0 when the DB has no vectors + * (brand-new install). + * + * Cheap replacement for the pre-fix + * `inferStoredEmbeddingDimension(collectEmbeddingSlots())` path — which had + * to hydrate every BLOB in memory before it could measure any single one. + * We now let SQLite do the length arithmetic and just pick the mode. + */ +export function inferStoredEmbeddingByteLen(db: StorageDb): number { + const rows = db + .prepare( + `SELECT LENGTH(vec_summary) AS byte_len, COUNT(*) AS n + FROM traces + WHERE vec_summary IS NOT NULL AND LENGTH(vec_summary) > 0 + GROUP BY LENGTH(vec_summary) + ORDER BY n DESC + LIMIT 1`, + ) + .all(); + if (rows.length === 0) return 0; + return rows[0]!.byte_len; +} diff --git a/apps/memos-local-plugin/core/storage/repos/index.ts b/apps/memos-local-plugin/core/storage/repos/index.ts index ec80392fd..5f3038dab 100644 --- a/apps/memos-local-plugin/core/storage/repos/index.ts +++ b/apps/memos-local-plugin/core/storage/repos/index.ts @@ -85,179 +85,15 @@ export { makeTracePolicyLinksRepo } from "./trace-policy-links.js"; export { makeTracesRepo } from "./traces.js"; export { makeWorldModelRepo } from "./world_model.js"; -// ─── Embedding maintenance — SQL fast path ────────────────────────────────── -// -// `GET /api/v1/embeddings/maintenance` used to paginate every row of -// `traces` / `policies` / `world_model` / `skills` and hydrate the BLOB -// vector columns into JS purely to inspect each vector's length. -// On a ~93K-row deployment that pulled ~270 MB through better-sqlite3 -// on the main thread and blocked the event loop for 4+ minutes -// (https://github.com/MemTensor/MemOS/issues/1929). -// -// This helper replaces that path with five `SELECT COUNT(*) + -// SUM(CASE WHEN ...)` queries — one per (table, vec column). Only the -// BLOB header is read (`LENGTH(vec)` does not deserialise the BLOB -// payload), so the call stays in single-millisecond territory even -// on multi-GB databases. - -/** Per-kind bucket of slot counts produced by `embeddingMaintenanceCounts`. */ -export interface EmbeddingCountsBucket { - /** Number of (row × vec column) slots considered by this bucket. */ - totalSlots: number; - /** Vec is non-null AND (expectedByteLen=0 OR LENGTH(vec) = expectedByteLen). */ - ready: number; - /** Vec column IS NULL. */ - missing: number; - /** Vec is non-null but its byte length ≠ expectedByteLen (only when expectedByteLen > 0). */ - dimMismatch: number; -} - -export interface EmbeddingCounts { - trace: EmbeddingCountsBucket; - policy: EmbeddingCountsBucket; - world_model: EmbeddingCountsBucket; - skill: EmbeddingCountsBucket; -} - -/** - * Reusable WHERE fragment that mirrors `shouldTraceHaveEmbeddings(row)` - * in `core/pipeline/memory-core.ts`. A trace only contributes to an - * embedding slot if at least one of user_text / agent_text has - * meaningful content (≥10 chars) AND the combined text is ≥20 chars. - * Without this filter the SQL counts would balloon every short - * "ok"/"got it" trace into a phantom "missing vector" row. - */ -const TRACE_QUALIFIES_FOR_VEC = - "(LENGTH(TRIM(COALESCE(user_text, ''))) >= 10 " + - "OR LENGTH(TRIM(COALESCE(agent_text, ''))) >= 10) " + - "AND (LENGTH(TRIM(COALESCE(user_text, ''))) " + - "+ LENGTH(TRIM(COALESCE(agent_text, ''))) >= 20)"; - -/** - * Action vectors are skipped for lightweight-memory traces. Matches - * `isLightweightMemoryTrace(row)` in `core/pipeline/memory-core.ts`: - * `row.tags.includes("lightweight_memory")`. - * `tags_json` is a JSON array stored as TEXT; we match the quoted - * element string so a tag named `"lightweight_memory_v2"` does not - * fire a false positive. - */ -const TRACE_NOT_LIGHTWEIGHT = - "instr(COALESCE(tags_json, ''), '\"lightweight_memory\"') = 0"; - -/** - * SQL-only embedding-slot counter. Returns the same per-kind counts the - * old slot-enumeration path used to compute in JS, but without reading - * a single vector BLOB into JS memory. - * - * `expectedByteLen` is the Float32-encoded byte length the BLOB must - * match to count as `ready` (i.e. `dimensions * 4`). Pass `0` when the - * embedder has not been probed yet — the helper then counts any - * non-null vector as `ready` and never reports `dimMismatch`. This - * mirrors the pre-fix fallback where `inferStoredEmbeddingDimension` - * returned `0` on a brand-new install. - */ -export function embeddingMaintenanceCounts( - db: StorageDb, - opts: { expectedByteLen: number }, -): EmbeddingCounts { - const expectedByteLen = Math.max(0, Math.floor(opts.expectedByteLen || 0)); - return { - trace: traceCounts(db, expectedByteLen), - policy: simpleCounts(db, "policies", "vec", expectedByteLen), - world_model: simpleCounts(db, "world_model", "vec", expectedByteLen), - skill: simpleCounts(db, "skills", "vec", expectedByteLen), - }; -} - -function traceCounts(db: StorageDb, expectedByteLen: number): EmbeddingCountsBucket { - const summary = countColumn(db, { - table: "traces", - column: "vec_summary", - expectedByteLen, - whereExtra: TRACE_QUALIFIES_FOR_VEC, - }); - const action = countColumn(db, { - table: "traces", - column: "vec_action", - expectedByteLen, - whereExtra: `${TRACE_QUALIFIES_FOR_VEC} AND ${TRACE_NOT_LIGHTWEIGHT}`, - }); - return { - totalSlots: summary.totalSlots + action.totalSlots, - ready: summary.ready + action.ready, - missing: summary.missing + action.missing, - dimMismatch: summary.dimMismatch + action.dimMismatch, - }; -} - -function simpleCounts( - db: StorageDb, - table: string, - column: string, - expectedByteLen: number, -): EmbeddingCountsBucket { - return countColumn(db, { table, column, expectedByteLen }); -} - -interface CountColumnOpts { - table: string; - column: string; - expectedByteLen: number; - whereExtra?: string; -} - -interface CountColumnRow { - total: number; - missing: number; - dim_mismatch: number; - ready: number; -} - -function countColumn(db: StorageDb, opts: CountColumnOpts): EmbeddingCountsBucket { - const { table, column, expectedByteLen, whereExtra } = opts; - // `expectedByteLen` is interpolated as a positive integer literal - // (not a parameter) so the comparison is constant-folded by SQLite - // and the helper has nothing to bind on small / empty databases. - // Values come from `Math.floor(...)` on a JS number — no user input - // reaches this string. - const lenLiteral = expectedByteLen.toString(); - const dimEnabled = expectedByteLen > 0; - const dimMismatchExpr = dimEnabled - ? `${column} IS NOT NULL AND LENGTH(${column}) <> ${lenLiteral}` - : "0"; - const readyExpr = dimEnabled - ? `${column} IS NOT NULL AND LENGTH(${column}) = ${lenLiteral}` - : `${column} IS NOT NULL`; - const whereClause = whereExtra ? ` WHERE ${whereExtra}` : ""; - const sql = - `SELECT COUNT(*) AS total, ` + - `SUM(CASE WHEN ${column} IS NULL THEN 1 ELSE 0 END) AS missing, ` + - `SUM(CASE WHEN ${dimMismatchExpr} THEN 1 ELSE 0 END) AS dim_mismatch, ` + - `SUM(CASE WHEN ${readyExpr} THEN 1 ELSE 0 END) AS ready ` + - `FROM ${table}${whereClause}`; - const row = db.prepare(sql).get(); - return { - totalSlots: row?.total ?? 0, - ready: row?.ready ?? 0, - missing: row?.missing ?? 0, - dimMismatch: row?.dim_mismatch ?? 0, - }; -} - /** - * Cheap mode-finder used when the embedder has not been probed yet so - * `dimensions` is still `0`. Returns the BLOB byte length that occurs - * most often in stored `traces.vec_summary` rows (or 0 if no vectors - * are stored at all). Uses a single `GROUP BY LENGTH(vec_summary)` — - * the BLOB header is touched, the BLOB body is not. + * SQL-only embedding maintenance stats — see `./embedding_maintenance.ts` + * for the design rationale. Regression pin for issue #1929: the old JS + * path hydrated 270 MB of BLOBs per request; this SQL fast path uses + * `LENGTH(vec)` alone and never touches the buffers. */ -export function inferStoredEmbeddingByteLen(db: StorageDb): number { - const sql = - "SELECT LENGTH(vec_summary) AS len, COUNT(*) AS n " + - "FROM traces WHERE vec_summary IS NOT NULL " + - "GROUP BY len ORDER BY n DESC LIMIT 1"; - const row = db - .prepare(sql) - .get(); - return row?.len ?? 0; -} +export { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, + FLOAT32_BYTES, +} from "./embedding_maintenance.js"; +export type { EmbeddingCounts, EmbeddingCountsBucket } from "./embedding_maintenance.js"; diff --git a/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts index dca4d7da4..e34db1891 100644 --- a/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/embedding-maintenance.test.ts @@ -1,289 +1,374 @@ /** - * Unit tests for the SQL-only embedding-maintenance count helper. + * SQL-only embedding maintenance stats. * - * Issue #1929 — `GET /api/v1/embeddings/maintenance` used to load every - * vector BLOB into JS just to count nulls and dimension mismatches. - * `embeddingMaintenanceCounts` replaces that with pure SQL `COUNT(*)` - * queries; these tests pin the bucket semantics + filter rules. + * Regression + spec pin for issue #1929: `/api/v1/embeddings/maintenance` + * used to paginate every trace/policy/world_model/skill row through JS just + * to inspect vector byte lengths, hydrating hundreds of MB of BLOBs into the + * Node heap and blocking the event loop for minutes on production DBs. + * + * The new helper `embeddingMaintenanceCounts()` MUST count purely with SQL + * (`COUNT(*)` + `SUM(CASE WHEN ... LENGTH(vec) ...)`), preserving the two + * pre-fix semantic filters: + * - short-text traces are skipped (mirrors `shouldTraceHaveEmbeddings`) + * - `lightweight_memory`-tagged traces don't get counted for `vec_action` */ + import { describe, expect, it } from "vitest"; import { encodeVector } from "../../../core/storage/vector.js"; -import { embeddingMaintenanceCounts } from "../../../core/storage/repos/index.js"; -import { makeTmpDb } from "../../helpers/tmp-db.js"; -import type { PolicyRow, SkillRow, TraceRow, WorldModelRow } from "../../../core/types.js"; +import { + embeddingMaintenanceCounts, + inferStoredEmbeddingByteLen, +} from "../../../core/storage/repos/index.js"; +import { makeTmpDb, type TmpDbHandle } from "../../helpers/tmp-db.js"; +import type { + EpisodeId, + SessionId, + SkillId, + TraceId, + WorldModelId, +} from "../../../core/types.js"; const DIM = 4; -const EXPECTED_BYTE_LEN = DIM * 4; // Float32 = 4 bytes per element +const EXPECTED_BYTE_LEN = DIM * 4; -function vec(values: number[]): Float32Array { - return new Float32Array(values); +function fullVec(): Float32Array { + return new Float32Array([0.1, 0.2, 0.3, 0.4]); } -function vecBlob(values: number[]): Float32Array { - // Same as `vec` — kept as a named alias so the test reads like - // "this is the BLOB shape we expect the SQL helper to see". - return new Float32Array(values); +function shortVec(): Float32Array { + return new Float32Array([9]); } -function ensureTraceParents(repos: ReturnType["repos"]): void { - // traces FK → episodes / sessions; seed those once per test to keep the - // per-trace seed helper noise-free. - if (!repos.sessions.getById("s0")) { - repos.sessions.upsert({ - id: "s0", - agent: "openclaw", - startedAt: 1_700_000_000_000, - lastSeenAt: 1_700_000_000_000, - meta: {}, - }); - } - if (!repos.episodes.getById("e0" as never)) { - repos.episodes.insert({ - id: "e0" as never, - sessionId: "s0" as never, - startedAt: 1_700_000_000_000, - endedAt: null, - traceIds: [], - rTask: null, - status: "open", - }); - } +function seedSessionAndEpisode(handle: TmpDbHandle): void { + handle.repos.sessions.upsert({ + id: "se" as SessionId, + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + lastSeenAt: 1_700_000_000_000, + meta: {}, + }); + handle.repos.episodes.insert({ + id: "ep" as EpisodeId, + sessionId: "se" as SessionId, + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: 1_700_000_000_000, + endedAt: null, + traceIds: [], + rTask: null, + status: "open", + meta: {}, + }); } function seedTrace( - repos: ReturnType["repos"], + handle: TmpDbHandle, id: string, - overrides: Partial, + opts: { + userText: string; + agentText: string; + tags?: string[]; + vecSummary?: Float32Array | null; + vecAction?: Float32Array | null; + }, ): void { - ensureTraceParents(repos); - const base: TraceRow = { - id, - episodeId: "e0", - sessionId: "s0", + handle.repos.traces.insert({ + id: id as TraceId, + episodeId: "ep" as EpisodeId, + sessionId: "se" as SessionId, + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, ts: 1_700_000_000_000, - userText: "user message that is comfortably longer than ten chars", - agentText: "agent reply that is also comfortably long", + userText: opts.userText, + agentText: opts.agentText, + summary: "summary text", + share: null, toolCalls: [], + agentThinking: null, reflection: null, value: 0, alpha: 0, rHuman: null, priority: 0, - tags: [], + tags: opts.tags ?? [], errorSignatures: [], - vecSummary: vec([0, 0, 0, 0]), - vecAction: vec([0, 0, 0, 0]), - turnId: 1_700_000_000_000 as never, + vecSummary: opts.vecSummary ?? null, + vecAction: opts.vecAction ?? null, + turnId: 1_700_000_000_000, schemaVersion: 1, - ...overrides, - } as TraceRow; - repos.traces.insert(base); + } as never); } -function seedPolicy( - repos: ReturnType["repos"], - id: string, - vector: Float32Array | null, -): void { - const row: PolicyRow = { - id, +function seedPolicy(handle: TmpDbHandle, id: string, vec: Float32Array | null): void { + handle.repos.policies.upsert({ + id: id as never, title: id, trigger: "", procedure: "", verification: "", boundary: "", - support: 1, + support: 0, gain: 0, status: "candidate", sourceEpisodeIds: [], - inducedBy: "test", + inducedBy: "proto", decisionGuidance: { preference: [], antiPattern: [] }, - vec: vector, - createdAt: 1_700_000_000_000 as never, - updatedAt: 1_700_000_000_000 as never, - } as PolicyRow; - repos.policies.upsert(row); + vec, + createdAt: 1, + updatedAt: 1, + }); } function seedWorldModel( - repos: ReturnType["repos"], + handle: TmpDbHandle, id: string, - vector: Float32Array | null, + vec: Float32Array | null, ): void { - const row: WorldModelRow = { - id, + handle.repos.worldModel.upsert({ + id: id as WorldModelId, title: id, - body: "world model", + body: "world body text", structure: { environment: [], inference: [], constraints: [] }, domainTags: [], - confidence: 0.5, + confidence: 0.9, policyIds: [], sourceEpisodeIds: [], - inducedBy: "test", - vec: vector, - createdAt: 1_700_000_000_000 as never, - updatedAt: 1_700_000_000_000 as never, + inducedBy: "", + vec, + createdAt: 1, + updatedAt: 1, version: 1, status: "active", - } as WorldModelRow; - repos.worldModel.upsert(row); + }); } -function seedSkill( - repos: ReturnType["repos"], - id: string, - vector: Float32Array | null, -): void { - const row: SkillRow = { - id, +function seedSkill(handle: TmpDbHandle, id: string, vec: Float32Array | null): void { + handle.repos.skills.insert({ + id: id as SkillId, name: id, - status: "active", - invocationGuide: "", + status: "candidate", + invocationGuide: "guide", procedureJson: null, - eta: 0.5, - support: 1, + eta: 0, + support: 0, gain: 0, trialsAttempted: 0, trialsPassed: 0, sourcePolicyIds: [], sourceWorldModelIds: [], evidenceAnchors: [], - vec: vector, - createdAt: 1_700_000_000_000 as never, - updatedAt: 1_700_000_000_000 as never, + vec, + createdAt: 1, + updatedAt: 1, version: 1, - } as SkillRow; - repos.skills.upsert(row); + }); } -describe("storage/repos — embeddingMaintenanceCounts", () => { +describe("storage/repos — embeddingMaintenanceCounts (issue #1929)", () => { it("counts ready / missing / dimMismatch per kind without decoding BLOBs", () => { - const { db, repos, cleanup } = makeTmpDb(); + const handle = makeTmpDb(); try { - // ── Traces ──────────────────────────────────────────────────── - // Two ready summary/action slots. - seedTrace(repos, "tr_ready", { - vecSummary: vec([1, 1, 1, 1]), - vecAction: vec([2, 2, 2, 2]), + seedSessionAndEpisode(handle); + + // traces + // - tr_ready: qualifying, has both summary+action vectors at correct dim + seedTrace(handle, "tr_ready", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + vecAction: fullVec(), }); - // Missing summary + missing action. - seedTrace(repos, "tr_missing", { + // - tr_missing: qualifying, no vectors at all + seedTrace(handle, "tr_missing", { + userText: "hello world what is up", + agentText: "another answer here", vecSummary: null, vecAction: null, }); - // Dimension mismatch on summary, correct on action. - seedTrace(repos, "tr_dim_mismatch", { - vecSummary: vec([1, 2]), - vecAction: vec([3, 3, 3, 3]), + // - tr_dim_mismatch: qualifying, but vec dims wrong + seedTrace(handle, "tr_dim_mismatch", { + userText: "hello world what is up", + agentText: "yet another answer", + vecSummary: shortVec(), + vecAction: shortVec(), }); - - // ── Short-text trace (should be filtered out — matches - // shouldTraceHaveEmbeddings) ───────────────────────────────── - seedTrace(repos, "tr_short", { + // - tr_short: NOT qualifying (both texts <10, sum <20) — should be excluded + seedTrace(handle, "tr_short", { userText: "hi", agentText: "ok", vecSummary: null, vecAction: null, }); - - // ── Lightweight-memory trace (vec_action slot excluded) ─────── - seedTrace(repos, "tr_lightweight", { + // - tr_lightweight: qualifying for vec_summary but NOT vec_action + seedTrace(handle, "tr_lightweight", { + userText: "hello world what is up", + agentText: "the lightweight answer", tags: ["lightweight_memory"], - vecSummary: vec([1, 1, 1, 1]), + vecSummary: fullVec(), vecAction: null, }); - // ── Policies / world_model / skills ────────────────────────── - seedPolicy(repos, "p_ready", vec([1, 1, 1, 1])); - seedPolicy(repos, "p_missing", null); - seedPolicy(repos, "p_dim", vec([1, 2, 3])); // dimension mismatch + // policies + seedPolicy(handle, "po_ready", fullVec()); + seedPolicy(handle, "po_missing", null); + seedPolicy(handle, "po_dim", shortVec()); - seedWorldModel(repos, "wm_ready", vec([2, 2, 2, 2])); - seedWorldModel(repos, "wm_missing", null); + // world_model + seedWorldModel(handle, "wm_ready", fullVec()); + seedWorldModel(handle, "wm_missing", null); - seedSkill(repos, "sk_ready", vec([3, 3, 3, 3])); + // skills + seedSkill(handle, "sk_ready", fullVec()); + seedSkill(handle, "sk_dim", shortVec()); - const counts = embeddingMaintenanceCounts(db, { + const counts = embeddingMaintenanceCounts(handle.db, { expectedByteLen: EXPECTED_BYTE_LEN, }); - // Trace bucket: - // summary slots qualifying: tr_ready, tr_missing, tr_dim_mismatch, tr_lightweight = 4 - // action slots qualifying: tr_ready, tr_missing, tr_dim_mismatch = 3 (lightweight excluded) - // short-text trace excluded from BOTH slot counts. - // ready summary: tr_ready, tr_lightweight = 2 - // ready action: tr_ready, tr_dim_mismatch = 2 - // missing summary: tr_missing = 1 - // missing action: tr_missing = 1 (lightweight excluded) - // dimMismatch summary: tr_dim_mismatch = 1 - expect(counts.trace.totalSlots).toBe(7); - expect(counts.trace.ready).toBe(4); - expect(counts.trace.missing).toBe(2); - expect(counts.trace.dimMismatch).toBe(1); + // trace bucket: + // summary qualifying rows: tr_ready, tr_missing, tr_dim_mismatch, tr_lightweight = 4 + // action qualifying rows: tr_ready, tr_missing, tr_dim_mismatch = 3 + // totalSlots = 4 + 3 = 7 + // ready = tr_ready(summary+action) + tr_lightweight(summary) = 3 + // missing = tr_missing(summary+action) = 2 + // dimMismatch = tr_dim_mismatch(summary+action) = 2 + expect(counts.trace).toEqual({ + totalSlots: 7, + ready: 3, + missing: 2, + dimMismatch: 2, + }); - // Policy bucket: 1 ready, 1 missing, 1 dim mismatch - expect(counts.policy.totalSlots).toBe(3); - expect(counts.policy.ready).toBe(1); - expect(counts.policy.missing).toBe(1); - expect(counts.policy.dimMismatch).toBe(1); + expect(counts.policy).toEqual({ + totalSlots: 3, + ready: 1, + missing: 1, + dimMismatch: 1, + }); - // World model: 1 ready, 1 missing - expect(counts.world_model.totalSlots).toBe(2); - expect(counts.world_model.ready).toBe(1); - expect(counts.world_model.missing).toBe(1); + expect(counts.world_model).toEqual({ + totalSlots: 2, + ready: 1, + missing: 1, + dimMismatch: 0, + }); - // Skill: 1 ready, 0 missing - expect(counts.skill.totalSlots).toBe(1); - expect(counts.skill.ready).toBe(1); - expect(counts.skill.missing).toBe(0); + expect(counts.skill).toEqual({ + totalSlots: 2, + ready: 1, + missing: 0, + dimMismatch: 1, + }); } finally { - cleanup(); + handle.cleanup(); } }); it("falls back to 'any non-null = ready' when expectedByteLen is 0", () => { - const { db, repos, cleanup } = makeTmpDb(); + const handle = makeTmpDb(); try { - seedTrace(repos, "tr_a", { vecSummary: vec([1, 2]), vecAction: null }); - seedTrace(repos, "tr_b", { vecSummary: vec([1, 2, 3, 4]), vecAction: vec([5, 6, 7, 8]) }); + seedSessionAndEpisode(handle); + // A brand-new install with no embedder probe yet: dimension unknown. + // Any stored BLOB (short or full) should count as ready. + seedTrace(handle, "tr_full", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + vecAction: shortVec(), + }); + seedTrace(handle, "tr_missing", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: null, + vecAction: null, + }); + + const counts = embeddingMaintenanceCounts(handle.db, { expectedByteLen: 0 }); - const counts = embeddingMaintenanceCounts(db, { expectedByteLen: 0 }); - // No expected length → every non-null vector counts as ready - // and dimMismatch is always zero. - expect(counts.trace.ready).toBe(3); // tr_a.summary + tr_b.summary + tr_b.action - expect(counts.trace.missing).toBe(1); // tr_a.action + // Both slots for tr_full count as ready regardless of BLOB length. + expect(counts.trace.ready).toBe(2); expect(counts.trace.dimMismatch).toBe(0); + expect(counts.trace.missing).toBe(2); + expect(counts.trace.totalSlots).toBe(4); } finally { - cleanup(); + handle.cleanup(); } }); it("returns zero counts for an empty database", () => { - const { db, cleanup } = makeTmpDb(); + const handle = makeTmpDb(); try { - const counts = embeddingMaintenanceCounts(db, { + const counts = embeddingMaintenanceCounts(handle.db, { expectedByteLen: EXPECTED_BYTE_LEN, }); - expect(counts.trace.totalSlots).toBe(0); - expect(counts.policy.totalSlots).toBe(0); - expect(counts.world_model.totalSlots).toBe(0); - expect(counts.skill.totalSlots).toBe(0); - for (const bucket of Object.values(counts)) { - expect(bucket.ready).toBe(0); - expect(bucket.missing).toBe(0); - expect(bucket.dimMismatch).toBe(0); - } + expect(counts.trace).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.policy).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.world_model).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + expect(counts.skill).toEqual({ + totalSlots: 0, + ready: 0, + missing: 0, + dimMismatch: 0, + }); + } finally { + handle.cleanup(); + } + }); + + it("infers stored byte length from the mode of trace vec_summary BLOBs", () => { + const handle = makeTmpDb(); + try { + seedSessionAndEpisode(handle); + // Three rows at 4-dim, one at 1-dim → mode = 16 bytes. + seedTrace(handle, "tr_a", { + userText: "hello world what is up", + agentText: "here is the answer", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_b", { + userText: "hello world what is up", + agentText: "another answer here", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_c", { + userText: "hello world what is up", + agentText: "yet another answer", + vecSummary: fullVec(), + }); + seedTrace(handle, "tr_odd", { + userText: "hello world what is up", + agentText: "the outlier answer", + vecSummary: shortVec(), + }); + + expect(inferStoredEmbeddingByteLen(handle.db)).toBe(EXPECTED_BYTE_LEN); } finally { - cleanup(); + handle.cleanup(); } }); - it("uses BLOB byte length for dimension comparison (encoded by encodeVector)", () => { - // Sanity check: the stored BLOB byte length must equal `dim * 4` - // so the SQL `LENGTH(vec) <> @expected_byte_len` comparison works. - const float32 = vecBlob([1, 2, 3, 4]); - const buf = encodeVector(float32); - expect(buf.byteLength).toBe(EXPECTED_BYTE_LEN); + it("uses BLOB byte length for dimension comparison", () => { + expect(encodeVector(fullVec()).byteLength).toBe(EXPECTED_BYTE_LEN); }); });