diff --git a/apps/memos-local-plugin/core/pipeline/memory-core.ts b/apps/memos-local-plugin/core/pipeline/memory-core.ts index c7b2782e4..9251c3977 100644 --- a/apps/memos-local-plugin/core/pipeline/memory-core.ts +++ b/apps/memos-local-plugin/core/pipeline/memory-core.ts @@ -88,7 +88,10 @@ import { FLOAT32_BYTES, } from "../storage/repos/index.js"; import type { EmbeddingCountsBucket } from "../storage/repos/index.js"; -import type { ClosedEpisodeCursor } from "../storage/repos/episodes.js"; +import type { + ClosedEpisodeCursor, + EpisodeMetaRow, +} from "../storage/repos/episodes.js"; import { createEmbedder } from "../embedding/embedder.js"; import { createLlmClient } from "../llm/client.js"; import { @@ -727,6 +730,14 @@ export function createMemoryCore( let startupRecoveryCancelled = false; let lastStaleScan = 0; let lastDirtyClosedScan = 0; + // Issue #2459 (review): episode ids whose dirty-rescore drain is still + // in flight (phase-1 emitted, phase-2 not yet settled). The head-page + // pass re-reads the newest page on every 10-minute tick, and an + // in-flight episode still carries `rewardDirty` until settlement — + // without this guard the periodic scan re-collected and re-emitted the + // whole batch every tick (the duplicate LLM spend gap 2 set out to + // eliminate). Cleared per episode as soon as its settlement runs. + const dirtyRescoreInFlight = new Set(); async function autoFinalizeStaleTasks(): Promise { if (!autoRecoveryEnabled) return; const nowMs = Date.now(); @@ -800,7 +811,14 @@ export function createMemoryCore( if (handle.algorithm.lightweightMemory.enabled) return; try { const allDirty = collectDirtyClosedEpisodes() - .filter((ep) => !isLightweightEpisode(ep) && episodeRewardIsDirty(ep)); + .filter((ep) => !isLightweightEpisode(ep) && episodeRewardIsDirty(ep)) + // Issue #2459 (review): rows whose drain is still mid-flight + // keep `rewardDirty` until phase-2 settles — skip them here so a + // tick racing an open drain cannot re-emit and re-run the whole + // batch (duplicate LLM spend). Freshly dirtied rows are NOT + // blocked by this: they are not in flight, so they are still + // collected on the very same tick. + .filter((ep) => !dirtyRescoreInFlight.has(ep.id as EpisodeId)); // Apply the same backoff filter as init() so the 10-min periodic // scan does not hammer episodes whose LLM call keeps failing. const dirtyClosed = allDirty.filter((ep) => dirtyEpisodeBackoffElapsed(ep, nowMs)); @@ -1197,7 +1215,12 @@ export function createMemoryCore( } else { const nowForDirty = Date.now(); const allDirty = collectDirtyClosedEpisodes() - .filter((ep) => !isLightweightEpisode(ep) && episodeRewardIsDirty(ep)); + .filter((ep) => !isLightweightEpisode(ep) && episodeRewardIsDirty(ep)) + // Issue #2459 (review): same in-flight guard as the periodic + // scan — rows already mid-drain keep `rewardDirty` until + // settlement, and re-emitting them here would queue duplicate + // reward runs. + .filter((ep) => !dirtyRescoreInFlight.has(ep.id as EpisodeId)); const dirtyClosed: typeof allDirty = []; for (const ep of allDirty) { if (dirtyEpisodeBackoffElapsed(ep, nowForDirty)) { @@ -1216,7 +1239,11 @@ export function createMemoryCore( dirtyClosedForBackground = dirtyClosed; } } catch (err) { - log.debug("init.orphan_scan.failed", { + // Issue #2459: a skipped startup classification used to be + // invisible at default log levels — the reporter only found it + // by diffing kv cursor writes. This failure means zero + // collection for the whole boot, so it must be visible. + log.warn("init.orphan_scan.failed", { err: err instanceof Error ? err.message : String(err), }); } @@ -1679,6 +1706,9 @@ export function createMemoryCore( ? prevDirty.failedAttempts : 0; priorFailedAttempts.set(episodeId, prevAttempts); + // Mark in-flight BEFORE emitting so a periodic tick racing this + // drain cannot queue a duplicate run for the same row. + dirtyRescoreInFlight.add(episodeId); handle.repos.episodes.updateMeta(episodeId, { closeReason: "finalized", recoveredAtStartup: endedAt, @@ -1693,31 +1723,58 @@ export function createMemoryCore( closedBy: "finalized", }); } - await handle.flush(); - if (startupRecoveryCancelled) return; - // After the reward / reflect chain has finished, account for the - // outcome: clear `meta.rewardDirty` on episodes that are no longer - // dirty (success), bump `failedAttempts + lastFailureAt` on episodes - // that still match the dirty predicate (LLM failure / no-op). This - // closes the "retried indefinitely" loop reported in issue #1808. - const now = Date.now(); - for (const [episodeId, prevAttempts] of priorFailedAttempts) { - const after = handle.repos.episodes.getById(episodeId); - if (!after) continue; - const stillDirty = episodeRewardIsDirty(after); - if (stillDirty) { - handle.repos.episodes.updateMeta(episodeId, { - rewardDirty: { - failedAttempts: prevAttempts + 1, - lastFailureAt: now, - }, - }); - } else if ( - after.meta && - typeof after.meta === "object" && - "rewardDirty" in after.meta - ) { - handle.repos.episodes.updateMeta(episodeId, { rewardDirty: undefined }); + try { + await handle.flush(); + } finally { + // After the reward / reflect chain has finished, account for the + // outcome: clear `meta.rewardDirty` on episodes that are no longer + // dirty (success), bump `failedAttempts + lastFailureAt` on + // episodes that still match the dirty predicate (LLM failure / + // no-op). This closes the "retried indefinitely" loop reported in + // issue #1808. + // + // Issue #2459 (gap 2): a shutdown cancel mid-drain used to `return` + // right after the flush, skipping this settlement for the *whole* + // batch — episodes whose chains had already rescored successfully + // kept their `rewardDirty` markers and were re-collected + re-run + // later (duplicate LLM spend), while failures lost their backoff + // counters. Settling is synchronous bookkeeping on rows whose + // chains already ran, so it happens even when cancelled (and even + // if the flush itself threw mid-teardown) — best-effort, one + // try/catch per row so a single late-teardown error cannot drop + // the rest of the batch. + const now = Date.now(); + for (const [episodeId, prevAttempts] of priorFailedAttempts) { + try { + const after = handle.repos.episodes.getById(episodeId); + if (after) { + const stillDirty = episodeRewardIsDirty(after); + if (stillDirty) { + handle.repos.episodes.updateMeta(episodeId, { + rewardDirty: { + failedAttempts: prevAttempts + 1, + lastFailureAt: now, + }, + }); + } else if ( + after.meta && + typeof after.meta === "object" && + "rewardDirty" in after.meta + ) { + handle.repos.episodes.updateMeta(episodeId, { + rewardDirty: undefined, + }); + } + } + } catch (err) { + log.warn("init.dirty_closed_episodes.settle_failed", { + episodeId, + err: err instanceof Error ? err.message : String(err), + }); + } + // The chain for this row has reached a terminal state (settled + // or un-settleable) — let the next scan consider it again. + dirtyRescoreInFlight.delete(episodeId); } } } @@ -1728,10 +1785,36 @@ export function createMemoryCore( if (storedCursor !== null && !cursor) { handle.repos.kv.del(DIRTY_CLOSED_SCAN_CURSOR_KEY); } - const page = handle.repos.episodes.listClosedPage({ + // Issue #2459 (gap 1): the stored cursor only ever moves *away* from + // the newest rows, so an episode that becomes dirty after the cursor + // paged past its position was not revisited until the walk reached + // the end of the closed set and the cursor reset — a full lap of + // `ceil(closed / 500) × 10 min` (≈70 min on the reporter's DB, + // hours on the 30k+-episode DBs of #1787). Take one bounded page + // from the newest end on every scan: freshly closed episodes and + // freshly re-marked recent rows are seen immediately, while the + // cursor walk below keeps draining the older backlog a page at a + // time. + const collected = new Map(); + const headPage = handle.repos.episodes.listClosedPage({ limit: DIRTY_CLOSED_SCAN_PAGE_SIZE, - before: cursor, + before: null, }); + for (const ep of headPage) { + collected.set(ep.id, ep); + } + // Without a stored cursor the head page *is* the walk's first page; + // only page again when a cursor is in play. + let page = headPage; + if (cursor) { + page = handle.repos.episodes.listClosedPage({ + limit: DIRTY_CLOSED_SCAN_PAGE_SIZE, + before: cursor, + }); + for (const ep of page) { + collected.set(ep.id, ep); + } + } if (page.length === 0) { handle.repos.kv.del(DIRTY_CLOSED_SCAN_CURSOR_KEY); return []; @@ -1750,7 +1833,7 @@ export function createMemoryCore( // `rewardDirty` marker on short/trivial episodes. The skip is terminal; // clean the contradictory marker while this bounded page is already in // memory so upgraded installations stop retrying those rows immediately. - for (const ep of page) { + for (const ep of collected.values()) { if (!rewardWasSkipped(ep) || !hasRewardDirtyMarker(ep)) continue; try { handle.repos.episodes.updateMeta(ep.id as EpisodeId, { @@ -1767,7 +1850,7 @@ export function createMemoryCore( }); } } - return page; + return [...collected.values()]; } function isDirtyClosedScanCursor(value: unknown): value is ClosedEpisodeCursor { diff --git a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts index 2759ae5bd..5192994e4 100644 --- a/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts +++ b/apps/memos-local-plugin/tests/unit/pipeline/memory-core.test.ts @@ -2609,6 +2609,305 @@ algorithm: } }); + it("collects a closed episode dirtied above the stored scan cursor in a single pass (issue #2459 gap 1)", async () => { + // The persisted cursor only ever moves away from the newest rows, so + // an episode that becomes dirty after the cursor paged past its + // position used to wait until the walk reached the end of the closed + // set and reset (~70 min on the reporter's DB, hours on #1787's). + // One collect pass must pick it up via a bounded newest-first page. + const base = 1_000; + db!.repos.sessions.upsert({ + id: "se_cursor_starve", + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: base, + lastSeenAt: base, + meta: {}, + }); + for (let i = 0; i < 6; i++) { + const startedAt = base + i; + const dirty = i === 5; + db!.repos.episodes.insert({ + id: `ep_cursor_${i}`, + sessionId: "se_cursor_starve", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + endedAt: startedAt + 1, + traceIds: [`tr_cursor_${i}`] as never, + rTask: dirty ? null : 1, + status: "closed", + meta: { + closeReason: "finalized", + ...(dirty + ? { rewardDirty: { failedAttempts: 0, lastFailureAt: 0 } } + : {}), + }, + }); + } + db!.repos.traces.insert({ + id: "tr_cursor_5", + episodeId: "ep_cursor_5", + sessionId: "se_cursor_starve", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: base + 6, + userText: "光标饥饿测试:新变脏的 episode 应当立刻被收集。", + agentText: "好的,新一轮扫描应包含它。", + summary: null, + toolCalls: [], + reflection: null, + agentThinking: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0.5, + tags: [], + errorSignatures: [], + vecSummary: null, + vecAction: null, + share: null, + turnId: base + 6, + schemaVersion: 1, + }); + // Park the cursor below the newest row: the downward walk can only + // ever see episodes older than `ep_cursor_2`, so `ep_cursor_5` sits + // "above" the cursor and is invisible to the historic walk. + db!.repos.kv.set("pipeline.dirty_closed_scan_cursor.v1", { + startedAt: base + 2, + id: "ep_cursor_2", + }); + + pipeline = createPipeline(buildDeps(db!)); + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "issue2459-cursor-starve", + ); + + await core.init(); + await core.waitForStartupRecovery?.(); + + const episode = db!.repos.episodes.getById("ep_cursor_5" as never); + expect(episode).toBeDefined(); + // Collected on this very pass: phase-1 stamps the row before its + // reward chain runs. + expect(episode?.meta?.recoveryReason).toBe( + RECOVERY_REASONS.DIRTY_REWARD_RESCORE, + ); + }); + + it("still settles phase-2 per episode when shutdown cancels the drain mid-flush (issue #2459 gap 2)", async () => { + // `recoverDirtyClosedEpisodes` used to return right after + // `await handle.flush()` when `startupRecoveryCancelled` was set, + // skipping the accounting pass for the whole batch: successes kept + // `rewardDirty` (re-run later, duplicate LLM spend) and failures + // lost their backoff counters. Settlement must happen even when the + // drain was cancelled. + const base = Date.now() - 5_000; + db!.repos.sessions.upsert({ + id: "se_cancel_drain", + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: base, + lastSeenAt: base, + meta: {}, + }); + for (const suffix of ["a", "b"]) { + const id = `ep_cancel_${suffix}`; + const startedAt = base + (suffix === "a" ? 0 : 10); + db!.repos.episodes.insert({ + id, + sessionId: "se_cancel_drain", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + endedAt: startedAt + 1, + traceIds: [`tr_cancel_${suffix}`] as never, + rTask: null, + status: "closed", + meta: { closeReason: "finalized" }, + }); + db!.repos.traces.insert({ + id: `tr_cancel_${suffix}`, + episodeId: id, + sessionId: "se_cancel_drain", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: startedAt + 1, + userText: "中断 drain 测试:取消后仍要逐条结算退避计数。", + agentText: "明白,结算不得整批跳过。", + summary: null, + toolCalls: [], + reflection: null, + agentThinking: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0.5, + tags: [], + errorSignatures: [], + vecSummary: null, + vecAction: null, + share: null, + turnId: startedAt + 1, + schemaVersion: 1, + }); + } + + pipeline = createPipeline(buildDeps(db!)); + // Park the reward chain mid-drain so the recovery is provably inside + // `await handle.flush()` when shutdown's grace expires and flips + // `startupRecoveryCancelled`. The simulated LLM outage keeps both + // episodes dirty, so phase-2 must bump each one's backoff counters. + let releaseReward!: () => void; + const rewardGate = new Promise((resolve) => { + releaseReward = resolve; + }); + pipeline.rewardRunner.run = async () => { + await rewardGate; + throw new Error("simulated llm outage during drain"); + }; + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "issue2459-cancel-drain", + { startupRecoveryShutdownGraceMs: 100 }, + ); + + await core.init(); + const shutdownPromise = core.shutdown(); + await new Promise((resolve) => setTimeout(resolve, 400)); + releaseReward(); + await shutdownPromise; + await core.waitForStartupRecovery?.(); + + for (const suffix of ["a", "b"]) { + const episode = db!.repos.episodes.getById( + `ep_cancel_${suffix}` as never, + ); + expect(episode).toBeDefined(); + const meta = episode?.meta ?? {}; + const rewardDirty = meta.rewardDirty as + | { failedAttempts?: number; lastFailureAt?: number } + | undefined; + // Settled per-episode: still dirty (LLM failed) → failedAttempts + // bumped 0 → 1 with a fresh lastFailureAt. + expect(rewardDirty?.failedAttempts).toBe(1); + expect(typeof rewardDirty?.lastFailureAt).toBe("number"); + } + }); + + it("does not re-emit episodes that are still mid-drain when the periodic rescan hits (issue #2459 review)", async () => { + // The head-page pass re-reads the newest page every 10-minute tick. + // Episodes whose drain is still in flight keep their `rewardDirty` + // markers until phase-2 settles, so without an in-flight guard the + // periodic scan re-collects and re-emits the whole batch — the + // duplicate LLM spend gap 2 set out to eliminate. While the first + // drain is parked inside the reward chain, a periodic tick must not + // queue a second run for the same episodes. + const base = Date.now() - 5_000; + db!.repos.sessions.upsert({ + id: "se_inflight", + agent: "openclaw", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt: base, + lastSeenAt: base, + meta: {}, + }); + for (const suffix of ["a", "b"]) { + const id = `ep_inflight_${suffix}`; + const startedAt = base + (suffix === "a" ? 0 : 10); + db!.repos.episodes.insert({ + id, + sessionId: "se_inflight", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + startedAt, + endedAt: startedAt + 1, + traceIds: [`tr_inflight_${suffix}`] as never, + rTask: null, + status: "closed", + meta: { closeReason: "finalized" }, + }); + db!.repos.traces.insert({ + id: `tr_inflight_${suffix}`, + episodeId: id, + sessionId: "se_inflight", + ownerAgentKind: "openclaw", + ownerProfileId: "main", + ownerWorkspaceId: null, + ts: startedAt + 1, + userText: "在途防重测试:drain 未结算时周期扫描不得重发。", + agentText: "明白,在途行跳过。", + summary: null, + toolCalls: [], + reflection: null, + agentThinking: null, + value: 0, + alpha: 0, + rHuman: null, + priority: 0.5, + tags: [], + errorSignatures: [], + vecSummary: null, + vecAction: null, + share: null, + turnId: startedAt + 1, + schemaVersion: 1, + }); + } + + pipeline = createPipeline(buildDeps(db!)); + let rewardRuns = 0; + let releaseReward!: () => void; + const rewardGate = new Promise((resolve) => { + releaseReward = resolve; + }); + pipeline.rewardRunner.run = async () => { + rewardRuns++; + await rewardGate; + throw new Error("simulated llm outage during drain"); + }; + core = createMemoryCore( + pipeline, + resolveHome("openclaw", "/tmp/memos-mc-test"), + "issue2459-inflight", + ); + + // Only hijack the 10-minute periodic interval; leave timers used by + // the flush chain (setImmediate / setTimeout) on the real clock. + vi.useFakeTimers({ toFake: ["setInterval"] }); + try { + await core.init(); + // First (startup) drain collects both episodes and parks inside + // the gated reward chain. + await vi.waitFor(() => expect(rewardRuns).toBe(2)); + + // One periodic tick while the drain is still in flight. + await vi.advanceTimersByTimeAsync(10 * 60 * 1000); + // Give a (buggy) second collection time to run its phase-1 + flush + // up to the reward gate. + await new Promise((resolve) => setTimeout(resolve, 300)); + // Still exactly one run per episode — no re-emission. + expect(rewardRuns).toBe(2); + } finally { + vi.useRealTimers(); + releaseReward(); + } + }); + it("shutdown() awaits background recovery before tearing down storage (issue #1808)", async () => { // If `shutdown()` did not await the background recovery promise we // would close SQLite while the reflect / reward listeners were