diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 7e19f25ca..649eacb84 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -45,6 +45,27 @@ import type { } from "./types.js"; const DEFAULT_MAX_TOKENS = 1024; +/** + * When a JSON completion stops with `finishReason: "length"`, retrying at the + * same `maxTokens` truncates at the same point. Thinking models share one + * budget between hidden reasoning and the answer, so a small budget can come + * back as an empty 200 OK. The retry therefore grows the budget (never shrinks + * it) up to this ceiling. The ceiling sits above the 4096 default that + * dedicated clients such as `l3Llm` / `skillEvolver` use, so their retries can + * still grow. + */ +const LENGTH_RETRY_MIN_TOKENS = 1024; +const LENGTH_RETRY_MAX_TOKENS = 8192; +const LENGTH_RETRY_GROWTH = 4; + +export function nextLengthRetryMaxTokens(current: number | undefined): number { + const base = current ?? DEFAULT_MAX_TOKENS; + const grown = Math.min( + Math.max(base * LENGTH_RETRY_GROWTH, LENGTH_RETRY_MIN_TOKENS), + LENGTH_RETRY_MAX_TOKENS, + ); + return Math.max(base, grown); +} // ─── Factory ───────────────────────────────────────────────────────────────── @@ -534,10 +555,27 @@ export function createLlmClientWithProvider( jsonLog.warn("malformed", { op, attempt, + finishReason: completion.finishReason, + usage: completion.usage, + maxTokens: call.maxTokens, err: summarizeErr(err), }); if (attempt <= maxMalformedRetries) { retries++; + // A truncated reply is truncated again at the same limit, so give + // the retry more room (see LENGTH_RETRY_* above). + if (completion.finishReason === "length") { + const fromMaxTokens = call.maxTokens; + call.maxTokens = nextLengthRetryMaxTokens(fromMaxTokens); + // Make the escalation (or a retry that cannot grow) visible in logs. + jsonLog.warn("length_retry_budget", { + op, + attempt, + fromMaxTokens, + toMaxTokens: call.maxTokens, + grew: call.maxTokens > (fromMaxTokens ?? 0), + }); + } continue; } } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index ee5988c00..b57fefc74 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -30,7 +30,9 @@ import type { } from "./types.js"; const DEFAULT_CANDIDATE_BODY_CHARS = 500; -const MIN_FILTER_OUTPUT_TOKENS = 160; +// Thinking models spend part of max_tokens on hidden reasoning; at 160 they +// routinely returned an empty answer with finishReason "length". +const MIN_FILTER_OUTPUT_TOKENS = 1024; const MAX_FILTER_OUTPUT_TOKENS = 2048; export interface FilterInput { diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index 51051ab00..ac942870a 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -7,6 +7,7 @@ import { createLlmClientWithProvider, registerHostLlmBridge, } from "../../../core/llm/index.js"; +import { nextLengthRetryMaxTokens } from "../../../core/llm/client.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmConfig, @@ -171,6 +172,50 @@ describe("llm/client", () => { expect(fake.invocations).toBe(2); }); + it("completeJson gives a length-truncated retry a larger maxTokens", async () => { + const seen: Array = []; + const provider: LlmProvider = { + name: "openai_compatible", + async complete(_m, opts) { + seen.push(opts.maxTokens); + return seen.length === 1 + ? { text: "", finishReason: "length", durationMs: 1 } + : { text: '{"x":1}', finishReason: "stop", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider(cfg(), provider); + const r = await client.completeJson<{ x: number }>("ask", { maxTokens: 160 }); + expect(r.value.x).toBe(1); + expect(seen).toEqual([160, 1024]); + }); + + it("completeJson keeps maxTokens on a retry that was not truncated", async () => { + const seen: Array = []; + const provider: LlmProvider = { + name: "openai_compatible", + async complete(_m, opts) { + seen.push(opts.maxTokens); + return seen.length === 1 + ? { text: "not json", finishReason: "stop", durationMs: 1 } + : { text: '{"x":1}', finishReason: "stop", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider(cfg(), provider); + await client.completeJson<{ x: number }>("ask", { maxTokens: 160 }); + expect(seen).toEqual([160, 160]); + }); + + it("nextLengthRetryMaxTokens grows, caps, and never shrinks", () => { + expect(nextLengthRetryMaxTokens(160)).toBe(1024); + expect(nextLengthRetryMaxTokens(512)).toBe(2048); + expect(nextLengthRetryMaxTokens(1024)).toBe(4096); + // 4096 is the l3Llm / skillEvolver default; its retry must still grow. + expect(nextLengthRetryMaxTokens(4096)).toBe(8192); + expect(nextLengthRetryMaxTokens(8192)).toBe(8192); + expect(nextLengthRetryMaxTokens(16384)).toBe(16384); + expect(nextLengthRetryMaxTokens(undefined)).toBe(4096); + }); + it("stream passes provider-native chunks through", async () => { const client = createLlmClientWithProvider(cfg(), new StreamingProvider()); const chunks: LlmStreamChunk[] = []; diff --git a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts index 4776efa39..325492385 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts @@ -387,4 +387,18 @@ describe("retrieval/llm-filter", () => { ); expect(llm.completeJson.mock.calls[0][1].maxTokens).toBeGreaterThan(512); }); + + it("LLM output budget leaves room for thinking models on short lists", async () => { + const llm: any = { + completeJson: vi.fn().mockResolvedValue({ + value: { ranked: [1], sufficient: false }, + servedBy: "fake", + }), + }; + await llmFilterCandidates( + { query: "q", ranked: [trace("a", 0.9), trace("b", 0.8)] }, + { llm, log, config: cfg }, + ); + expect(llm.completeJson.mock.calls[0][1].maxTokens).toBeGreaterThanOrEqual(1024); + }); });