From 044d0f0b73599841f42024ea5ff3acaf362e216e Mon Sep 17 00:00:00 2001 From: jeff-ang92 <330780721+jeff-ang92@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:31:40 +0700 Subject: [PATCH 1/2] fix(plugin): grow maxTokens when a JSON reply is cut off at the limit completeJson retried a finishReason "length" reply at the same maxTokens, so it was cut off again at the same point. Thinking models share one budget between hidden reasoning and the answer, so small budgets come back as an empty 200 OK. The retry now grows the budget (x4, min 1024, max 4096, never smaller than before), and the malformed log records finishReason, usage and maxTokens. The retrieval filter's output floor was 160 tokens, which a thinking model used up before answering. Raise it to 1024. Co-Authored-By: Claude Opus 5.5 --- apps/memos-local-plugin/core/llm/client.ts | 27 ++++++++++++ .../core/retrieval/llm-filter.ts | 4 +- .../tests/unit/llm/client.test.ts | 43 +++++++++++++++++++ .../tests/unit/retrieval/llm-filter.test.ts | 14 ++++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 7e19f25ca..3a8171788 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -45,6 +45,25 @@ import type { } from "./types.js"; const DEFAULT_MAX_TOKENS = 1024; +/** + * When a JSON completion stops with `finishReason: "length"`, retrying at the + * same `maxTokens` truncates at the same point. Thinking models share one + * budget between hidden reasoning and the answer, so a small budget can come + * back as an empty 200 OK. The retry therefore grows the budget (never shrinks + * it) up to this ceiling. + */ +const LENGTH_RETRY_MIN_TOKENS = 1024; +const LENGTH_RETRY_MAX_TOKENS = 4096; +const LENGTH_RETRY_GROWTH = 4; + +export function nextLengthRetryMaxTokens(current: number | undefined): number { + const base = current ?? DEFAULT_MAX_TOKENS; + const grown = Math.min( + Math.max(base * LENGTH_RETRY_GROWTH, LENGTH_RETRY_MIN_TOKENS), + LENGTH_RETRY_MAX_TOKENS, + ); + return Math.max(base, grown); +} // ─── Factory ───────────────────────────────────────────────────────────────── @@ -534,10 +553,18 @@ export function createLlmClientWithProvider( jsonLog.warn("malformed", { op, attempt, + finishReason: completion.finishReason, + usage: completion.usage, + maxTokens: call.maxTokens, err: summarizeErr(err), }); if (attempt <= maxMalformedRetries) { retries++; + // A truncated reply is truncated again at the same limit, so give + // the retry more room (see LENGTH_RETRY_* above). + if (completion.finishReason === "length") { + call.maxTokens = nextLengthRetryMaxTokens(call.maxTokens); + } continue; } } diff --git a/apps/memos-local-plugin/core/retrieval/llm-filter.ts b/apps/memos-local-plugin/core/retrieval/llm-filter.ts index ee5988c00..b57fefc74 100644 --- a/apps/memos-local-plugin/core/retrieval/llm-filter.ts +++ b/apps/memos-local-plugin/core/retrieval/llm-filter.ts @@ -30,7 +30,9 @@ import type { } from "./types.js"; const DEFAULT_CANDIDATE_BODY_CHARS = 500; -const MIN_FILTER_OUTPUT_TOKENS = 160; +// Thinking models spend part of max_tokens on hidden reasoning; at 160 they +// routinely returned an empty answer with finishReason "length". +const MIN_FILTER_OUTPUT_TOKENS = 1024; const MAX_FILTER_OUTPUT_TOKENS = 2048; export interface FilterInput { diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index 51051ab00..79fb5f1a6 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -7,6 +7,7 @@ import { createLlmClientWithProvider, registerHostLlmBridge, } from "../../../core/llm/index.js"; +import { nextLengthRetryMaxTokens } from "../../../core/llm/client.js"; import { initTestLogger } from "../../../core/logger/index.js"; import type { LlmConfig, @@ -171,6 +172,48 @@ describe("llm/client", () => { expect(fake.invocations).toBe(2); }); + it("completeJson gives a length-truncated retry a larger maxTokens", async () => { + const seen: Array = []; + const provider: LlmProvider = { + name: "openai_compatible", + async complete(_m, opts) { + seen.push(opts.maxTokens); + return seen.length === 1 + ? { text: "", finishReason: "length", durationMs: 1 } + : { text: '{"x":1}', finishReason: "stop", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider(cfg(), provider); + const r = await client.completeJson<{ x: number }>("ask", { maxTokens: 160 }); + expect(r.value.x).toBe(1); + expect(seen).toEqual([160, 1024]); + }); + + it("completeJson keeps maxTokens on a retry that was not truncated", async () => { + const seen: Array = []; + const provider: LlmProvider = { + name: "openai_compatible", + async complete(_m, opts) { + seen.push(opts.maxTokens); + return seen.length === 1 + ? { text: "not json", finishReason: "stop", durationMs: 1 } + : { text: '{"x":1}', finishReason: "stop", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider(cfg(), provider); + await client.completeJson<{ x: number }>("ask", { maxTokens: 160 }); + expect(seen).toEqual([160, 160]); + }); + + it("nextLengthRetryMaxTokens grows, caps, and never shrinks", () => { + expect(nextLengthRetryMaxTokens(160)).toBe(1024); + expect(nextLengthRetryMaxTokens(512)).toBe(2048); + expect(nextLengthRetryMaxTokens(1024)).toBe(4096); + expect(nextLengthRetryMaxTokens(2048)).toBe(4096); + expect(nextLengthRetryMaxTokens(8192)).toBe(8192); + expect(nextLengthRetryMaxTokens(undefined)).toBe(4096); + }); + it("stream passes provider-native chunks through", async () => { const client = createLlmClientWithProvider(cfg(), new StreamingProvider()); const chunks: LlmStreamChunk[] = []; diff --git a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts index 4776efa39..325492385 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts @@ -387,4 +387,18 @@ describe("retrieval/llm-filter", () => { ); expect(llm.completeJson.mock.calls[0][1].maxTokens).toBeGreaterThan(512); }); + + it("LLM output budget leaves room for thinking models on short lists", async () => { + const llm: any = { + completeJson: vi.fn().mockResolvedValue({ + value: { ranked: [1], sufficient: false }, + servedBy: "fake", + }), + }; + await llmFilterCandidates( + { query: "q", ranked: [trace("a", 0.9), trace("b", 0.8)] }, + { llm, log, config: cfg }, + ); + expect(llm.completeJson.mock.calls[0][1].maxTokens).toBeGreaterThanOrEqual(1024); + }); }); From 00b3d15b520e16b3bb89a9c1a97dd5f671d38e64 Mon Sep 17 00:00:00 2001 From: jeff-ang92 <330780721+jeff-ang92@users.noreply.github.com> Date: Fri, 6 Nov 2026 23:01:09 +0700 Subject: [PATCH 2/2] fix(plugin): let 4096-token length retries grow and log the new budget Review follow-up. l3Llm and skillEvolver default to maxTokens 4096, so a 4096 ceiling left their length retry at the same budget. Raise the ceiling to 8192. Log a length_retry_budget warning with the old and new budget, and whether it grew, so the escalation is visible. Co-Authored-By: Claude Opus 5.5 --- apps/memos-local-plugin/core/llm/client.ts | 17 ++++++++++++++--- .../tests/unit/llm/client.test.ts | 4 +++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 3a8171788..649eacb84 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -50,10 +50,12 @@ const DEFAULT_MAX_TOKENS = 1024; * same `maxTokens` truncates at the same point. Thinking models share one * budget between hidden reasoning and the answer, so a small budget can come * back as an empty 200 OK. The retry therefore grows the budget (never shrinks - * it) up to this ceiling. + * it) up to this ceiling. The ceiling sits above the 4096 default that + * dedicated clients such as `l3Llm` / `skillEvolver` use, so their retries can + * still grow. */ const LENGTH_RETRY_MIN_TOKENS = 1024; -const LENGTH_RETRY_MAX_TOKENS = 4096; +const LENGTH_RETRY_MAX_TOKENS = 8192; const LENGTH_RETRY_GROWTH = 4; export function nextLengthRetryMaxTokens(current: number | undefined): number { @@ -563,7 +565,16 @@ export function createLlmClientWithProvider( // A truncated reply is truncated again at the same limit, so give // the retry more room (see LENGTH_RETRY_* above). if (completion.finishReason === "length") { - call.maxTokens = nextLengthRetryMaxTokens(call.maxTokens); + const fromMaxTokens = call.maxTokens; + call.maxTokens = nextLengthRetryMaxTokens(fromMaxTokens); + // Make the escalation (or a retry that cannot grow) visible in logs. + jsonLog.warn("length_retry_budget", { + op, + attempt, + fromMaxTokens, + toMaxTokens: call.maxTokens, + grew: call.maxTokens > (fromMaxTokens ?? 0), + }); } continue; } diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index 79fb5f1a6..ac942870a 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -209,8 +209,10 @@ describe("llm/client", () => { expect(nextLengthRetryMaxTokens(160)).toBe(1024); expect(nextLengthRetryMaxTokens(512)).toBe(2048); expect(nextLengthRetryMaxTokens(1024)).toBe(4096); - expect(nextLengthRetryMaxTokens(2048)).toBe(4096); + // 4096 is the l3Llm / skillEvolver default; its retry must still grow. + expect(nextLengthRetryMaxTokens(4096)).toBe(8192); expect(nextLengthRetryMaxTokens(8192)).toBe(8192); + expect(nextLengthRetryMaxTokens(16384)).toBe(16384); expect(nextLengthRetryMaxTokens(undefined)).toBe(4096); });