Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions apps/memos-local-plugin/core/llm/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,27 @@ import type {
} from "./types.js";

const DEFAULT_MAX_TOKENS = 1024;
/**
* When a JSON completion stops with `finishReason: "length"`, retrying at the
* same `maxTokens` truncates at the same point. Thinking models share one
* budget between hidden reasoning and the answer, so a small budget can come
* back as an empty 200 OK. The retry therefore grows the budget (never shrinks
* it) up to this ceiling. The ceiling sits above the 4096 default that
* dedicated clients such as `l3Llm` / `skillEvolver` use, so their retries can
* still grow.
*/
const LENGTH_RETRY_MIN_TOKENS = 1024;
const LENGTH_RETRY_MAX_TOKENS = 8192;
const LENGTH_RETRY_GROWTH = 4;

export function nextLengthRetryMaxTokens(current: number | undefined): number {
const base = current ?? DEFAULT_MAX_TOKENS;
const grown = Math.min(
Math.max(base * LENGTH_RETRY_GROWTH, LENGTH_RETRY_MIN_TOKENS),
LENGTH_RETRY_MAX_TOKENS,
);
return Math.max(base, grown);
}

// ─── Factory ─────────────────────────────────────────────────────────────────

Expand Down Expand Up @@ -534,10 +555,27 @@ export function createLlmClientWithProvider(
jsonLog.warn("malformed", {
op,
attempt,
finishReason: completion.finishReason,
usage: completion.usage,
maxTokens: call.maxTokens,
err: summarizeErr(err),
});
if (attempt <= maxMalformedRetries) {
retries++;
// A truncated reply is truncated again at the same limit, so give
// the retry more room (see LENGTH_RETRY_* above).
if (completion.finishReason === "length") {
const fromMaxTokens = call.maxTokens;
call.maxTokens = nextLengthRetryMaxTokens(fromMaxTokens);
// Make the escalation (or a retry that cannot grow) visible in logs.
jsonLog.warn("length_retry_budget", {
op,
attempt,
fromMaxTokens,
toMaxTokens: call.maxTokens,
grew: call.maxTokens > (fromMaxTokens ?? 0),
});
}
continue;
}
}
Expand Down
4 changes: 3 additions & 1 deletion apps/memos-local-plugin/core/retrieval/llm-filter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ import type {
} from "./types.js";

const DEFAULT_CANDIDATE_BODY_CHARS = 500;
const MIN_FILTER_OUTPUT_TOKENS = 160;
// Thinking models spend part of max_tokens on hidden reasoning; at 160 they
// routinely returned an empty answer with finishReason "length".
const MIN_FILTER_OUTPUT_TOKENS = 1024;
const MAX_FILTER_OUTPUT_TOKENS = 2048;

export interface FilterInput {
Expand Down
45 changes: 45 additions & 0 deletions apps/memos-local-plugin/tests/unit/llm/client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
createLlmClientWithProvider,
registerHostLlmBridge,
} from "../../../core/llm/index.js";
import { nextLengthRetryMaxTokens } from "../../../core/llm/client.js";
import { initTestLogger } from "../../../core/logger/index.js";
import type {
LlmConfig,
Expand Down Expand Up @@ -171,6 +172,50 @@ describe("llm/client", () => {
expect(fake.invocations).toBe(2);
});

it("completeJson gives a length-truncated retry a larger maxTokens", async () => {
const seen: Array<number | undefined> = [];
const provider: LlmProvider = {
name: "openai_compatible",
async complete(_m, opts) {
seen.push(opts.maxTokens);
return seen.length === 1
? { text: "", finishReason: "length", durationMs: 1 }
: { text: '{"x":1}', finishReason: "stop", durationMs: 1 };
},
};
const client = createLlmClientWithProvider(cfg(), provider);
const r = await client.completeJson<{ x: number }>("ask", { maxTokens: 160 });
expect(r.value.x).toBe(1);
expect(seen).toEqual([160, 1024]);
});

it("completeJson keeps maxTokens on a retry that was not truncated", async () => {
const seen: Array<number | undefined> = [];
const provider: LlmProvider = {
name: "openai_compatible",
async complete(_m, opts) {
seen.push(opts.maxTokens);
return seen.length === 1
? { text: "not json", finishReason: "stop", durationMs: 1 }
: { text: '{"x":1}', finishReason: "stop", durationMs: 1 };
},
};
const client = createLlmClientWithProvider(cfg(), provider);
await client.completeJson<{ x: number }>("ask", { maxTokens: 160 });
expect(seen).toEqual([160, 160]);
});

it("nextLengthRetryMaxTokens grows, caps, and never shrinks", () => {
expect(nextLengthRetryMaxTokens(160)).toBe(1024);
expect(nextLengthRetryMaxTokens(512)).toBe(2048);
expect(nextLengthRetryMaxTokens(1024)).toBe(4096);
// 4096 is the l3Llm / skillEvolver default; its retry must still grow.
expect(nextLengthRetryMaxTokens(4096)).toBe(8192);
expect(nextLengthRetryMaxTokens(8192)).toBe(8192);
expect(nextLengthRetryMaxTokens(16384)).toBe(16384);
expect(nextLengthRetryMaxTokens(undefined)).toBe(4096);
});

it("stream passes provider-native chunks through", async () => {
const client = createLlmClientWithProvider(cfg(), new StreamingProvider());
const chunks: LlmStreamChunk[] = [];
Expand Down
14 changes: 14 additions & 0 deletions apps/memos-local-plugin/tests/unit/retrieval/llm-filter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -387,4 +387,18 @@ describe("retrieval/llm-filter", () => {
);
expect(llm.completeJson.mock.calls[0][1].maxTokens).toBeGreaterThan(512);
});

it("LLM output budget leaves room for thinking models on short lists", async () => {
const llm: any = {
completeJson: vi.fn().mockResolvedValue({
value: { ranked: [1], sufficient: false },
servedBy: "fake",
}),
};
await llmFilterCandidates(
{ query: "q", ranked: [trace("a", 0.9), trace("b", 0.8)] },
{ llm, log, config: cfg },
);
expect(llm.completeJson.mock.calls[0][1].maxTokens).toBeGreaterThanOrEqual(1024);
});
});