From 50b1e5d3b2a4e2d1099762adefc919aad2e650c8 Mon Sep 17 00:00:00 2001 From: autodev Date: Fri, 9 Oct 2026 04:26:13 +0800 Subject: [PATCH 1/2] fix(memos-local-plugin): make capture summarizer language-stable (#2469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The capture SYSTEM_PROMPT anchored its language rule on an ambiguous referent ("the user's original language") and carried a lone CJK example ("用户说了") that primed some models (observed with openai/gpt-6-luna at temp 0) to emit Chinese summaries for English conversations — up to 89% CJK on tool-call summaries and 66% on conversational summaries. Fix per issue #2469 (reporter-validated on the same model, 0/30 CJK on English inputs after the change): - Replace "in the user's original language" with "written in the same language as the USER text (English text gets an English summary)". - Remove the Chinese example from the "Do NOT prefix" rule, keeping only "The user said". Also adds a regression unit test that spies on the system message sent to the LLM and asserts the Chinese example is gone and the language rule is anchored on the USER text. --- .../core/capture/summarizer.ts | 5 +- .../unit/capture/summarizer-prompt.test.ts | 128 ++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts diff --git a/apps/memos-local-plugin/core/capture/summarizer.ts b/apps/memos-local-plugin/core/capture/summarizer.ts index 9903f0048..a547b2fee 100644 --- a/apps/memos-local-plugin/core/capture/summarizer.ts +++ b/apps/memos-local-plugin/core/capture/summarizer.ts @@ -103,10 +103,11 @@ const SYSTEM_PROMPT = `You condense a single agent/user exchange into ONE short Rules: - Output MUST be a single JSON object: { "summary": "..." } -- The summary must be ≤ 100 characters, in the user's original language. +- The summary must be ≤ 100 characters, written in the same language as the + USER text (English text gets an English summary). - Focus on the *fact worth remembering next time* — a preference, a name, a decision, a file path, an error signature, an answer that was confirmed. -- Do NOT prefix with "The user said" / "用户说了". Just state the fact. +- Do NOT prefix with "The user said". Just state the fact. - Do NOT quote whole sentences. Compress. - If nothing is worth remembering, still produce a short summary (e.g. the main topic of the exchange) — never return an empty string.`; diff --git a/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts b/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts new file mode 100644 index 000000000..c1cc18184 --- /dev/null +++ b/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts @@ -0,0 +1,128 @@ +/** + * Regression test for issue #2469 — the capture summarizer was writing + * Chinese summaries for English conversations because the SYSTEM_PROMPT + * (a) anchored the language rule on an ambiguous referent ("the user's + * original language") and (b) carried a single CJK example ("用户说了") + * that primed the model toward Chinese output under the 100-char cap. + * + * These tests do not call a real LLM. Instead they install a spy + * `LlmClient.completeJson` that records the exact `messages` array + * the summarizer sends, and assert on the system-prompt content. + * + * If these assertions start failing, the prompt text has drifted; + * revisit issue #2469 before loosening them. + */ + +import { describe, expect, it } from "vitest"; + +import { createSummarizer } from "../../../core/capture/summarizer.js"; +import type { NormalizedStep } from "../../../core/capture/types.js"; +import type { + LlmClient, + LlmJsonCompletion, + LlmMessage, +} from "../../../core/llm/types.js"; +import type { EpochMs } from "../../../core/types.js"; + +function makeStep(partial: Partial = {}): NormalizedStep { + return { + key: partial.key ?? "s1", + ts: partial.ts ?? ((1_700_000_000_000 as unknown) as EpochMs), + userText: partial.userText ?? "", + agentText: partial.agentText ?? "", + toolCalls: partial.toolCalls ?? [], + rawReflection: partial.rawReflection ?? null, + depth: partial.depth ?? 0, + isSubagent: partial.isSubagent ?? false, + meta: partial.meta ?? {}, + truncated: partial.truncated ?? false, + }; +} + +interface SpyRecorder { + calls: Array<{ messages: LlmMessage[] }>; +} + +function spyLlm(recorder: SpyRecorder, summary = "ok"): LlmClient { + return { + provider: "openai_compatible", + model: "spy-model", + canStream: false, + async complete() { + throw new Error("spyLlm.complete not expected"); + }, + async completeJson( + messages: LlmMessage[] | string, + ): Promise> { + recorder.calls.push({ + messages: Array.isArray(messages) ? messages : [], + }); + const value = { summary } as unknown as T; + return { + value, + raw: JSON.stringify(value), + provider: "openai_compatible", + model: "spy-model", + finishReason: "stop", + servedBy: "openai_compatible", + durationMs: 1, + }; + }, + stream(): AsyncIterable { + return (async function* () {})(); + }, + stats() { + return { + provider: "openai_compatible", + model: "spy-model", + requests: 0, + failures: 0, + malformedJson: 0, + timeouts: 0, + circuitOpen: false, + circuitOpenUntil: null, + circuitOpenedReason: null, + }; + }, + resetStats() {}, + async close() {}, + }; +} + +describe("capture/summarizer SYSTEM_PROMPT (issue #2469)", () => { + it("does not contain the Chinese '用户说了' example", async () => { + const recorder: SpyRecorder = { calls: [] }; + const summarizer = createSummarizer({ llm: spyLlm(recorder) }); + + await summarizer.summarize( + makeStep({ userText: "Hey, kick off a heartbeat please" }), + ); + + expect(recorder.calls).toHaveLength(1); + const sys = recorder.calls[0]!.messages.find((m) => m.role === "system"); + expect(sys).toBeTruthy(); + // No CJK example may leak into the prompt — that is what primed the + // model toward Chinese output on some backends. + expect(sys!.content).not.toContain("用户说了"); + // Nothing with Han characters at all; keep the rail tight. + expect(/[一-鿿]/.test(sys!.content)).toBe(false); + }); + + it("anchors the language rule on the USER text, not an ambiguous referent", async () => { + const recorder: SpyRecorder = { calls: [] }; + const summarizer = createSummarizer({ llm: spyLlm(recorder) }); + + await summarizer.summarize(makeStep({ userText: "hello" })); + + const sys = recorder.calls[0]!.messages.find((m) => m.role === "system"); + expect(sys).toBeTruthy(); + // Must point at the USER text as the language source so the model + // cannot drift toward a different language when the only non-English + // token in context is incidental. + expect(sys!.content).toContain("USER text"); + // The pre-fix wording ("in the user's original language") was the + // ambiguous one that triggered this bug. Guard against regressions + // that silently reintroduce it. + expect(sys!.content).not.toContain("original language"); + }); +}); From 7818adfd88585d2d4984661b4ce6dcd0e7ad7347 Mon Sep 17 00:00:00 2001 From: autodev Date: Fri, 9 Oct 2026 04:34:06 +0800 Subject: [PATCH 2/2] fix(memos-local-plugin): broaden summarizer prompt language coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address OCR findings on PR #2471 without reintroducing the CJK anchor that originally caused issue #2469. OCR finding 1 (summarizer.ts L106-L107): the single English example ("English text gets an English summary") may still anchor the model to English for non-English conversations. Add a French worked example and spell out that the rule applies to every other language. OCR finding 2 (summarizer.ts L110): the English-only "Do NOT prefix with 'The user said'" rule may let the model prepend an equivalent opener in a non-English conversation (e.g. a Chinese "用户说了…"). Rather than re-adding the specific CJK anchor — issue #2469 proved that a lone "用户说了" example drove 89% CJK output on tool-call summaries with openai/gpt-6-luna, which the removal fixed to 0/30 — make the rule explicitly language-agnostic ("the equivalent 'the user said…' opener in any other language"). That covers Chinese, Japanese, Korean, French, etc. without priming any one of them. Extend the regression test with two new cases: one asserts the positive rule carries both English and French worked examples, the other asserts the negative-prefix rule reaches beyond English via an "any other language" clause. The two original guards (no "用户说了", no Han characters at all) still hold. --- .../core/capture/summarizer.ts | 7 ++- .../unit/capture/summarizer-prompt.test.ts | 44 ++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/apps/memos-local-plugin/core/capture/summarizer.ts b/apps/memos-local-plugin/core/capture/summarizer.ts index a547b2fee..29fa4a467 100644 --- a/apps/memos-local-plugin/core/capture/summarizer.ts +++ b/apps/memos-local-plugin/core/capture/summarizer.ts @@ -104,10 +104,13 @@ const SYSTEM_PROMPT = `You condense a single agent/user exchange into ONE short Rules: - Output MUST be a single JSON object: { "summary": "..." } - The summary must be ≤ 100 characters, written in the same language as the - USER text (English text gets an English summary). + USER text (e.g. English USER text → English summary, French USER text → + French summary; the same rule applies to every other language). - Focus on the *fact worth remembering next time* — a preference, a name, a decision, a file path, an error signature, an answer that was confirmed. -- Do NOT prefix with "The user said". Just state the fact. +- Do NOT prefix with "The user said", or with the equivalent "the user + said…" opener in any other language (whatever language the summary is + in). Just state the fact. - Do NOT quote whole sentences. Compress. - If nothing is worth remembering, still produce a short summary (e.g. the main topic of the exchange) — never return an empty string.`; diff --git a/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts b/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts index c1cc18184..e2a760fcc 100644 --- a/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts +++ b/apps/memos-local-plugin/tests/unit/capture/summarizer-prompt.test.ts @@ -9,8 +9,12 @@ * `LlmClient.completeJson` that records the exact `messages` array * the summarizer sends, and assert on the system-prompt content. * + * The last two cases guard against OCR review feedback on PR #2471 + * (English-only positive example, English-only negative-prefix rule). + * * If these assertions start failing, the prompt text has drifted; - * revisit issue #2469 before loosening them. + * revisit issue #2469 and the PR #2471 OCR findings before loosening + * them. */ import { describe, expect, it } from "vitest"; @@ -125,4 +129,42 @@ describe("capture/summarizer SYSTEM_PROMPT (issue #2469)", () => { // that silently reintroduce it. expect(sys!.content).not.toContain("original language"); }); + + it("gives the language rule more than one worked example", async () => { + // OCR review of PR #2471 flagged that anchoring the language rule on + // a single English example may still prime models to default to + // English for non-English conversations. The prompt now carries at + // least one additional, non-English worked example (currently + // French) so no single language dominates the positive rule. + const recorder: SpyRecorder = { calls: [] }; + const summarizer = createSummarizer({ llm: spyLlm(recorder) }); + + await summarizer.summarize(makeStep({ userText: "hello" })); + + const sys = recorder.calls[0]!.messages.find((m) => m.role === "system"); + expect(sys).toBeTruthy(); + expect(sys!.content).toContain("English USER text"); + expect(sys!.content).toContain("French USER text"); + }); + + it("extends the negative-prefix rule to languages beyond English", async () => { + // OCR review of PR #2471 was concerned that removing the Chinese + // "用户说了" example could regress Chinese conversations — the model + // might still prepend a Chinese "the user said…" opener because + // only the English form was called out. We cover that by making + // the rule explicitly language-agnostic rather than by re-adding a + // CJK example (the lone CJK anchor was the proven cause of + // #2469), so the guard holds for Chinese, Japanese, Korean, + // French, and every other language without priming any one of + // them. + const recorder: SpyRecorder = { calls: [] }; + const summarizer = createSummarizer({ llm: spyLlm(recorder) }); + + await summarizer.summarize(makeStep({ userText: "hello" })); + + const sys = recorder.calls[0]!.messages.find((m) => m.role === "system"); + expect(sys).toBeTruthy(); + expect(sys!.content).toContain("The user said"); + expect(sys!.content.toLowerCase()).toContain("any other language"); + }); });