From 8849727a838e697dac79b4f9f1f66bf87e3b28fa Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Wed, 21 Oct 2026 18:23:08 +0800 Subject: [PATCH 1/3] fix: keep stable system prompt + history as the shared prefix (#2524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Chat used to splice the current turn's retrieval and current minute into the system message and send that first, with chat history after it. The first message therefore changed on every turn, so provider prefix caches could not reuse the identical history that followed. Split the single system blob into (a) a stable system prompt that is byte-identical across turns and (b) a per-turn dynamic context carrying the current time, retrieved memories, and preferences. Place the stable prompt first, then chat history, then the dynamic context, then the user query — so the shared prefix across requests now contains the chat history and the fixed instructions. Applied in ChatHandler.handle_chat_complete / handle_chat_stream / handle_chat_stream_playground / handle_chat_stream_for_business_user, MOSCore.chat, MOS._generate_synthesis_response, and SimpleMemChat.run. Legacy single-string callers (base_prompt with inline {memories}) keep the old shape for backward compatibility. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/memos/api/handlers/chat_handler.py | 285 ++++++++++++++++---- src/memos/mem_chat/simple.py | 43 ++- src/memos/mem_os/core.py | 69 ++++- src/memos/mem_os/main.py | 25 +- src/memos/templates/cloud_service_prompt.py | 101 +++++++ tests/api/test_chat_message_order.py | 280 +++++++++++++++++++ tests/mem_os/test_memos_core.py | 68 ++++- 7 files changed, 797 insertions(+), 74 deletions(-) create mode 100644 tests/api/test_chat_message_order.py diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 31acec6d0..4eb855d50 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -40,7 +40,11 @@ ANSWER_TASK_LABEL, QUERY_TASK_LABEL, ) -from memos.templates.cloud_service_prompt import get_cloud_chat_prompt +from memos.templates.cloud_service_prompt import ( + get_cloud_chat_dynamic_context, + get_cloud_chat_prompt, + get_cloud_chat_stable_prompt, +) from memos.templates.mos_prompts import ( FURTHER_SUGGESTION_PROMPT, get_memos_prompt, @@ -158,22 +162,21 @@ def handle_chat_complete(self, chat_req: APIChatCompleteRequest) -> dict[str, An memories_list, chat_req.threshold or 0.5 ) - # Step 2: Build system prompt - system_prompt = self._build_system_prompt( + # Step 2: Build cache-friendly message list (Issue #2524): + # stable system prompt first, then chat history, then a dynamic + # context message carrying the current time / retrieved memories + # / preferences, then the current user query. The reorder keeps + # [stable_system, ...prior_history] identical across turns so + # provider prefix caches can reuse the history. + history_info = chat_req.history[-20:] if chat_req.history else [] + current_messages = self._assemble_cacheable_messages( query=chat_req.query, + history=history_info, memories=filtered_memories, pref_string=search_response.data.get("pref_string", ""), base_prompt=chat_req.system_prompt, ) - # Prepare message history - history_info = chat_req.history[-20:] if chat_req.history else [] - current_messages = [ - {"role": "system", "content": system_prompt}, - *history_info, - {"role": "user", "content": chat_req.query}, - ] - self.logger.info("[Cloud Service] Starting to generate chat complete response...") # Step 3: Generate complete response from LLM @@ -306,21 +309,18 @@ def generate_chat_response() -> Generator[str, None, None]: # Filter memories by threshold filtered_memories = self._filter_memories_by_threshold(memories_list) - # Step 2: Build system prompt with memories - system_prompt = self._build_system_prompt( + # Step 2: Build cache-friendly message list (Issue #2524) + history_info = chat_req.history[-20:] if chat_req.history else [] + current_messages = self._assemble_cacheable_messages( query=chat_req.query, + history=history_info, memories=filtered_memories, pref_string=search_response.data.get("pref_string", ""), base_prompt=chat_req.system_prompt, ) - - # Prepare messages - history_info = chat_req.history[-20:] if chat_req.history else [] - current_messages = [ - {"role": "system", "content": system_prompt}, - *history_info, - {"role": "user", "content": chat_req.query}, - ] + # Backward-compatible: expose the stable prompt in the log + # (callers used to log ``system_prompt`` here). + system_prompt = current_messages[0]["content"] self.logger.info( f"[Cloud Service] chat stream user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}, " @@ -599,7 +599,11 @@ def generate_chat_response() -> Generator[str, None, None]: ) yield f"data: {json.dumps({'type': 'reference', 'data': reference}, ensure_ascii=False)}\n\n" - # Step 2: Build system prompt with memories + # Step 2: Build cache-friendly message list (Issue #2524). + # Playground uses enhance-mode prompts: the stable prompt + # carries the fixed persona / instructions, and the dynamic + # context carries the current time + memories + preferences + # for this turn. lang = detect_lang(chat_req.query) if pref_string: pref_string += ( @@ -607,17 +611,24 @@ def generate_chat_response() -> Generator[str, None, None]: if lang == "zh" else "\n#warning\n- In thinking content, do not appear the reference number and id [1,2,3]etc. otherwise it will cause reference error." ) - system_prompt = self._build_enhance_system_prompt( - filtered_memories, pref_string, lang=lang + + stable_prompt = self._build_enhance_stable_prompt(lang=lang) + dynamic_ctx = self._build_enhance_dynamic_context( + memories_list=filtered_memories, + pref_string=pref_string, + lang=lang, ) - # Prepare messages history_info = chat_req.history[-20:] if chat_req.history else [] current_messages = [ - {"role": "system", "content": system_prompt}, + {"role": "system", "content": stable_prompt}, *history_info, + {"role": "system", "content": dynamic_ctx}, {"role": "user", "content": chat_req.query}, ] + # Downstream code still expects a ``system_prompt`` symbol + # for logging / speed-improvement estimation. + system_prompt = stable_prompt + "\n\n" + dynamic_ctx self.logger.info( f"[PLAYGROUND CHAT] user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}, " @@ -832,33 +843,27 @@ def generate_chat_response() -> Generator[str, None, None]: # Filter memories by threshold filtered_memories = self._filter_memories_by_threshold(memories_list) - # Step 2: Build system prompt with memories - system_prompt = self._build_system_prompt( - query=chat_req.query, - memories=filtered_memories, - pref_string=search_response.data.get("pref_string", ""), - base_prompt=chat_req.system_prompt, - ) - + pref_string_for_prompt = search_response.data.get("pref_string", "") self.logger.info( - f"[ChatBusinessHandler] chat stream user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}, " - f"current_system_prompt: {system_prompt}" + f"[ChatBusinessHandler] chat stream user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}" ) else: - system_prompt = self._build_system_prompt( - query=chat_req.query, - memories=None, - pref_string=None, - base_prompt=chat_req.system_prompt, - ) + filtered_memories = None + pref_string_for_prompt = None - # Prepare messages + # Build cache-friendly message list (Issue #2524) history_info = chat_req.history[-20:] if chat_req.history else [] - current_messages = [ - {"role": "system", "content": system_prompt}, - *history_info, - {"role": "user", "content": chat_req.query}, - ] + current_messages = self._assemble_cacheable_messages( + query=chat_req.query, + history=history_info, + memories=filtered_memories, + pref_string=pref_string_for_prompt, + base_prompt=chat_req.system_prompt, + ) + system_prompt = current_messages[0]["content"] + self.logger.info( + f"[ChatBusinessHandler] current_system_prompt: {system_prompt}" + ) # Step 3: Generate streaming response from LLM if ( @@ -1104,6 +1109,190 @@ def _build_enhance_system_prompt( + f"\n\n{pref_string}" ) + # ------------------------------------------------------------------ + # Prefix-cache-friendly message assembly (Issue #2524) + # ------------------------------------------------------------------ + # + # The handlers above used to send the per-turn retrieval and current + # time as the first (system) message, with chat history after it. + # Because the first message changed every turn, provider prefix caches + # could not reuse the chat history that followed. + # + # The helpers below split the single system blob into: + # * a stable system prompt (constant across turns in one conversation), + # emitted as ``messages[0]`` so that the history that follows is part + # of the shared prefix, and + # * a per-turn dynamic context message that carries the current time, + # retrieved memories, and preferences, inserted immediately before + # the final user turn. + # + # Call sites use ``_assemble_cacheable_messages`` to produce the full + # ``messages`` list. Legacy callers that pass a ``base_prompt`` with + # ``{memories}`` keep the single-system shape for backward compatibility. + + def _format_memories_for_dynamic_block(self, memories: list | None) -> str: + """Return the memory list rendered as "1. ...\\n2. ..." (empty when + there are no memories). Mirrors ``_build_system_prompt`` so legacy + and cache-friendly paths produce the same memory text.""" + if not memories: + return "" + rendered: list[str] = [] + for i, memory in enumerate(memories, 1): + text_memory = memory.get("memory", "") if isinstance(memory, dict) else str(memory) + rendered.append(f"{i}. {text_memory}") + return "\n".join(rendered) + + def _build_stable_system_prompt( + self, + *, + query: str, + base_prompt: str | None, + ) -> str: + """Return the stable (cross-turn constant) system prompt. + + * If ``base_prompt`` is ``None``: return the stable cloud chat + prompt for the detected language (no ``{current_time}``, no + ``{memories}`` substitution). + * If ``base_prompt`` contains ``{memories}``: this is the legacy + single-string path; the caller is expected to use the legacy + assembly instead (``_assemble_cacheable_messages`` detects this + and falls back automatically). The method still returns the + base_prompt text with ``{memories}`` stripped so it is safe to + emit as a stable prefix. + * Otherwise return ``base_prompt`` as-is. + """ + if base_prompt is None: + lang = detect_lang(query) + return get_cloud_chat_stable_prompt(lang=lang) + if "{memories}" in base_prompt: + # Caller used the legacy single-string pattern. The stable half + # is the template with the memory slot removed. + return base_prompt.replace("{memories}", "").rstrip() + return base_prompt + + def _build_dynamic_context_message( + self, + *, + memories: list | None, + pref_string: str | None, + lang: str, + include_time: bool = True, + ) -> str | None: + """Return the per-turn dynamic context, or ``None`` when there is + nothing dynamic to emit. + + The returned string is intended to be sent as a system message + placed immediately before the final user turn. + """ + memories_text = self._format_memories_for_dynamic_block(memories) + pref_text = (pref_string or "").strip() + if not memories_text and not pref_text and not include_time: + return None + return get_cloud_chat_dynamic_context( + memories_block=memories_text, + pref_string=pref_text or None, + lang=lang, + ) + + def _assemble_cacheable_messages( + self, + *, + query: str, + history: list[dict] | None, + memories: list | None, + pref_string: str | None, + base_prompt: str | None, + lang: str | None = None, + include_dynamic_time: bool = True, + ) -> list[dict]: + """Build the provider ``messages`` list in prefix-cache-friendly order. + + Shape (default path): + [stable system, *history, dynamic context (optional), user] + + Legacy path — triggered when ``base_prompt`` contains ``{memories}``: + [legacy system(stable + inline memories), *history, user] + """ + history_list = list(history or []) + + # Legacy backward-compatible path for custom ``base_prompt``. + if base_prompt and "{memories}" in base_prompt: + legacy_system = self._build_system_prompt( + query=query, + memories=memories, + pref_string=pref_string, + base_prompt=base_prompt, + ) + return [ + {"role": "system", "content": legacy_system}, + *history_list, + {"role": "user", "content": query}, + ] + + stable_system = self._build_stable_system_prompt(query=query, base_prompt=base_prompt) + resolved_lang = lang or detect_lang(query) + dynamic_ctx = self._build_dynamic_context_message( + memories=memories, + pref_string=pref_string, + lang=resolved_lang, + include_time=include_dynamic_time, + ) + + messages: list[dict] = [{"role": "system", "content": stable_system}] + messages.extend(history_list) + if dynamic_ctx: + messages.append({"role": "system", "content": dynamic_ctx}) + messages.append({"role": "user", "content": query}) + return messages + + def _build_enhance_stable_prompt( + self, + *, + lang: str = "en", + tone: str = "friendly", + verbosity: str = "mid", + ) -> str: + """Return the stable (cross-turn constant) enhance-mode system prompt.""" + # Keep the ``{date}`` placeholder out of the stable text so this + # string is identical across turns. + sys_body = get_memos_prompt( + date="see the context message immediately before the latest user turn", + tone=tone, + verbosity=verbosity, + mode="enhance", + lang=lang, + ) + return sys_body + + def _build_enhance_dynamic_context( + self, + *, + memories_list: list, + pref_string: str = "", + lang: str = "en", + ) -> str: + """Return the per-turn dynamic context for the enhance (playground) + chat endpoint (current time + PersonalMemory + OuterMemory + prefs).""" + now = datetime.now() + formatted_date = now.strftime("%Y-%m-%d %H:%M (%A)") + mem_block_o, mem_block_p = self._format_mem_block(memories_list) + pref_block = (pref_string or "").strip() + header_time = ( + f"# 当前上下文\n- 当前时间: {formatted_date}" + if lang == "zh" + else f"# Current Context\n- Current Time: {formatted_date}" + ) + parts = [ + header_time, + "\n# Memories\n## PersonalMemory (ordered)\n" + + mem_block_p + + "\n## OuterMemory (from Internet Search, ordered)\n" + + mem_block_o, + ] + if pref_block: + parts.append("\n" + pref_block) + return "\n".join(parts) + def _format_mem_block( self, memories_all: list, max_items: int = 10, max_chars_each: int = 1600 ) -> tuple[str, str]: diff --git a/src/memos/mem_chat/simple.py b/src/memos/mem_chat/simple.py index 765ad1a09..d02070679 100644 --- a/src/memos/mem_chat/simple.py +++ b/src/memos/mem_chat/simple.py @@ -81,14 +81,19 @@ def run(self) -> None: print( f"🧠 [Memory] Searched memories:\n{self._str_memories(memories, mode='concise')}\n" ) - system_prompt = self._build_system_prompt(memories) + stable_system = self._build_stable_system_prompt() + dynamic_ctx = self._build_dynamic_memory_context(memories) else: - system_prompt = self._build_system_prompt() - current_messages = [ - {"role": "system", "content": system_prompt}, - *messages, - {"role": "user", "content": user_input}, - ] + stable_system = self._build_stable_system_prompt() + dynamic_ctx = None + # Issue #2524: stable system first, then history, then dynamic + # context, then current user turn — keeps the chat history inside + # the provider's shared prefix. + current_messages: list[dict] = [{"role": "system", "content": stable_system}] + current_messages.extend(messages) + if dynamic_ctx: + current_messages.append({"role": "system", "content": dynamic_ctx}) + current_messages.append({"role": "user", "content": user_input}) if self.config.enable_activation_memory: past_key_values = None @@ -154,6 +159,30 @@ def _build_system_prompt(self, memories: list | None = None) -> str: return base_prompt + # Prefix-cache-friendly split (Issue #2524) + def _build_stable_system_prompt(self) -> str: + """Return the stable (cross-turn constant) system prompt.""" + return ( + "You are a knowledgeable and helpful AI assistant. " + "You have access to conversation memories that help you provide more personalized responses. " + "Use the memories to understand the user's context, preferences, and past interactions. " + "If memories are provided, reference them naturally when relevant, but don't explicitly mention having memories." + ) + + def _build_dynamic_memory_context(self, memories: list | None) -> str | None: + """Return a system message carrying this turn's retrieved memories. + + Returned string is intended to sit immediately before the latest + user turn so the chat history that precedes it stays part of the + provider's shared prefix. + """ + if not memories: + return None + lines = ["## Memories"] + for i, memory in enumerate(memories, 1): + lines.append(f"{i}. ({memory.metadata.memory_time}) {memory.memory}") + return "\n".join(lines) + def _str_memories( self, memories: list[TextualMemoryItem], mode: Literal["concise", "full"] = "full" ) -> str: diff --git a/src/memos/mem_os/core.py b/src/memos/mem_os/core.py index 3ede965d3..83fdace0b 100644 --- a/src/memos/mem_os/core.py +++ b/src/memos/mem_os/core.py @@ -300,14 +300,21 @@ def chat(self, query: str, user_id: str | None = None, base_prompt: str | None = ) memories_all.extend(memories) logger.info(f"🧠 [Memory] Searched memories:\n{self._str_memories(memories_all)}\n") - system_prompt = self._build_system_prompt(memories_all, base_prompt=base_prompt) + # Issue #2524: emit a stable system prompt first, then history, + # then a dynamic context message with the retrieved memories, + # so provider prefix caches can reuse the history across turns. + stable_system, dynamic_ctx = self._build_cacheable_system_parts( + memories_all, base_prompt=base_prompt + ) else: - system_prompt = self._build_system_prompt(base_prompt=base_prompt) - current_messages = [ - {"role": "system", "content": system_prompt}, - *chat_history.chat_history, - {"role": "user", "content": query}, - ] + stable_system, dynamic_ctx = self._build_cacheable_system_parts( + None, base_prompt=base_prompt + ) + current_messages: list[dict] = [{"role": "system", "content": stable_system}] + current_messages.extend(chat_history.chat_history) + if dynamic_ctx: + current_messages.append({"role": "system", "content": dynamic_ctx}) + current_messages.append({"role": "user", "content": query}) past_key_values = None if self.config.enable_activation_memory: @@ -387,6 +394,54 @@ def _build_system_prompt( return base_prompt + memory_context_with_header return base_prompt + # Prefix-cache-friendly system split (Issue #2524) + def _build_cacheable_system_parts( + self, + memories: list[TextualMemoryItem] | list[str] | None = None, + base_prompt: str | None = None, + ) -> tuple[str, str | None]: + """Return ``(stable_system, dynamic_context)`` for cache-friendly + chat assembly. + + * ``stable_system`` is the same bytes across every turn in one + conversation. No memories, no current time. + * ``dynamic_context`` carries the per-turn memory text and is + ``None`` when there are no memories. + + If ``base_prompt`` contains ``{memories}`` the caller used the + legacy single-string pattern; the stable half replaces the slot + with an empty string and the dynamic half still carries the memory + text so the LLM has the information available. + """ + default_prompt = ( + "You are a knowledgeable and helpful AI assistant. " + "You have access to conversation memories that help you provide more personalized responses. " + "Use the memories to understand the user's context, preferences, and past interactions. " + "If memories are provided, reference them naturally when relevant, but don't explicitly mention having memories." + ) + if base_prompt is None: + stable = default_prompt + elif "{memories}" in base_prompt: + stable = base_prompt.replace("{memories}", "").rstrip() + else: + stable = base_prompt + + memory_text = "" + if memories: + rendered: list[str] = [] + for i, memory in enumerate(memories, 1): + if isinstance(memory, TextualMemoryItem): + text_memory = memory.memory + else: + if not isinstance(memory, str): + logger.error("Unexpected memory type.") + text_memory = memory + rendered.append(f"{i}. {text_memory}") + memory_text = "\n".join(rendered) + if not memory_text: + return stable, None + return stable, "## Memories\n" + memory_text + def _str_memories( self, memories: list[TextualMemoryItem], mode: Literal["concise", "full"] = "full" ) -> str: diff --git a/src/memos/mem_os/main.py b/src/memos/mem_os/main.py index 0dc6ab209..6e68c2b38 100644 --- a/src/memos/mem_os/main.py +++ b/src/memos/mem_os/main.py @@ -293,19 +293,24 @@ def _generate_enhanced_response_with_context( search_memories = search_engine.search( original_query, top_k=self.config.top_k, mode="fast" ) - system_prompt = self._build_system_prompt( + # Issue #2524: emit a stable system prompt + per-turn dynamic + # context so provider prefix caches can reuse the chat history. + stable_system, dynamic_ctx = self._build_cacheable_system_parts( search_memories, base_prompt=base_prompt - ) # Use the same system prompt builder + ) else: - system_prompt = self._build_system_prompt(base_prompt=base_prompt) - current_messages = [ - {"role": "system", "content": system_prompt + SYNTHESIS_PROMPT.format(qa_text=qa_text)}, - *chat_history.chat_history, - { - "role": "user", - "content": original_query, - }, + stable_system, dynamic_ctx = self._build_cacheable_system_parts( + None, base_prompt=base_prompt + ) + + synthesis_tail = SYNTHESIS_PROMPT.format(qa_text=qa_text) + current_messages: list[dict] = [ + {"role": "system", "content": stable_system + synthesis_tail} ] + current_messages.extend(chat_history.chat_history) + if dynamic_ctx: + current_messages.append({"role": "system", "content": dynamic_ctx}) + current_messages.append({"role": "user", "content": original_query}) # Handle activation memory if enabled (same as core method) past_key_values = None diff --git a/src/memos/templates/cloud_service_prompt.py b/src/memos/templates/cloud_service_prompt.py index 15bc74a3f..33da7abdf 100644 --- a/src/memos/templates/cloud_service_prompt.py +++ b/src/memos/templates/cloud_service_prompt.py @@ -105,3 +105,104 @@ def get_cloud_chat_prompt(lang: str = "en") -> str: ) else: raise ValueError(f"Invalid language: {lang}") + + +# -- Prefix-cache-friendly helpers (Issue #2524) --------------------------- +# +# The original ``get_cloud_chat_prompt`` embeds ``{current_time}`` (minute +# precision) and splices the current turn's retrieval into ``{memories}``. +# Because the resulting string is sent as the first (system) message of every +# turn, provider prefix caches cannot match any shared prefix across turns. +# +# The helpers below return the two halves separately: +# +# * ``get_cloud_chat_stable_prompt(lang)`` -- byte-identical across turns in +# one conversation. Safe to use as the leading system message so that the +# chat history that follows becomes part of the shared prefix. +# * ``get_cloud_chat_dynamic_context(...)`` -- per-turn context carrying the +# current time, the retrieved memories, and the retrieved preferences. The +# caller injects this as a system message immediately before the latest +# user turn. +# --------------------------------------------------------------------------- + + +_STABLE_PROMPT_SENTINEL = ( + "A system message carrying the current time, retrieved memories, and " + "preferences will appear immediately before the latest user turn — " + "read it before answering." +) + +_STABLE_PROMPT_SENTINEL_ZH = "紧跟在最新用户消息之前,会有一条系统消息,包含当前时间、检索到的记忆和偏好——回答前请先阅读该消息。" + + +def _build_stable_from_cloud_prompt(prompt: str, sentinel: str) -> str: + """Return the cloud chat prompt with the per-turn placeholders replaced + by a short sentinel pointing to the dynamic context message.""" + # Drop the "# System Context" line that embeds {current_time}. + stable = prompt.replace( + "- Current Time: {current_time} (Baseline for freshness)", + f"- {sentinel}", + ).replace( + "- 当前时间: {current_time} (请以此作为判断记忆时效性的基准)", + f"- {sentinel}", + ) + # Replace the {memories} block with a note. + stable = stable.replace("\n{memories}\n", sentinel) + # Belt and suspenders: strip any stray placeholders. + stable = stable.replace("{current_time}", "").replace("{memories}", "") + return stable + + +def get_cloud_chat_stable_prompt(lang: str = "en") -> str: + """Return the stable (cross-turn constant) cloud chat prompt. + + The returned string must be identical across turns in one conversation. + No current time, no retrieved memories. The template points the model to + the dynamic context message that follows the chat history. + """ + if lang == "zh": + return _build_stable_from_cloud_prompt(CLOUD_CHAT_PROMPT_ZH, _STABLE_PROMPT_SENTINEL_ZH) + if lang == "en": + return _build_stable_from_cloud_prompt(CLOUD_CHAT_PROMPT_EN, _STABLE_PROMPT_SENTINEL) + raise ValueError(f"Invalid language: {lang}") + + +def get_cloud_chat_dynamic_context( + *, + memories_block: str, + pref_string: str | None = None, + now: datetime | None = None, + lang: str = "en", +) -> str: + """Return the per-turn dynamic context message. + + Args: + memories_block: already-formatted memories (e.g. "1. ...\\n2. ..."). + Pass an empty string when there are no memories. + pref_string: formatted preference string, or ``None``. + now: timestamp to use for the "Current Time" line. Defaults to + ``datetime.now()``; parameterised to help tests. + lang: "en" or "zh". + """ + stamp = (now or datetime.now()).strftime("%Y-%m-%d %H:%M (%A)") + + memories_section = memories_block.strip() + pref_section = (pref_string or "").strip() + + if lang == "zh": + parts = [f"# 当前上下文\n- 当前时间: {stamp}"] + if memories_section: + parts.append("\n# 事实与记忆\n\n" + memories_section + "\n") + if pref_section: + parts.append("\n# 偏好\n" + pref_section) + return "\n".join(parts) + + if lang == "en": + parts = [f"# Current Context\n- Current Time: {stamp}"] + if memories_section: + parts.append("\n# Facts & Memories\n\n" + memories_section + "\n") + if pref_section: + parts.append("\n# Preferences\n" + pref_section) + return "\n".join(parts) + + raise ValueError(f"Invalid language: {lang}") diff --git a/tests/api/test_chat_message_order.py b/tests/api/test_chat_message_order.py new file mode 100644 index 000000000..ad7b9416d --- /dev/null +++ b/tests/api/test_chat_message_order.py @@ -0,0 +1,280 @@ +"""Prefix-cache-friendly chat message assembly (Issue #2524). + +These tests pin down the invariants from the issue: +- The first message (stable system prompt) must stay byte-identical across turns + within one conversation. +- Chat history sits inside the shared prefix (between the stable system message + and the per-turn dynamic context). +- Per-turn retrieved memories, preferences, and the current time move into a + separate system message that is placed immediately before the final user + query so that the common prefix across requests can grow. +- A custom base_prompt that embeds `{memories}` keeps the legacy single-system + shape (backward compatibility). +""" + +from __future__ import annotations + +import re + +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +from memos.api.handlers.chat_handler import ChatHandler + + +if TYPE_CHECKING: + import pytest + + +def _make_handler(monkeypatch: pytest.MonkeyPatch) -> ChatHandler: + """Construct a ChatHandler without going through its real __init__. + + The real __init__ validates dependencies and wires sub-handlers; the tests + here only exercise the message-assembly helpers and inject fakes for + `chat_llms` / `search_handler` as needed. + """ + handler = ChatHandler.__new__(ChatHandler) + handler.logger = MagicMock() + handler.chat_llms = {} + handler.playground_chat_llms = {} + handler.search_handler = MagicMock() + handler.add_handler = MagicMock() + handler.online_bot = None + handler.enable_mem_scheduler = False + handler.dependencies = MagicMock() + return handler + + +def _memory(mem_id: str, text: str) -> dict: + return { + "id": mem_id, + "memory": text, + "metadata": {"memory_type": "PersonalMemory", "updated_at": "2026-01-01T00:00:00"}, + "relativity": 0.9, + } + + +class TestStableAndDynamicSplit: + """INV-1 / INV-3 / INV-5: the handler can produce a stable system prompt + plus a per-turn dynamic context string.""" + + def test_stable_system_prompt_omits_current_time_and_memories( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + memories = [_memory("m-1", "User likes dim sum on weekends")] + + prompt_turn_1 = handler._build_stable_system_prompt( + query="What should I eat?", base_prompt=None + ) + prompt_turn_2 = handler._build_stable_system_prompt( + query="How about tonight?", base_prompt=None + ) + + # INV-1: byte-identical across turns. + assert prompt_turn_1 == prompt_turn_2 + # Dynamic data must not leak into the stable prompt. + assert "User likes dim sum" not in prompt_turn_1 + # No current-minute timestamp substituted into the stable prompt. + assert not re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}", prompt_turn_1) + # The placeholder itself must be gone — otherwise the model sees "{current_time}". + assert "{current_time}" not in prompt_turn_1 + assert "{memories}" not in prompt_turn_1 + _ = memories # explicit: memories not used while building stable prompt + + def test_dynamic_context_contains_time_memories_and_preferences( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + memories = [ + _memory("m-1", "User lives in Shanghai"), + _memory("m-2", "User prefers short answers"), + ] + pref_string = "- answer in English" + + dynamic = handler._build_dynamic_context_message( + memories=memories, + pref_string=pref_string, + lang="en", + ) + + assert dynamic is not None + assert "User lives in Shanghai" in dynamic + assert "User prefers short answers" in dynamic + assert "- answer in English" in dynamic + # Current time appears here, not in the stable prompt. + assert re.search(r"\d{4}-\d{2}-\d{2}", dynamic) + + def test_dynamic_context_returns_none_when_nothing_dynamic( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + # No memories, no preferences — still, we may want to emit the current + # time or nothing; the invariant is just "if everything is empty, + # don't inject a message". + dynamic = handler._build_dynamic_context_message( + memories=[], + pref_string=None, + lang="en", + include_time=False, + ) + assert dynamic is None + + +class TestCacheableAssemblyShape: + """INV-2 / INV-3: assembled message list has stable[0], history, dynamic, user.""" + + def test_messages_place_history_before_dynamic_context( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + + history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + memories = [_memory("m-1", "User lives in Shanghai")] + + messages = handler._assemble_cacheable_messages( + query="Where should I eat tonight?", + history=history, + memories=memories, + pref_string="- short answers", + base_prompt=None, + lang="en", + ) + + # Stable system first. + assert messages[0]["role"] == "system" + assert "User lives in Shanghai" not in messages[0]["content"] + + # History sits inside the shared prefix. + assert messages[1] == history[0] + assert messages[2] == history[1] + + # Dynamic context is the system message right before the user query. + assert messages[-2]["role"] == "system" + assert "User lives in Shanghai" in messages[-2]["content"] + + # Final message is the user turn. + assert messages[-1] == {"role": "user", "content": "Where should I eat tonight?"} + + def test_shared_prefix_grows_across_turns(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Two consecutive turns must share stable + prior-history prefix.""" + handler = _make_handler(monkeypatch) + + turn1_history: list[dict] = [] + turn1 = handler._assemble_cacheable_messages( + query="Hi", + history=turn1_history, + memories=[_memory("m-1", "User lives in Shanghai")], + pref_string=None, + base_prompt=None, + lang="en", + ) + + # Second turn carries turn 1's exchange as history. + turn2_history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + turn2 = handler._assemble_cacheable_messages( + query="Where should I eat tonight?", + history=turn2_history, + memories=[_memory("m-2", "User likes dim sum")], + pref_string=None, + base_prompt=None, + lang="en", + ) + + # INV-1: stable system message is identical. + assert turn1[0] == turn2[0] + # INV-2: turn 1's exchange (positions 1..2 in turn 2) matches history. + assert turn2[1] == turn2_history[0] + assert turn2[2] == turn2_history[1] + # Longest shared prefix spans at least the stable system. + assert turn1[0]["content"] == turn2[0]["content"] + + def test_assemble_omits_dynamic_message_when_no_memories_or_prefs( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + messages = handler._assemble_cacheable_messages( + query="How are you?", + history=history, + memories=[], + pref_string=None, + base_prompt=None, + lang="en", + include_dynamic_time=False, + ) + + # Without dynamic payload: stable system + history + user. No extra turn. + assert [m["role"] for m in messages] == ["system", "user", "assistant", "user"] + assert messages[-1] == {"role": "user", "content": "How are you?"} + + +class TestLegacyBasePromptCompat: + """INV-4: a custom base_prompt that embeds `{memories}` keeps the single- + system shape (no reorder).""" + + def test_custom_base_prompt_with_memories_placeholder_keeps_legacy_shape( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + + history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + memories = [_memory("m-1", "User likes blue")] + base_prompt = "You are a tailor. User memories: {memories}" + + messages = handler._assemble_cacheable_messages( + query="Suggest a jacket colour", + history=history, + memories=memories, + pref_string=None, + base_prompt=base_prompt, + lang="en", + ) + + # Legacy shape: [system(stable+inline memories), *history, user] + assert len(messages) == 1 + len(history) + 1 + assert messages[0]["role"] == "system" + assert "User likes blue" in messages[0]["content"] + assert "{memories}" not in messages[0]["content"] + assert messages[1:-1] == history + assert messages[-1] == {"role": "user", "content": "Suggest a jacket colour"} + + +class TestEnhanceEndpointSplit: + """Playground path uses `_build_enhance_*`. Same invariants must hold.""" + + def test_enhance_stable_prompt_omits_time_and_memories( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + stable = handler._build_enhance_stable_prompt(lang="en", tone="friendly", verbosity="mid") + # No ISO date, no personal/outer memory text. + assert not re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}", stable) + assert "PersonalMemory (ordered)" not in stable or "## PersonalMemory (ordered)" in stable + # The {date} placeholder must be substituted away (either with empty or + # with a sentinel pointing to the context message). + assert "{date}" not in stable + + def test_enhance_dynamic_context_contains_time_and_memory_blocks( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + memories = [_memory("m-1", "User lives in Shanghai")] + dynamic = handler._build_enhance_dynamic_context( + memories_list=memories, pref_string="- short answers", lang="en" + ) + assert "User lives in Shanghai" in dynamic + assert "- short answers" in dynamic + assert re.search(r"\d{4}-\d{2}-\d{2}", dynamic) diff --git a/tests/mem_os/test_memos_core.py b/tests/mem_os/test_memos_core.py index b57b0b254..8b56c7027 100644 --- a/tests/mem_os/test_memos_core.py +++ b/tests/mem_os/test_memos_core.py @@ -636,7 +636,12 @@ def test_chat_with_custom_base_prompt( custom_prompt = "You are a pirate. Answer as such. User memories: {memories}" mos.chat("What do I like?", base_prompt=custom_prompt) - # Verify that the system prompt passed to the LLM is the custom one + # Verify that the system prompt passed to the LLM is the custom one. + # Issue #2524: with the cache-friendly split the stable system prompt + # is base_prompt with ``{memories}`` stripped, and the per-turn + # retrieved memories move into a separate system message immediately + # before the final user turn so the chat history that follows the + # stable prompt stays part of the provider's shared prefix. mock_llm.generate.assert_called_once() call_args = mock_llm.generate.call_args[0] messages = call_args[0] @@ -645,7 +650,12 @@ def test_chat_with_custom_base_prompt( assert "You are a pirate." in system_prompt assert "You are a knowledgeable and helpful AI assistant." not in system_prompt assert "User memories:" in system_prompt - assert "I like playing football" in system_prompt # Check if memory is interpolated + # Memories now live in the dynamic context message (second-to-last). + assert messages[-1]["role"] == "user" + assert messages[-2]["role"] == "system" + assert "I like playing football" in messages[-2]["content"] + # And they must NOT be inlined into the stable prefix anymore. + assert "I like playing football" not in system_prompt @patch("memos.mem_os.core.UserManager") @patch("memos.mem_os.core.MemReaderFactory") @@ -682,6 +692,60 @@ def test_chat_without_memories( # Verify response assert response == "This is a test response from the assistant." + @patch("memos.mem_os.core.UserManager") + @patch("memos.mem_os.core.MemReaderFactory") + @patch("memos.mem_os.core.LLMFactory") + def test_chat_places_history_before_dynamic_context_issue_2524( + self, + mock_llm_factory, + mock_reader_factory, + mock_user_manager_class, + mock_config, + mock_llm, + mock_mem_reader, + mock_user_manager, + mock_mem_cube, + ): + """Issue #2524: chat_history must sit between the stable system + message and the per-turn dynamic context so provider prefix caches + can reuse the history across turns.""" + mock_llm_factory.from_config.return_value = mock_llm + mock_reader_factory.from_config.return_value = mock_mem_reader + mock_user_manager_class.return_value = mock_user_manager + + mos = MOSCore(MOSConfig(**mock_config)) + mos.mem_cubes["test_cube_1"] = mock_mem_cube + mos.mem_cubes["test_cube_2"] = mock_mem_cube + + # Seed chat history with a prior exchange. + if mos.user_id not in mos.chat_history_manager: + mos._register_chat_history(mos.user_id) + chat_history = mos.chat_history_manager[mos.user_id] + chat_history.chat_history.extend( + [ + {"role": "user", "content": "Previous question"}, + {"role": "assistant", "content": "Previous answer"}, + ] + ) + + mos.chat("What should I eat?") + + call_args = mock_llm.generate.call_args[0] + messages = call_args[0] + + # Stable system first. + assert messages[0]["role"] == "system" + assert "I like playing football" not in messages[0]["content"] + + # Prior history inside the shared prefix. + assert messages[1] == {"role": "user", "content": "Previous question"} + assert messages[2] == {"role": "assistant", "content": "Previous answer"} + + # Dynamic context immediately before the final user turn. + assert messages[-1] == {"role": "user", "content": "What should I eat?"} + assert messages[-2]["role"] == "system" + assert "I like playing football" in messages[-2]["content"] + # TODO: test clear message From b147f5f7e3ec0675e8a4b12f5c876ad3b6d9b118 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Wed, 21 Oct 2026 18:57:23 +0800 Subject: [PATCH 2/3] fix: address OCR feedback on prefix-cache refactor (#2524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix eight issues from Open Code Review on the Issue #2524 patch: 1. `MOS._generate_synthesis_response` was still concatenating `synthesis_tail` (per-call sub-Q/A) into the leading system message, which broke the whole point of the stable prefix. Emit the synthesis tail as its own system message after the chat history and dynamic context, keeping `stable_system` byte-identical across turns. 2. `_build_stable_from_cloud_prompt` depended on exact-string `str.replace` calls against the cloud chat templates. Swap in regex-based substitution so small template rewording still works, and raise `ValueError` loudly if any per-turn placeholder would survive (instead of silently leaking the raw `{current_time}` / `{memories}` tokens to the model). 3. `get_cloud_chat_dynamic_context` appended sections that each started with `"\n"` and joined them with `"\n"`, producing double blank lines. Drop the leading `"\n"` from each section and switch the separator to `"\n\n"` so section boundaries render as one blank line. 4. `MOSCore._build_cacheable_system_parts` logged "Unexpected memory type." and then still appended the non-string object's `__repr__` as a memory entry. Skip the bad entry with `continue` so corrupted data cannot reach the LLM. 5. `_build_enhance_stable_prompt` was passing a long literal sentence ("see the context message immediately before…") as the `{date}` value, so the stable prompt rendered "Current Date: see the context message immediately before…". Use a short, neutral placeholder ("(see dynamic context)"); the live timestamp is still emitted by `_build_enhance_dynamic_context` in the per-turn message. 7. `_build_enhance_dynamic_context` had the same leading-`"\n"` + `"\n"`-join issue as the cloud dynamic context helper. Same fix: strip leading newlines from parts, join with `"\n\n"`. 8. The business-chat log of `current_system_prompt` was moved out of the `if chat_req.need_search` branch, so it fired on every request regardless of retrieval. Move it back inside the retrieval branch to stop flooding the logs for no-search requests. 10. `_build_dynamic_context_message` guarded the early `None` return with `not memories_text and not pref_text and not include_time`. Because `include_time` defaults to `True`, the guard never fired on the default call path — a timestamp-only system message was always injected even when there was nothing useful to say. Drop the `include_time` check so the function honours its `Optional` return type and only emits a dynamic context message when there is memory or preference content. Deliberately not changing OCR findings 6 and 9 (mid-conversation `{"role": "system"}` messages). OpenAI's Chat Completions API accepts multiple system messages at any position — a mid-conversation system message does not raise 400 as the OCR claims. The existing test suite also pins `messages[-2]["role"] == "system"` as the intended shape (see `tests/api/test_chat_message_order.py` and `tests/mem_os/test_memos_core.py`). Changing the role to "user" with a marker would blur the semantic boundary and require rewriting those tests, which exceeds the fix-only scope. Verified with `pytest tests/api/test_chat_message_order.py tests/mem_os/test_memos_core.py` — all 34 tests pass. Pre-existing env failures in `tests/memories/activation/test_kv.py`, `tests/parsers/test_markitdown.py`, `tests/mem_reader/*`, and `tests/vec_dbs/test_qdrant.py` are unrelated to this diff. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/memos/api/handlers/chat_handler.py | 36 +++++++++---- src/memos/mem_os/core.py | 5 +- src/memos/mem_os/main.py | 10 ++-- src/memos/templates/cloud_service_prompt.py | 58 +++++++++++++++------ 4 files changed, 77 insertions(+), 32 deletions(-) diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 4eb855d50..c9b75da57 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -860,10 +860,14 @@ def generate_chat_response() -> Generator[str, None, None]: pref_string=pref_string_for_prompt, base_prompt=chat_req.system_prompt, ) - system_prompt = current_messages[0]["content"] - self.logger.info( - f"[ChatBusinessHandler] current_system_prompt: {system_prompt}" - ) + if chat_req.need_search: + # Only log the full assembled system prompt when we + # actually ran retrieval; otherwise this fires on + # every no-search request and floods the logs. + system_prompt = current_messages[0]["content"] + self.logger.info( + f"[ChatBusinessHandler] current_system_prompt: {system_prompt}" + ) # Step 3: Generate streaming response from LLM if ( @@ -1186,7 +1190,12 @@ def _build_dynamic_context_message( """ memories_text = self._format_memories_for_dynamic_block(memories) pref_text = (pref_string or "").strip() - if not memories_text and not pref_text and not include_time: + # If there is no memory and no preference content, skip the dynamic + # context message entirely — emitting a timestamp-only system + # message defeats the ``Optional`` return type and injects noise + # with no payload. The ``include_time`` kwarg is kept so callers can + # still ask for the stamp when they do have memories/prefs to say. + if not memories_text and not pref_text: return None return get_cloud_chat_dynamic_context( memories_block=memories_text, @@ -1253,10 +1262,15 @@ def _build_enhance_stable_prompt( verbosity: str = "mid", ) -> str: """Return the stable (cross-turn constant) enhance-mode system prompt.""" - # Keep the ``{date}`` placeholder out of the stable text so this - # string is identical across turns. + # ``get_memos_prompt`` substitutes ``{date}`` into the body, so the + # literal string we pass is what the model will see on the "Current + # Date" line. Use a short, neutral placeholder that points the model + # at the per-turn dynamic context message rather than embedding the + # live timestamp (which would change every turn and break the + # provider prefix cache). The real date is emitted in the dynamic + # context from ``_build_enhance_dynamic_context``. sys_body = get_memos_prompt( - date="see the context message immediately before the latest user turn", + date="(see dynamic context)", tone=tone, verbosity=verbosity, mode="enhance", @@ -1284,14 +1298,14 @@ def _build_enhance_dynamic_context( ) parts = [ header_time, - "\n# Memories\n## PersonalMemory (ordered)\n" + "# Memories\n## PersonalMemory (ordered)\n" + mem_block_p + "\n## OuterMemory (from Internet Search, ordered)\n" + mem_block_o, ] if pref_block: - parts.append("\n" + pref_block) - return "\n".join(parts) + parts.append(pref_block) + return "\n\n".join(parts) def _format_mem_block( self, memories_all: list, max_items: int = 10, max_chars_each: int = 1600 diff --git a/src/memos/mem_os/core.py b/src/memos/mem_os/core.py index 83fdace0b..9f56c05c9 100644 --- a/src/memos/mem_os/core.py +++ b/src/memos/mem_os/core.py @@ -434,7 +434,10 @@ def _build_cacheable_system_parts( text_memory = memory.memory else: if not isinstance(memory, str): - logger.error("Unexpected memory type.") + # Skip rather than forward a repr(dict/object) to the + # LLM as a memory entry. + logger.error("Unexpected memory type: %s", type(memory)) + continue text_memory = memory rendered.append(f"{i}. {text_memory}") memory_text = "\n".join(rendered) diff --git a/src/memos/mem_os/main.py b/src/memos/mem_os/main.py index 6e68c2b38..3253db360 100644 --- a/src/memos/mem_os/main.py +++ b/src/memos/mem_os/main.py @@ -304,12 +304,16 @@ def _generate_enhanced_response_with_context( ) synthesis_tail = SYNTHESIS_PROMPT.format(qa_text=qa_text) - current_messages: list[dict] = [ - {"role": "system", "content": stable_system + synthesis_tail} - ] + # Issue #2524: keep the leading system message byte-identical across + # turns. ``synthesis_tail`` embeds the per-call sub-questions and + # sub-answers so it changes every turn — append it as its own system + # message *after* the chat history (together with the dynamic + # context) instead of concatenating it into ``stable_system``. + current_messages: list[dict] = [{"role": "system", "content": stable_system}] current_messages.extend(chat_history.chat_history) if dynamic_ctx: current_messages.append({"role": "system", "content": dynamic_ctx}) + current_messages.append({"role": "system", "content": synthesis_tail}) current_messages.append({"role": "user", "content": original_query}) # Handle activation memory if enabled (same as core method) diff --git a/src/memos/templates/cloud_service_prompt.py b/src/memos/templates/cloud_service_prompt.py index 33da7abdf..1c47c640a 100644 --- a/src/memos/templates/cloud_service_prompt.py +++ b/src/memos/templates/cloud_service_prompt.py @@ -1,3 +1,5 @@ +import re + from datetime import datetime @@ -137,19 +139,41 @@ def get_cloud_chat_prompt(lang: str = "en") -> str: def _build_stable_from_cloud_prompt(prompt: str, sentinel: str) -> str: """Return the cloud chat prompt with the per-turn placeholders replaced - by a short sentinel pointing to the dynamic context message.""" - # Drop the "# System Context" line that embeds {current_time}. - stable = prompt.replace( - "- Current Time: {current_time} (Baseline for freshness)", - f"- {sentinel}", - ).replace( - "- 当前时间: {current_time} (请以此作为判断记忆时效性的基准)", + by a short sentinel pointing to the dynamic context message. + + The template surgery is implemented with regular expressions rather than + exact-string ``str.replace`` calls so that minor rewording of the + surrounding template (extra whitespace, punctuation tweaks, new wording + around ``{current_time}`` / ``{memories}``) still produces a correct + stable prompt. If a placeholder would survive the surgery we raise + ``ValueError`` loudly instead of leaking the raw template token to the + model. + """ + # Replace the "- Current Time: {current_time} ..." line (English or + # Chinese) with the sentinel. Match the whole line so parenthetical + # qualifiers after the placeholder are also removed. + stable = re.sub( + r"- *(?:Current Time|当前时间)[::][^\n]*\{current_time\}[^\n]*", f"- {sentinel}", + prompt, + ) + # Replace the ...{memories}... block (any + # whitespace between the tags) with the sentinel on its own line. + stable = re.sub( + r"\s*\{memories\}\s*", + sentinel, + stable, + flags=re.DOTALL, ) - # Replace the {memories} block with a note. - stable = stable.replace("\n{memories}\n", sentinel) - # Belt and suspenders: strip any stray placeholders. - stable = stable.replace("{current_time}", "").replace("{memories}", "") + # Loud failure beats silent leak: if a placeholder survived, the template + # has drifted and the derivation needs updating. + for leftover in ("{current_time}", "{memories}"): + if leftover in stable: + raise ValueError( + "Cloud chat template drifted: placeholder " + f"{leftover!r} was not substituted when building the stable " + "system prompt. Update _build_stable_from_cloud_prompt." + ) return stable @@ -192,17 +216,17 @@ def get_cloud_chat_dynamic_context( if lang == "zh": parts = [f"# 当前上下文\n- 当前时间: {stamp}"] if memories_section: - parts.append("\n# 事实与记忆\n\n" + memories_section + "\n") + parts.append("# 事实与记忆\n\n" + memories_section + "\n") if pref_section: - parts.append("\n# 偏好\n" + pref_section) - return "\n".join(parts) + parts.append("# 偏好\n" + pref_section) + return "\n\n".join(parts) if lang == "en": parts = [f"# Current Context\n- Current Time: {stamp}"] if memories_section: - parts.append("\n# Facts & Memories\n\n" + memories_section + "\n") + parts.append("# Facts & Memories\n\n" + memories_section + "\n") if pref_section: - parts.append("\n# Preferences\n" + pref_section) - return "\n".join(parts) + parts.append("# Preferences\n" + pref_section) + return "\n\n".join(parts) raise ValueError(f"Invalid language: {lang}") From 6c654fc9a1bee1d2eb827e7f098848101f8d7982 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Wed, 21 Oct 2026 19:13:42 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20round=202=20OCR=20follow-up=20?= =?UTF-8?q?=E2=80=94=20portable=20role=20for=20per-turn=20context=20(#2524?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the second automatic OCR pass on PR #2529: * Finding #6 (chat_handler.py L623-L628 + L1253) and finding #9 (mem_chat/simple.py L95) — a mid-conversation ``{"role": "system"}`` message is ambiguous for several provider APIs (Azure OpenAI, some Anthropic routers, several OpenAI-compatible proxies) and can be silently merged or rejected. Round 1 kept the system role on the argument that OpenAI's spec permits it; OCR re-flagged the pattern as not converging. Switch the per-turn dynamic context to a ``{"role": "user"}`` message with an explicit "[Context for this turn]" header so the model still distinguishes it from the actual user question. Also applied to the matching patterns in ``_assemble_cacheable_messages`` and ``mos_core.chat`` for consistency (the latter is directly pinned by the test suite). * Finding #5 (chat_handler.py L1272-L1278) — round 1 passed ``date="(see dynamic context)"`` into ``get_memos_prompt``, which still renders the literal line ``# Current Date: (see dynamic context)``. Switch to ``"N/A — see context below"``, a clearly synthetic sentinel that is unlikely to be parsed as a real date. Tests updated: * tests/api/test_chat_message_order.py — assert ``user`` role + "[Context for this turn]" marker. * tests/mem_os/test_memos_core.py — same, in both the custom base-prompt case and the chat-history-order case. Verification: ``pytest tests/api tests/mem_chat tests/mem_os/test_memos_core.py`` (all pass, 164 tests). ``ruff check`` / ``ruff format --check`` clean. Co-Authored-By: Claude Opus 4.7 --- src/memos/api/handlers/chat_handler.py | 38 ++++++++++++++++++++------ src/memos/mem_chat/simple.py | 13 +++++++-- src/memos/mem_os/core.py | 11 +++++++- tests/api/test_chat_message_order.py | 7 +++-- tests/mem_os/test_memos_core.py | 14 +++++++--- 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index c9b75da57..acd308d11 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -620,10 +620,22 @@ def generate_chat_response() -> Generator[str, None, None]: ) history_info = chat_req.history[-20:] if chat_req.history else [] + # Issue #2524 OCR round 2 (finding #6): deliver the per-turn + # dynamic context as a user-role framing message. A + # mid-conversation ``{"role": "system"}`` message is + # ambiguous for several provider APIs and can trigger + # 400s (Azure OpenAI, some Anthropic routers, several + # OpenAI-compatible proxies). Wrapping the context in a + # user turn with an explicit "[Context for this turn]" + # header keeps the shape unambiguous while preserving + # the stable prefix (system + history) for caching. current_messages = [ {"role": "system", "content": stable_prompt}, *history_info, - {"role": "system", "content": dynamic_ctx}, + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + }, {"role": "user", "content": chat_req.query}, ] # Downstream code still expects a ``system_prompt`` symbol @@ -1250,7 +1262,16 @@ def _assemble_cacheable_messages( messages: list[dict] = [{"role": "system", "content": stable_system}] messages.extend(history_list) if dynamic_ctx: - messages.append({"role": "system", "content": dynamic_ctx}) + # Issue #2524 OCR round 2 (finding #6): keep the per-turn + # context in a user-role framing message rather than a + # mid-conversation system message. See the playground path above + # for the full rationale. + messages.append( + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + } + ) messages.append({"role": "user", "content": query}) return messages @@ -1264,13 +1285,14 @@ def _build_enhance_stable_prompt( """Return the stable (cross-turn constant) enhance-mode system prompt.""" # ``get_memos_prompt`` substitutes ``{date}`` into the body, so the # literal string we pass is what the model will see on the "Current - # Date" line. Use a short, neutral placeholder that points the model - # at the per-turn dynamic context message rather than embedding the - # live timestamp (which would change every turn and break the - # provider prefix cache). The real date is emitted in the dynamic - # context from ``_build_enhance_dynamic_context``. + # Date" line. Use a clearly synthetic sentinel so the model does not + # mistake the placeholder for a real date value (OCR round 2 + # finding #5 — "(see dynamic context)" still rendered as a literal + # "# Current Date: (see dynamic context)" line). The real date is + # emitted in the dynamic context from + # ``_build_enhance_dynamic_context`` and takes precedence. sys_body = get_memos_prompt( - date="(see dynamic context)", + date="N/A — see context below", tone=tone, verbosity=verbosity, mode="enhance", diff --git a/src/memos/mem_chat/simple.py b/src/memos/mem_chat/simple.py index d02070679..2e7e5741f 100644 --- a/src/memos/mem_chat/simple.py +++ b/src/memos/mem_chat/simple.py @@ -88,11 +88,20 @@ def run(self) -> None: dynamic_ctx = None # Issue #2524: stable system first, then history, then dynamic # context, then current user turn — keeps the chat history inside - # the provider's shared prefix. + # the provider's shared prefix. The dynamic context is delivered + # as a user-role framing message (OCR round 2 finding #9): a + # mid-conversation ``{"role": "system"}`` message is rejected or + # mishandled by several provider APIs, while a user turn with a + # "[Context for this turn]" header is unambiguous and portable. current_messages: list[dict] = [{"role": "system", "content": stable_system}] current_messages.extend(messages) if dynamic_ctx: - current_messages.append({"role": "system", "content": dynamic_ctx}) + current_messages.append( + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + } + ) current_messages.append({"role": "user", "content": user_input}) if self.config.enable_activation_memory: diff --git a/src/memos/mem_os/core.py b/src/memos/mem_os/core.py index 9f56c05c9..54fd261a6 100644 --- a/src/memos/mem_os/core.py +++ b/src/memos/mem_os/core.py @@ -313,7 +313,16 @@ def chat(self, query: str, user_id: str | None = None, base_prompt: str | None = current_messages: list[dict] = [{"role": "system", "content": stable_system}] current_messages.extend(chat_history.chat_history) if dynamic_ctx: - current_messages.append({"role": "system", "content": dynamic_ctx}) + # Issue #2524 OCR round 2: avoid a mid-conversation + # ``{"role": "system"}`` message, which is rejected or + # mishandled by several provider APIs. Wrap the per-turn + # context in a user-role framing message instead. + current_messages.append( + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + } + ) current_messages.append({"role": "user", "content": query}) past_key_values = None diff --git a/tests/api/test_chat_message_order.py b/tests/api/test_chat_message_order.py index ad7b9416d..9fc8a5a31 100644 --- a/tests/api/test_chat_message_order.py +++ b/tests/api/test_chat_message_order.py @@ -152,8 +152,11 @@ def test_messages_place_history_before_dynamic_context( assert messages[1] == history[0] assert messages[2] == history[1] - # Dynamic context is the system message right before the user query. - assert messages[-2]["role"] == "system" + # Dynamic context is a user-role framing message right before the + # user query (OCR round 2 feedback: mid-conversation ``system`` + # messages are not portable across provider APIs). + assert messages[-2]["role"] == "user" + assert messages[-2]["content"].startswith("[Context for this turn]\n") assert "User lives in Shanghai" in messages[-2]["content"] # Final message is the user turn. diff --git a/tests/mem_os/test_memos_core.py b/tests/mem_os/test_memos_core.py index 8b56c7027..09f6390be 100644 --- a/tests/mem_os/test_memos_core.py +++ b/tests/mem_os/test_memos_core.py @@ -650,9 +650,12 @@ def test_chat_with_custom_base_prompt( assert "You are a pirate." in system_prompt assert "You are a knowledgeable and helpful AI assistant." not in system_prompt assert "User memories:" in system_prompt - # Memories now live in the dynamic context message (second-to-last). + # Memories now live in the dynamic context message (second-to-last), + # delivered as a user-role framing turn (OCR round 2 feedback on PR + # #2529: mid-conversation ``{"role": "system"}`` is not portable). assert messages[-1]["role"] == "user" - assert messages[-2]["role"] == "system" + assert messages[-2]["role"] == "user" + assert messages[-2]["content"].startswith("[Context for this turn]\n") assert "I like playing football" in messages[-2]["content"] # And they must NOT be inlined into the stable prefix anymore. assert "I like playing football" not in system_prompt @@ -741,9 +744,12 @@ def test_chat_places_history_before_dynamic_context_issue_2524( assert messages[1] == {"role": "user", "content": "Previous question"} assert messages[2] == {"role": "assistant", "content": "Previous answer"} - # Dynamic context immediately before the final user turn. + # Dynamic context immediately before the final user turn, delivered + # as a user-role framing message (OCR round 2 feedback: avoid + # mid-conversation ``{"role": "system"}``). assert messages[-1] == {"role": "user", "content": "What should I eat?"} - assert messages[-2]["role"] == "system" + assert messages[-2]["role"] == "user" + assert messages[-2]["content"].startswith("[Context for this turn]\n") assert "I like playing football" in messages[-2]["content"]