diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 31acec6d0..acd308d11 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -40,7 +40,11 @@ ANSWER_TASK_LABEL, QUERY_TASK_LABEL, ) -from memos.templates.cloud_service_prompt import get_cloud_chat_prompt +from memos.templates.cloud_service_prompt import ( + get_cloud_chat_dynamic_context, + get_cloud_chat_prompt, + get_cloud_chat_stable_prompt, +) from memos.templates.mos_prompts import ( FURTHER_SUGGESTION_PROMPT, get_memos_prompt, @@ -158,22 +162,21 @@ def handle_chat_complete(self, chat_req: APIChatCompleteRequest) -> dict[str, An memories_list, chat_req.threshold or 0.5 ) - # Step 2: Build system prompt - system_prompt = self._build_system_prompt( + # Step 2: Build cache-friendly message list (Issue #2524): + # stable system prompt first, then chat history, then a dynamic + # context message carrying the current time / retrieved memories + # / preferences, then the current user query. The reorder keeps + # [stable_system, ...prior_history] identical across turns so + # provider prefix caches can reuse the history. + history_info = chat_req.history[-20:] if chat_req.history else [] + current_messages = self._assemble_cacheable_messages( query=chat_req.query, + history=history_info, memories=filtered_memories, pref_string=search_response.data.get("pref_string", ""), base_prompt=chat_req.system_prompt, ) - # Prepare message history - history_info = chat_req.history[-20:] if chat_req.history else [] - current_messages = [ - {"role": "system", "content": system_prompt}, - *history_info, - {"role": "user", "content": chat_req.query}, - ] - self.logger.info("[Cloud Service] Starting to generate chat complete response...") # Step 3: Generate complete response from LLM @@ -306,21 +309,18 @@ def generate_chat_response() -> Generator[str, None, None]: # Filter memories by threshold filtered_memories = self._filter_memories_by_threshold(memories_list) - # Step 2: Build system prompt with memories - system_prompt = self._build_system_prompt( + # Step 2: Build cache-friendly message list (Issue #2524) + history_info = chat_req.history[-20:] if chat_req.history else [] + current_messages = self._assemble_cacheable_messages( query=chat_req.query, + history=history_info, memories=filtered_memories, pref_string=search_response.data.get("pref_string", ""), base_prompt=chat_req.system_prompt, ) - - # Prepare messages - history_info = chat_req.history[-20:] if chat_req.history else [] - current_messages = [ - {"role": "system", "content": system_prompt}, - *history_info, - {"role": "user", "content": chat_req.query}, - ] + # Backward-compatible: expose the stable prompt in the log + # (callers used to log ``system_prompt`` here). + system_prompt = current_messages[0]["content"] self.logger.info( f"[Cloud Service] chat stream user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}, " @@ -599,7 +599,11 @@ def generate_chat_response() -> Generator[str, None, None]: ) yield f"data: {json.dumps({'type': 'reference', 'data': reference}, ensure_ascii=False)}\n\n" - # Step 2: Build system prompt with memories + # Step 2: Build cache-friendly message list (Issue #2524). + # Playground uses enhance-mode prompts: the stable prompt + # carries the fixed persona / instructions, and the dynamic + # context carries the current time + memories + preferences + # for this turn. lang = detect_lang(chat_req.query) if pref_string: pref_string += ( @@ -607,17 +611,36 @@ def generate_chat_response() -> Generator[str, None, None]: if lang == "zh" else "\n#warning\n- In thinking content, do not appear the reference number and id [1,2,3]etc. otherwise it will cause reference error." ) - system_prompt = self._build_enhance_system_prompt( - filtered_memories, pref_string, lang=lang + + stable_prompt = self._build_enhance_stable_prompt(lang=lang) + dynamic_ctx = self._build_enhance_dynamic_context( + memories_list=filtered_memories, + pref_string=pref_string, + lang=lang, ) - # Prepare messages history_info = chat_req.history[-20:] if chat_req.history else [] + # Issue #2524 OCR round 2 (finding #6): deliver the per-turn + # dynamic context as a user-role framing message. A + # mid-conversation ``{"role": "system"}`` message is + # ambiguous for several provider APIs and can trigger + # 400s (Azure OpenAI, some Anthropic routers, several + # OpenAI-compatible proxies). Wrapping the context in a + # user turn with an explicit "[Context for this turn]" + # header keeps the shape unambiguous while preserving + # the stable prefix (system + history) for caching. current_messages = [ - {"role": "system", "content": system_prompt}, + {"role": "system", "content": stable_prompt}, *history_info, + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + }, {"role": "user", "content": chat_req.query}, ] + # Downstream code still expects a ``system_prompt`` symbol + # for logging / speed-improvement estimation. + system_prompt = stable_prompt + "\n\n" + dynamic_ctx self.logger.info( f"[PLAYGROUND CHAT] user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}, " @@ -832,33 +855,31 @@ def generate_chat_response() -> Generator[str, None, None]: # Filter memories by threshold filtered_memories = self._filter_memories_by_threshold(memories_list) - # Step 2: Build system prompt with memories - system_prompt = self._build_system_prompt( - query=chat_req.query, - memories=filtered_memories, - pref_string=search_response.data.get("pref_string", ""), - base_prompt=chat_req.system_prompt, - ) - + pref_string_for_prompt = search_response.data.get("pref_string", "") self.logger.info( - f"[ChatBusinessHandler] chat stream user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}, " - f"current_system_prompt: {system_prompt}" + f"[ChatBusinessHandler] chat stream user_id: {chat_req.user_id}, readable_cube_ids: {readable_cube_ids}" ) else: - system_prompt = self._build_system_prompt( - query=chat_req.query, - memories=None, - pref_string=None, - base_prompt=chat_req.system_prompt, - ) + filtered_memories = None + pref_string_for_prompt = None - # Prepare messages + # Build cache-friendly message list (Issue #2524) history_info = chat_req.history[-20:] if chat_req.history else [] - current_messages = [ - {"role": "system", "content": system_prompt}, - *history_info, - {"role": "user", "content": chat_req.query}, - ] + current_messages = self._assemble_cacheable_messages( + query=chat_req.query, + history=history_info, + memories=filtered_memories, + pref_string=pref_string_for_prompt, + base_prompt=chat_req.system_prompt, + ) + if chat_req.need_search: + # Only log the full assembled system prompt when we + # actually ran retrieval; otherwise this fires on + # every no-search request and floods the logs. + system_prompt = current_messages[0]["content"] + self.logger.info( + f"[ChatBusinessHandler] current_system_prompt: {system_prompt}" + ) # Step 3: Generate streaming response from LLM if ( @@ -1104,6 +1125,210 @@ def _build_enhance_system_prompt( + f"\n\n{pref_string}" ) + # ------------------------------------------------------------------ + # Prefix-cache-friendly message assembly (Issue #2524) + # ------------------------------------------------------------------ + # + # The handlers above used to send the per-turn retrieval and current + # time as the first (system) message, with chat history after it. + # Because the first message changed every turn, provider prefix caches + # could not reuse the chat history that followed. + # + # The helpers below split the single system blob into: + # * a stable system prompt (constant across turns in one conversation), + # emitted as ``messages[0]`` so that the history that follows is part + # of the shared prefix, and + # * a per-turn dynamic context message that carries the current time, + # retrieved memories, and preferences, inserted immediately before + # the final user turn. + # + # Call sites use ``_assemble_cacheable_messages`` to produce the full + # ``messages`` list. Legacy callers that pass a ``base_prompt`` with + # ``{memories}`` keep the single-system shape for backward compatibility. + + def _format_memories_for_dynamic_block(self, memories: list | None) -> str: + """Return the memory list rendered as "1. ...\\n2. ..." (empty when + there are no memories). Mirrors ``_build_system_prompt`` so legacy + and cache-friendly paths produce the same memory text.""" + if not memories: + return "" + rendered: list[str] = [] + for i, memory in enumerate(memories, 1): + text_memory = memory.get("memory", "") if isinstance(memory, dict) else str(memory) + rendered.append(f"{i}. {text_memory}") + return "\n".join(rendered) + + def _build_stable_system_prompt( + self, + *, + query: str, + base_prompt: str | None, + ) -> str: + """Return the stable (cross-turn constant) system prompt. + + * If ``base_prompt`` is ``None``: return the stable cloud chat + prompt for the detected language (no ``{current_time}``, no + ``{memories}`` substitution). + * If ``base_prompt`` contains ``{memories}``: this is the legacy + single-string path; the caller is expected to use the legacy + assembly instead (``_assemble_cacheable_messages`` detects this + and falls back automatically). The method still returns the + base_prompt text with ``{memories}`` stripped so it is safe to + emit as a stable prefix. + * Otherwise return ``base_prompt`` as-is. + """ + if base_prompt is None: + lang = detect_lang(query) + return get_cloud_chat_stable_prompt(lang=lang) + if "{memories}" in base_prompt: + # Caller used the legacy single-string pattern. The stable half + # is the template with the memory slot removed. + return base_prompt.replace("{memories}", "").rstrip() + return base_prompt + + def _build_dynamic_context_message( + self, + *, + memories: list | None, + pref_string: str | None, + lang: str, + include_time: bool = True, + ) -> str | None: + """Return the per-turn dynamic context, or ``None`` when there is + nothing dynamic to emit. + + The returned string is intended to be sent as a system message + placed immediately before the final user turn. + """ + memories_text = self._format_memories_for_dynamic_block(memories) + pref_text = (pref_string or "").strip() + # If there is no memory and no preference content, skip the dynamic + # context message entirely — emitting a timestamp-only system + # message defeats the ``Optional`` return type and injects noise + # with no payload. The ``include_time`` kwarg is kept so callers can + # still ask for the stamp when they do have memories/prefs to say. + if not memories_text and not pref_text: + return None + return get_cloud_chat_dynamic_context( + memories_block=memories_text, + pref_string=pref_text or None, + lang=lang, + ) + + def _assemble_cacheable_messages( + self, + *, + query: str, + history: list[dict] | None, + memories: list | None, + pref_string: str | None, + base_prompt: str | None, + lang: str | None = None, + include_dynamic_time: bool = True, + ) -> list[dict]: + """Build the provider ``messages`` list in prefix-cache-friendly order. + + Shape (default path): + [stable system, *history, dynamic context (optional), user] + + Legacy path — triggered when ``base_prompt`` contains ``{memories}``: + [legacy system(stable + inline memories), *history, user] + """ + history_list = list(history or []) + + # Legacy backward-compatible path for custom ``base_prompt``. + if base_prompt and "{memories}" in base_prompt: + legacy_system = self._build_system_prompt( + query=query, + memories=memories, + pref_string=pref_string, + base_prompt=base_prompt, + ) + return [ + {"role": "system", "content": legacy_system}, + *history_list, + {"role": "user", "content": query}, + ] + + stable_system = self._build_stable_system_prompt(query=query, base_prompt=base_prompt) + resolved_lang = lang or detect_lang(query) + dynamic_ctx = self._build_dynamic_context_message( + memories=memories, + pref_string=pref_string, + lang=resolved_lang, + include_time=include_dynamic_time, + ) + + messages: list[dict] = [{"role": "system", "content": stable_system}] + messages.extend(history_list) + if dynamic_ctx: + # Issue #2524 OCR round 2 (finding #6): keep the per-turn + # context in a user-role framing message rather than a + # mid-conversation system message. See the playground path above + # for the full rationale. + messages.append( + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + } + ) + messages.append({"role": "user", "content": query}) + return messages + + def _build_enhance_stable_prompt( + self, + *, + lang: str = "en", + tone: str = "friendly", + verbosity: str = "mid", + ) -> str: + """Return the stable (cross-turn constant) enhance-mode system prompt.""" + # ``get_memos_prompt`` substitutes ``{date}`` into the body, so the + # literal string we pass is what the model will see on the "Current + # Date" line. Use a clearly synthetic sentinel so the model does not + # mistake the placeholder for a real date value (OCR round 2 + # finding #5 — "(see dynamic context)" still rendered as a literal + # "# Current Date: (see dynamic context)" line). The real date is + # emitted in the dynamic context from + # ``_build_enhance_dynamic_context`` and takes precedence. + sys_body = get_memos_prompt( + date="N/A — see context below", + tone=tone, + verbosity=verbosity, + mode="enhance", + lang=lang, + ) + return sys_body + + def _build_enhance_dynamic_context( + self, + *, + memories_list: list, + pref_string: str = "", + lang: str = "en", + ) -> str: + """Return the per-turn dynamic context for the enhance (playground) + chat endpoint (current time + PersonalMemory + OuterMemory + prefs).""" + now = datetime.now() + formatted_date = now.strftime("%Y-%m-%d %H:%M (%A)") + mem_block_o, mem_block_p = self._format_mem_block(memories_list) + pref_block = (pref_string or "").strip() + header_time = ( + f"# 当前上下文\n- 当前时间: {formatted_date}" + if lang == "zh" + else f"# Current Context\n- Current Time: {formatted_date}" + ) + parts = [ + header_time, + "# Memories\n## PersonalMemory (ordered)\n" + + mem_block_p + + "\n## OuterMemory (from Internet Search, ordered)\n" + + mem_block_o, + ] + if pref_block: + parts.append(pref_block) + return "\n\n".join(parts) + def _format_mem_block( self, memories_all: list, max_items: int = 10, max_chars_each: int = 1600 ) -> tuple[str, str]: diff --git a/src/memos/mem_chat/simple.py b/src/memos/mem_chat/simple.py index 765ad1a09..2e7e5741f 100644 --- a/src/memos/mem_chat/simple.py +++ b/src/memos/mem_chat/simple.py @@ -81,14 +81,28 @@ def run(self) -> None: print( f"🧠 [Memory] Searched memories:\n{self._str_memories(memories, mode='concise')}\n" ) - system_prompt = self._build_system_prompt(memories) + stable_system = self._build_stable_system_prompt() + dynamic_ctx = self._build_dynamic_memory_context(memories) else: - system_prompt = self._build_system_prompt() - current_messages = [ - {"role": "system", "content": system_prompt}, - *messages, - {"role": "user", "content": user_input}, - ] + stable_system = self._build_stable_system_prompt() + dynamic_ctx = None + # Issue #2524: stable system first, then history, then dynamic + # context, then current user turn — keeps the chat history inside + # the provider's shared prefix. The dynamic context is delivered + # as a user-role framing message (OCR round 2 finding #9): a + # mid-conversation ``{"role": "system"}`` message is rejected or + # mishandled by several provider APIs, while a user turn with a + # "[Context for this turn]" header is unambiguous and portable. + current_messages: list[dict] = [{"role": "system", "content": stable_system}] + current_messages.extend(messages) + if dynamic_ctx: + current_messages.append( + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + } + ) + current_messages.append({"role": "user", "content": user_input}) if self.config.enable_activation_memory: past_key_values = None @@ -154,6 +168,30 @@ def _build_system_prompt(self, memories: list | None = None) -> str: return base_prompt + # Prefix-cache-friendly split (Issue #2524) + def _build_stable_system_prompt(self) -> str: + """Return the stable (cross-turn constant) system prompt.""" + return ( + "You are a knowledgeable and helpful AI assistant. " + "You have access to conversation memories that help you provide more personalized responses. " + "Use the memories to understand the user's context, preferences, and past interactions. " + "If memories are provided, reference them naturally when relevant, but don't explicitly mention having memories." + ) + + def _build_dynamic_memory_context(self, memories: list | None) -> str | None: + """Return a system message carrying this turn's retrieved memories. + + Returned string is intended to sit immediately before the latest + user turn so the chat history that precedes it stays part of the + provider's shared prefix. + """ + if not memories: + return None + lines = ["## Memories"] + for i, memory in enumerate(memories, 1): + lines.append(f"{i}. ({memory.metadata.memory_time}) {memory.memory}") + return "\n".join(lines) + def _str_memories( self, memories: list[TextualMemoryItem], mode: Literal["concise", "full"] = "full" ) -> str: diff --git a/src/memos/mem_os/core.py b/src/memos/mem_os/core.py index 3ede965d3..54fd261a6 100644 --- a/src/memos/mem_os/core.py +++ b/src/memos/mem_os/core.py @@ -300,14 +300,30 @@ def chat(self, query: str, user_id: str | None = None, base_prompt: str | None = ) memories_all.extend(memories) logger.info(f"🧠 [Memory] Searched memories:\n{self._str_memories(memories_all)}\n") - system_prompt = self._build_system_prompt(memories_all, base_prompt=base_prompt) + # Issue #2524: emit a stable system prompt first, then history, + # then a dynamic context message with the retrieved memories, + # so provider prefix caches can reuse the history across turns. + stable_system, dynamic_ctx = self._build_cacheable_system_parts( + memories_all, base_prompt=base_prompt + ) else: - system_prompt = self._build_system_prompt(base_prompt=base_prompt) - current_messages = [ - {"role": "system", "content": system_prompt}, - *chat_history.chat_history, - {"role": "user", "content": query}, - ] + stable_system, dynamic_ctx = self._build_cacheable_system_parts( + None, base_prompt=base_prompt + ) + current_messages: list[dict] = [{"role": "system", "content": stable_system}] + current_messages.extend(chat_history.chat_history) + if dynamic_ctx: + # Issue #2524 OCR round 2: avoid a mid-conversation + # ``{"role": "system"}`` message, which is rejected or + # mishandled by several provider APIs. Wrap the per-turn + # context in a user-role framing message instead. + current_messages.append( + { + "role": "user", + "content": "[Context for this turn]\n" + dynamic_ctx, + } + ) + current_messages.append({"role": "user", "content": query}) past_key_values = None if self.config.enable_activation_memory: @@ -387,6 +403,57 @@ def _build_system_prompt( return base_prompt + memory_context_with_header return base_prompt + # Prefix-cache-friendly system split (Issue #2524) + def _build_cacheable_system_parts( + self, + memories: list[TextualMemoryItem] | list[str] | None = None, + base_prompt: str | None = None, + ) -> tuple[str, str | None]: + """Return ``(stable_system, dynamic_context)`` for cache-friendly + chat assembly. + + * ``stable_system`` is the same bytes across every turn in one + conversation. No memories, no current time. + * ``dynamic_context`` carries the per-turn memory text and is + ``None`` when there are no memories. + + If ``base_prompt`` contains ``{memories}`` the caller used the + legacy single-string pattern; the stable half replaces the slot + with an empty string and the dynamic half still carries the memory + text so the LLM has the information available. + """ + default_prompt = ( + "You are a knowledgeable and helpful AI assistant. " + "You have access to conversation memories that help you provide more personalized responses. " + "Use the memories to understand the user's context, preferences, and past interactions. " + "If memories are provided, reference them naturally when relevant, but don't explicitly mention having memories." + ) + if base_prompt is None: + stable = default_prompt + elif "{memories}" in base_prompt: + stable = base_prompt.replace("{memories}", "").rstrip() + else: + stable = base_prompt + + memory_text = "" + if memories: + rendered: list[str] = [] + for i, memory in enumerate(memories, 1): + if isinstance(memory, TextualMemoryItem): + text_memory = memory.memory + else: + if not isinstance(memory, str): + # Skip rather than forward a repr(dict/object) to the + # LLM as a memory entry. + logger.error("Unexpected memory type: %s", type(memory)) + continue + text_memory = memory + rendered.append(f"{i}. {text_memory}") + memory_text = "\n".join(rendered) + if not memory_text: + return stable, None + return stable, "## Memories\n" + memory_text + def _str_memories( self, memories: list[TextualMemoryItem], mode: Literal["concise", "full"] = "full" ) -> str: diff --git a/src/memos/mem_os/main.py b/src/memos/mem_os/main.py index 0dc6ab209..031148797 100644 --- a/src/memos/mem_os/main.py +++ b/src/memos/mem_os/main.py @@ -293,17 +293,26 @@ def _generate_enhanced_response_with_context( search_memories = search_engine.search( original_query, top_k=self.config.top_k, mode="fast" ) - system_prompt = self._build_system_prompt( + # Issue #2524: emit a stable system prompt + per-turn dynamic + # context so provider prefix caches can reuse the chat history. + stable_system, dynamic_ctx = self._build_cacheable_system_parts( search_memories, base_prompt=base_prompt - ) # Use the same system prompt builder + ) else: - system_prompt = self._build_system_prompt(base_prompt=base_prompt) + stable_system, dynamic_ctx = self._build_cacheable_system_parts( + None, base_prompt=base_prompt + ) + + system_prompt = f"{stable_system}\n\n{SYNTHESIS_PROMPT}" + turn_context = f"Sub-questions and answers:\n{qa_text}" + if dynamic_ctx: + turn_context = f"{dynamic_ctx}\n\n{turn_context}" current_messages = [ - {"role": "system", "content": system_prompt + SYNTHESIS_PROMPT.format(qa_text=qa_text)}, + {"role": "system", "content": system_prompt}, *chat_history.chat_history, { "role": "user", - "content": original_query, + "content": f"[Context for this turn]\n{turn_context}\n[User question]\n{original_query}", }, ] diff --git a/src/memos/templates/cloud_service_prompt.py b/src/memos/templates/cloud_service_prompt.py index 15bc74a3f..0b8a84bd7 100644 --- a/src/memos/templates/cloud_service_prompt.py +++ b/src/memos/templates/cloud_service_prompt.py @@ -1,3 +1,5 @@ +import re + from datetime import datetime @@ -105,3 +107,124 @@ def get_cloud_chat_prompt(lang: str = "en") -> str: ) else: raise ValueError(f"Invalid language: {lang}") + + +# -- Prefix-cache-friendly helpers (Issue #2524) --------------------------- +# +# The original ``get_cloud_chat_prompt`` embeds ``{current_time}`` (minute +# precision) and splices the current turn's retrieval into ``{memories}``. +# Because the resulting string is sent as the first (system) message of every +# turn, provider prefix caches cannot match any shared prefix across turns. +# +# The helpers below return the two halves separately: +# +# * ``get_cloud_chat_stable_prompt(lang)`` -- byte-identical across turns in +# one conversation. Safe to use as the leading system message so that the +# chat history that follows becomes part of the shared prefix. +# * ``get_cloud_chat_dynamic_context(...)`` -- per-turn context carrying the +# current time, the retrieved memories, and the retrieved preferences. The +# caller prepends this as a context message before the latest question. +# --------------------------------------------------------------------------- + + +_STABLE_PROMPT_SENTINEL = ( + "When present, the [Context for this turn] message before the latest question contains " + "the current time, retrieved memories, or preferences. Read it as context before answering." +) + +_STABLE_PROMPT_SENTINEL_ZH = "如果最新问题前有以 [Context for this turn] 开头的消息,请在回答前读取其中的时间、记忆或偏好作为上下文。" + + +def _build_stable_from_cloud_prompt(prompt: str, sentinel: str) -> str: + """Return the cloud chat prompt with the per-turn placeholders replaced + by a short sentinel pointing to the dynamic context message. + + The template surgery is implemented with regular expressions rather than + exact-string ``str.replace`` calls so that minor rewording of the + surrounding template (extra whitespace, punctuation tweaks, new wording + around ``{current_time}`` / ``{memories}``) still produces a correct + stable prompt. If a placeholder would survive the surgery we raise + ``ValueError`` loudly instead of leaking the raw template token to the + model. + """ + # Replace the "- Current Time: {current_time} ..." line (English or + # Chinese) with the sentinel. Match the whole line so parenthetical + # qualifiers after the placeholder are also removed. + stable = re.sub( + r"- *(?:Current Time|当前时间)[::][^\n]*\{current_time\}[^\n]*", + f"- {sentinel}", + prompt, + ) + # Replace the ...{memories}... block (any + # whitespace between the tags) with the sentinel on its own line. + stable = re.sub( + r"\s*\{memories\}\s*", + sentinel, + stable, + flags=re.DOTALL, + ) + # Loud failure beats silent leak: if a placeholder survived, the template + # has drifted and the derivation needs updating. + for leftover in ("{current_time}", "{memories}"): + if leftover in stable: + raise ValueError( + "Cloud chat template drifted: placeholder " + f"{leftover!r} was not substituted when building the stable " + "system prompt. Update _build_stable_from_cloud_prompt." + ) + return stable + + +def get_cloud_chat_stable_prompt(lang: str = "en") -> str: + """Return the stable (cross-turn constant) cloud chat prompt. + + The returned string must be identical across turns in one conversation. + No current time, no retrieved memories. The template points the model to + the dynamic context message that follows the chat history. + """ + if lang == "zh": + return _build_stable_from_cloud_prompt(CLOUD_CHAT_PROMPT_ZH, _STABLE_PROMPT_SENTINEL_ZH) + if lang == "en": + return _build_stable_from_cloud_prompt(CLOUD_CHAT_PROMPT_EN, _STABLE_PROMPT_SENTINEL) + raise ValueError(f"Invalid language: {lang}") + + +def get_cloud_chat_dynamic_context( + *, + memories_block: str, + pref_string: str | None = None, + now: datetime | None = None, + lang: str = "en", +) -> str: + """Return the per-turn dynamic context message. + + Args: + memories_block: already-formatted memories (e.g. "1. ...\\n2. ..."). + Pass an empty string when there are no memories. + pref_string: formatted preference string, or ``None``. + now: timestamp to use for the "Current Time" line. Defaults to + ``datetime.now()``; parameterised to help tests. + lang: "en" or "zh". + """ + stamp = (now or datetime.now()).strftime("%Y-%m-%d %H:%M (%A)") + + memories_section = memories_block.strip() + pref_section = (pref_string or "").strip() + + if lang == "zh": + parts = [f"# 当前上下文\n- 当前时间: {stamp}"] + if memories_section: + parts.append("# 事实与记忆\n\n" + memories_section + "\n") + if pref_section: + parts.append("# 偏好\n" + pref_section) + return "\n\n".join(parts) + + if lang == "en": + parts = [f"# Current Context\n- Current Time: {stamp}"] + if memories_section: + parts.append("# Facts & Memories\n\n" + memories_section + "\n") + if pref_section: + parts.append("# Preferences\n" + pref_section) + return "\n\n".join(parts) + + raise ValueError(f"Invalid language: {lang}") diff --git a/src/memos/templates/mos_prompts.py b/src/memos/templates/mos_prompts.py index ab0226a30..add9119a8 100644 --- a/src/memos/templates/mos_prompts.py +++ b/src/memos/templates/mos_prompts.py @@ -51,9 +51,7 @@ SYNTHESIS_PROMPT = """ exclude memory information, synthesizing information from multiple sources to provide comprehensive answers. -I will give you chain of thought for sub-questions and their answers. -Sub-questions and their answers: -{qa_text} +Use the sub-questions and answers in the latest user message as evidence. Please synthesize these answers into a comprehensive response that: 1. Addresses the original question completely diff --git a/tests/api/test_chat_message_order.py b/tests/api/test_chat_message_order.py new file mode 100644 index 000000000..fcc896246 --- /dev/null +++ b/tests/api/test_chat_message_order.py @@ -0,0 +1,283 @@ +"""Prefix-cache-friendly chat message assembly (Issue #2524). + +These tests pin down the invariants from the issue: +- The first message (stable system prompt) must stay byte-identical across turns + within one conversation. +- Chat history sits inside the shared prefix (between the stable system message + and the per-turn dynamic context). +- Per-turn retrieved memories, preferences, and the current time move into a + separate context message that is placed immediately before the final user + query so that the common prefix across requests can grow. +- A custom base_prompt that embeds `{memories}` keeps the legacy single-system + shape (backward compatibility). +""" + +from __future__ import annotations + +import re + +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +from memos.api.handlers.chat_handler import ChatHandler + + +if TYPE_CHECKING: + import pytest + + +def _make_handler(monkeypatch: pytest.MonkeyPatch) -> ChatHandler: + """Construct a ChatHandler without going through its real __init__. + + The real __init__ validates dependencies and wires sub-handlers; the tests + here only exercise the message-assembly helpers and inject fakes for + `chat_llms` / `search_handler` as needed. + """ + handler = ChatHandler.__new__(ChatHandler) + handler.logger = MagicMock() + handler.chat_llms = {} + handler.playground_chat_llms = {} + handler.search_handler = MagicMock() + handler.add_handler = MagicMock() + handler.online_bot = None + handler.enable_mem_scheduler = False + handler.dependencies = MagicMock() + return handler + + +def _memory(mem_id: str, text: str) -> dict: + return { + "id": mem_id, + "memory": text, + "metadata": {"memory_type": "PersonalMemory", "updated_at": "2026-01-01T00:00:00"}, + "relativity": 0.9, + } + + +class TestStableAndDynamicSplit: + """INV-1 / INV-3 / INV-5: the handler can produce a stable system prompt + plus a per-turn dynamic context string.""" + + def test_stable_system_prompt_omits_current_time_and_memories( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + memories = [_memory("m-1", "User likes dim sum on weekends")] + + prompt_turn_1 = handler._build_stable_system_prompt( + query="What should I eat?", base_prompt=None + ) + prompt_turn_2 = handler._build_stable_system_prompt( + query="How about tonight?", base_prompt=None + ) + + # INV-1: byte-identical across turns. + assert prompt_turn_1 == prompt_turn_2 + # Dynamic data must not leak into the stable prompt. + assert "User likes dim sum" not in prompt_turn_1 + # No current-minute timestamp substituted into the stable prompt. + assert not re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}", prompt_turn_1) + # The placeholder itself must be gone — otherwise the model sees "{current_time}". + assert "{current_time}" not in prompt_turn_1 + assert "{memories}" not in prompt_turn_1 + _ = memories # explicit: memories not used while building stable prompt + + def test_dynamic_context_contains_time_memories_and_preferences( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + memories = [ + _memory("m-1", "User lives in Shanghai"), + _memory("m-2", "User prefers short answers"), + ] + pref_string = "- answer in English" + + dynamic = handler._build_dynamic_context_message( + memories=memories, + pref_string=pref_string, + lang="en", + ) + + assert dynamic is not None + assert "User lives in Shanghai" in dynamic + assert "User prefers short answers" in dynamic + assert "- answer in English" in dynamic + # Current time appears here, not in the stable prompt. + assert re.search(r"\d{4}-\d{2}-\d{2}", dynamic) + + def test_dynamic_context_returns_none_when_nothing_dynamic( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + # No memories, no preferences — still, we may want to emit the current + # time or nothing; the invariant is just "if everything is empty, + # don't inject a message". + dynamic = handler._build_dynamic_context_message( + memories=[], + pref_string=None, + lang="en", + include_time=False, + ) + assert dynamic is None + + +class TestCacheableAssemblyShape: + """INV-2 / INV-3: assembled message list has stable[0], history, dynamic, user.""" + + def test_messages_place_history_before_dynamic_context( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + + history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + memories = [_memory("m-1", "User lives in Shanghai")] + + messages = handler._assemble_cacheable_messages( + query="Where should I eat tonight?", + history=history, + memories=memories, + pref_string="- short answers", + base_prompt=None, + lang="en", + ) + + # Stable system first. + assert messages[0]["role"] == "system" + assert "User lives in Shanghai" not in messages[0]["content"] + + # History sits inside the shared prefix. + assert messages[1] == history[0] + assert messages[2] == history[1] + + # Dynamic context is a user-role framing message right before the + # user query (OCR round 2 feedback: mid-conversation ``system`` + # messages are not portable across provider APIs). + assert messages[-2]["role"] == "user" + assert messages[-2]["content"].startswith("[Context for this turn]\n") + assert "User lives in Shanghai" in messages[-2]["content"] + + # Final message is the user turn. + assert messages[-1] == {"role": "user", "content": "Where should I eat tonight?"} + + def test_shared_prefix_grows_across_turns(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Two consecutive turns must share stable + prior-history prefix.""" + handler = _make_handler(monkeypatch) + + turn1_history: list[dict] = [] + turn1 = handler._assemble_cacheable_messages( + query="Hi", + history=turn1_history, + memories=[_memory("m-1", "User lives in Shanghai")], + pref_string=None, + base_prompt=None, + lang="en", + ) + + # Second turn carries turn 1's exchange as history. + turn2_history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + turn2 = handler._assemble_cacheable_messages( + query="Where should I eat tonight?", + history=turn2_history, + memories=[_memory("m-2", "User likes dim sum")], + pref_string=None, + base_prompt=None, + lang="en", + ) + + # INV-1: stable system message is identical. + assert turn1[0] == turn2[0] + # INV-2: turn 1's exchange (positions 1..2 in turn 2) matches history. + assert turn2[1] == turn2_history[0] + assert turn2[2] == turn2_history[1] + # Longest shared prefix spans at least the stable system. + assert turn1[0]["content"] == turn2[0]["content"] + + def test_assemble_omits_dynamic_message_when_no_memories_or_prefs( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + messages = handler._assemble_cacheable_messages( + query="How are you?", + history=history, + memories=[], + pref_string=None, + base_prompt=None, + lang="en", + include_dynamic_time=False, + ) + + # Without dynamic payload: stable system + history + user. No extra turn. + assert [m["role"] for m in messages] == ["system", "user", "assistant", "user"] + assert messages[-1] == {"role": "user", "content": "How are you?"} + + +class TestLegacyBasePromptCompat: + """INV-4: a custom base_prompt that embeds `{memories}` keeps the single- + system shape (no reorder).""" + + def test_custom_base_prompt_with_memories_placeholder_keeps_legacy_shape( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + + history = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + memories = [_memory("m-1", "User likes blue")] + base_prompt = "You are a tailor. User memories: {memories}" + + messages = handler._assemble_cacheable_messages( + query="Suggest a jacket colour", + history=history, + memories=memories, + pref_string=None, + base_prompt=base_prompt, + lang="en", + ) + + # Legacy shape: [system(stable+inline memories), *history, user] + assert len(messages) == 1 + len(history) + 1 + assert messages[0]["role"] == "system" + assert "User likes blue" in messages[0]["content"] + assert "{memories}" not in messages[0]["content"] + assert messages[1:-1] == history + assert messages[-1] == {"role": "user", "content": "Suggest a jacket colour"} + + +class TestEnhanceEndpointSplit: + """Playground path uses `_build_enhance_*`. Same invariants must hold.""" + + def test_enhance_stable_prompt_omits_time_and_memories( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + stable = handler._build_enhance_stable_prompt(lang="en", tone="friendly", verbosity="mid") + # No ISO date, no personal/outer memory text. + assert not re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}", stable) + assert "PersonalMemory (ordered)" not in stable or "## PersonalMemory (ordered)" in stable + # The {date} placeholder must be substituted away (either with empty or + # with a sentinel pointing to the context message). + assert "{date}" not in stable + + def test_enhance_dynamic_context_contains_time_and_memory_blocks( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + handler = _make_handler(monkeypatch) + memories = [_memory("m-1", "User lives in Shanghai")] + dynamic = handler._build_enhance_dynamic_context( + memories_list=memories, pref_string="- short answers", lang="en" + ) + assert "User lives in Shanghai" in dynamic + assert "- short answers" in dynamic + assert re.search(r"\d{4}-\d{2}-\d{2}", dynamic) diff --git a/tests/mem_os/test_cot_message_order.py b/tests/mem_os/test_cot_message_order.py new file mode 100644 index 000000000..79a923761 --- /dev/null +++ b/tests/mem_os/test_cot_message_order.py @@ -0,0 +1,34 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +from memos.mem_os.main import MOS + + +def test_cot_synthesis_keeps_history_and_instructions_in_shared_prefix(): + mos = MOS.__new__(MOS) + mos.config = SimpleNamespace(enable_textual_memory=False, enable_activation_memory=False) + mos.chat_llm = Mock() + mos.chat_llm.generate.return_value = "answer" + history_messages = [ + {"role": "user", "content": "previous question"}, + {"role": "assistant", "content": "previous answer"}, + ] + history = SimpleNamespace(chat_history=history_messages) + + for turn in ("first", "second"): + mos._generate_enhanced_response_with_context( + original_query=f"{turn} user question", + sub_questions=[f"{turn} subquestion"], + sub_answers=[f"{turn} subanswer"], + chat_history=history, + ) + + first, second = (call.args[0] for call in mos.chat_llm.generate.call_args_list) + assert first[:-1] == second[:-1] + assert first[1:-1] == history_messages + assert [message["role"] for message in first] == ["system", "user", "assistant", "user"] + assert "comprehensive response" in first[0]["content"] + for turn, messages in (("first", first), ("second", second)): + assert f"{turn} subquestion" in messages[-1]["content"] + assert f"{turn} subanswer" in messages[-1]["content"] + assert messages[-1]["content"].endswith(f"{turn} user question") diff --git a/tests/mem_os/test_memos_core.py b/tests/mem_os/test_memos_core.py index b57b0b254..09f6390be 100644 --- a/tests/mem_os/test_memos_core.py +++ b/tests/mem_os/test_memos_core.py @@ -636,7 +636,12 @@ def test_chat_with_custom_base_prompt( custom_prompt = "You are a pirate. Answer as such. User memories: {memories}" mos.chat("What do I like?", base_prompt=custom_prompt) - # Verify that the system prompt passed to the LLM is the custom one + # Verify that the system prompt passed to the LLM is the custom one. + # Issue #2524: with the cache-friendly split the stable system prompt + # is base_prompt with ``{memories}`` stripped, and the per-turn + # retrieved memories move into a separate system message immediately + # before the final user turn so the chat history that follows the + # stable prompt stays part of the provider's shared prefix. mock_llm.generate.assert_called_once() call_args = mock_llm.generate.call_args[0] messages = call_args[0] @@ -645,7 +650,15 @@ def test_chat_with_custom_base_prompt( assert "You are a pirate." in system_prompt assert "You are a knowledgeable and helpful AI assistant." not in system_prompt assert "User memories:" in system_prompt - assert "I like playing football" in system_prompt # Check if memory is interpolated + # Memories now live in the dynamic context message (second-to-last), + # delivered as a user-role framing turn (OCR round 2 feedback on PR + # #2529: mid-conversation ``{"role": "system"}`` is not portable). + assert messages[-1]["role"] == "user" + assert messages[-2]["role"] == "user" + assert messages[-2]["content"].startswith("[Context for this turn]\n") + assert "I like playing football" in messages[-2]["content"] + # And they must NOT be inlined into the stable prefix anymore. + assert "I like playing football" not in system_prompt @patch("memos.mem_os.core.UserManager") @patch("memos.mem_os.core.MemReaderFactory") @@ -682,6 +695,63 @@ def test_chat_without_memories( # Verify response assert response == "This is a test response from the assistant." + @patch("memos.mem_os.core.UserManager") + @patch("memos.mem_os.core.MemReaderFactory") + @patch("memos.mem_os.core.LLMFactory") + def test_chat_places_history_before_dynamic_context_issue_2524( + self, + mock_llm_factory, + mock_reader_factory, + mock_user_manager_class, + mock_config, + mock_llm, + mock_mem_reader, + mock_user_manager, + mock_mem_cube, + ): + """Issue #2524: chat_history must sit between the stable system + message and the per-turn dynamic context so provider prefix caches + can reuse the history across turns.""" + mock_llm_factory.from_config.return_value = mock_llm + mock_reader_factory.from_config.return_value = mock_mem_reader + mock_user_manager_class.return_value = mock_user_manager + + mos = MOSCore(MOSConfig(**mock_config)) + mos.mem_cubes["test_cube_1"] = mock_mem_cube + mos.mem_cubes["test_cube_2"] = mock_mem_cube + + # Seed chat history with a prior exchange. + if mos.user_id not in mos.chat_history_manager: + mos._register_chat_history(mos.user_id) + chat_history = mos.chat_history_manager[mos.user_id] + chat_history.chat_history.extend( + [ + {"role": "user", "content": "Previous question"}, + {"role": "assistant", "content": "Previous answer"}, + ] + ) + + mos.chat("What should I eat?") + + call_args = mock_llm.generate.call_args[0] + messages = call_args[0] + + # Stable system first. + assert messages[0]["role"] == "system" + assert "I like playing football" not in messages[0]["content"] + + # Prior history inside the shared prefix. + assert messages[1] == {"role": "user", "content": "Previous question"} + assert messages[2] == {"role": "assistant", "content": "Previous answer"} + + # Dynamic context immediately before the final user turn, delivered + # as a user-role framing message (OCR round 2 feedback: avoid + # mid-conversation ``{"role": "system"}``). + assert messages[-1] == {"role": "user", "content": "What should I eat?"} + assert messages[-2]["role"] == "user" + assert messages[-2]["content"].startswith("[Context for this turn]\n") + assert "I like playing football" in messages[-2]["content"] + # TODO: test clear message