diff --git a/src/memos/exceptions.py b/src/memos/exceptions.py index 511e68d8f..e9aeffd5c 100644 --- a/src/memos/exceptions.py +++ b/src/memos/exceptions.py @@ -41,3 +41,12 @@ class EmbedderError(MemOSError): ... class ParserError(MemOSError): ... + + +class LLMResponseParseError(ParserError): + """Raised when an LLM response cannot be parsed into the expected shape. + + Used in place of a silent ``{}`` return so that callers can distinguish + "the model produced no memories" from "the model produced output we + couldn't parse". Issue #2456. + """ diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index 8e5778ce5..c96f8c76a 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -10,7 +10,7 @@ from memos import log from memos.configs.mem_reader import MultiModalStructMemReaderConfig from memos.context.context import ContextThreadPoolExecutor -from memos.exceptions import EmbedderError +from memos.exceptions import EmbedderError, LLMResponseParseError from memos.mem_reader.read_multi_modal import MultiModalParser, detect_lang from memos.mem_reader.read_multi_modal.base import _derive_key from memos.mem_reader.read_pref_memory.process_preference_memory import process_preference_fine @@ -539,6 +539,8 @@ def _get_llm_response( try: response_text = self.llm.generate(messages) response_json = parse_json_result(response_text) + except LLMResponseParseError: + raise except Exception as e: logger.error(f"[LLM] Exception during chat generation: {e}") response_json = { @@ -732,6 +734,8 @@ def _process_one_item( # ========== Stage 1: Normal extraction (without reference) ========== try: resp = self._get_llm_response(mem_str, custom_tags, sources, prompt_type) + except LLMResponseParseError: + raise except Exception as e: logger.error(f"[MultiModalFine] Error calling LLM: {e}") return fine_items @@ -876,6 +880,8 @@ def _process_one_item( result = future.result() if result: fine_memory_items.extend(result) + except LLMResponseParseError: + raise except Exception as e: logger.error(f"[MultiModalFine] worker error: {e} {traceback.format_exc()}") @@ -1352,7 +1358,7 @@ def _read_memory( res_memory = future.result() if res_memory is not None: memory_list.append(res_memory) - except EmbedderError: + except (EmbedderError, LLMResponseParseError): raise except Exception as e: logger.error(f"Task failed with exception: {e}") diff --git a/src/memos/mem_reader/simple_struct.py b/src/memos/mem_reader/simple_struct.py index 2804bac23..78885ecfd 100644 --- a/src/memos/mem_reader/simple_struct.py +++ b/src/memos/mem_reader/simple_struct.py @@ -15,6 +15,7 @@ from memos.configs.mem_reader import SimpleStructMemReaderConfig from memos.context.context import ContextThreadPoolExecutor from memos.embedders.factory import EmbedderFactory +from memos.exceptions import LLMResponseParseError from memos.llms.factory import LLMFactory from memos.mem_reader.base import BaseMemReader @@ -119,6 +120,8 @@ def _build_node(idx, message, info, source_info, llm, parse_json_result, embedde if not chunk_res: logger.warning(f"[Parse] Failed to parse result: {raw}") return None + except LLMResponseParseError: + raise except Exception as e: logger.error(f"[Parse] Exception during JSON parsing: {e}") return None @@ -263,10 +266,12 @@ def _safe_generate(self, messages: list[dict]) -> str | None: return None def _safe_parse(self, text: str | None) -> dict | None: - if not text: + if text is None: return None try: return parse_json_result(text) + except LLMResponseParseError: + raise except Exception: logger.warning("[LLM] JSON parse failed") return None @@ -291,7 +296,7 @@ def _get_llm_response(self, mem_str: str, custom_tags: list[str] | None) -> dict response_text = self._safe_generate(messages) response_json = self._safe_parse(response_text) - if not response_json: + if response_json is None: # NOTE: the key MUST be ``"memory list"`` (with a space) — the # downstream consumers in ``_process_chat_data`` / # ``_process_transfer_chat_data`` read via @@ -702,6 +707,8 @@ def _read_memory( res_memory = future.result() if res_memory is not None: memory_list.append(res_memory) + except LLMResponseParseError: + raise except Exception as e: logger.error(f"Task failed with exception: {e}") logger.error(traceback.format_exc()) @@ -776,6 +783,8 @@ def fine_transfer_simple_mem( res_memory = future.result() if res_memory is not None: memory_list.append(res_memory) + except LLMResponseParseError: + raise except Exception as e: logger.error(f"Task failed with exception: {e}") logger.error(traceback.format_exc()) @@ -966,6 +975,8 @@ def _process_doc_data(self, scene_data_info, info, **kwargs): node = future.result() if node: doc_nodes.append(node) + except LLMResponseParseError: + raise except Exception as e: tqdm.write(f"[ERROR] {e}") logger.error(f"[DocReader] Future task failed: {e}") diff --git a/src/memos/mem_reader/utils.py b/src/memos/mem_reader/utils.py index c08e8a4ed..38ef4e587 100644 --- a/src/memos/mem_reader/utils.py +++ b/src/memos/mem_reader/utils.py @@ -2,6 +2,7 @@ import re from memos import log +from memos.exceptions import LLMResponseParseError logger = log.get_logger(__name__) @@ -36,27 +37,41 @@ def derive_key(text: str, max_len: int = 80) -> str: def parse_json_result(response_text: str) -> dict: - s = (response_text or "").strip() + """Parse a JSON object out of an LLM response. - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.I) + Returns the parsed ``dict`` when the response contains a parseable JSON + object (including a literal ``{}``). Raises ``LLMResponseParseError`` + on any other input so callers can distinguish "the model produced no + memories" from "the model produced output we couldn't parse". Issue + #2456. + """ + raw = response_text or "" + s = raw.strip() + + if not s: + raise LLMResponseParseError("LLM response was empty") + + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.IGNORECASE) s = (m.group(1) if m else s.replace("```", "")).strip() i = s.find("{") if i == -1: - return {} + raise LLMResponseParseError("No JSON object found in LLM response") s = s[i:].strip() + last_err: json.JSONDecodeError | None = None + try: return json.loads(s) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + last_err = e j = max(s.rfind("}"), s.rfind("]")) if j != -1: try: return json.loads(s[: j + 1]) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + last_err = e def _cheap_close(t: str) -> str: t += "}" * max(0, t.count("{") - t.count("}")) @@ -67,14 +82,15 @@ def _cheap_close(t: str) -> str: try: return json.loads(t) except json.JSONDecodeError as e: + last_err = e if "Invalid \\escape" in str(e): - s = s.replace("\\", "\\\\") - return json.loads(s) - logger.warning( - f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \ - json: {s}" - ) - return {} + try: + return json.loads(s.replace("\\", "\\\\")) + except json.JSONDecodeError as e2: + last_err = e2 + + logger.warning("[JSONParse] Failed to decode JSON at position %s", last_err.pos) + raise LLMResponseParseError("Failed to decode JSON from LLM response") from last_err def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]: @@ -83,7 +99,7 @@ def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]: Returns (success, parsed_dict) with int keys. """ try: - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I) + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE) s = (m.group(1) if m else text).strip() data = json.loads(s) except Exception: @@ -127,7 +143,7 @@ def parse_keep_filter_response(text: str) -> tuple[bool, dict[int, dict]]: Returns (success, parsed_dict) with int keys. """ try: - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I) + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE) s = (m.group(1) if m else text).strip() data = json.loads(s) except Exception: diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index 5e663c99e..305648298 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING from memos.context.context import ContextThreadPoolExecutor +from memos.exceptions import LLMResponseParseError from memos.log import get_logger from memos.mem_scheduler.schemas.message_schemas import ScheduleMessageItem from memos.mem_scheduler.schemas.task_schemas import ( @@ -178,6 +179,8 @@ def _process_memories_with_reader( user_context=user_context, is_upload_skill=is_upload_skill, ) + except LLMResponseParseError: + raise except Exception as e: logger.warning("%s: Fail to transfer mem: %s", e, memory_items, exc_info=True) return diff --git a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py index 0563d6c7a..99a69d7e6 100644 --- a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py +++ b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py @@ -1167,7 +1167,14 @@ def _sort_and_trim( include_preference_memory=False, pref_mem_top_k=6, ): - """Sort results by score and trim to top_k""" + """Sort results by score and trim to top_k. + + Note: + Never mutates the input items: selected textual items are returned + as independent copies (``model_copy``, no revalidation) carrying the + final ``relativity``; other metadata types are rebuilt into + ``SearchedTreeNodeTextualMemoryMetadata`` as before. + """ final_items = [] if search_tool_memory: tool_schema_results = [ @@ -1268,15 +1275,25 @@ def _sort_and_trim( for item, score in sorted_results: if plugin and round(score, 2) == 0.00: continue - meta_data = item.metadata.model_dump() - meta_data["relativity"] = score - final_items.append( - TextualMemoryItem( - id=item.id, - memory=item.memory, - metadata=SearchedTreeNodeTextualMemoryMetadata(**meta_data), + metadata = item.metadata + if isinstance(metadata, SearchedTreeNodeTextualMemoryMetadata): + # Decouple from caller-owned items: deep-copy metadata without + # revalidation (cheap), writing the final score on the copy + # with the same float() coercion the rebuild path applies. + metadata = metadata.model_copy(deep=True, update={"relativity": float(score)}) + final_items.append(item.model_copy(update={"metadata": metadata})) + else: + # Type upgrade (e.g. graph/bm25 results without relativity): + # rebuild into the searched metadata type as before. + meta_data = metadata.model_dump() + meta_data["relativity"] = score + final_items.append( + TextualMemoryItem( + id=item.id, + memory=item.memory, + metadata=SearchedTreeNodeTextualMemoryMetadata(**meta_data), + ) ) - ) return final_items @timed diff --git a/tests/api/test_server_router.py b/tests/api/test_server_router.py index 5906697d9..424df4e37 100644 --- a/tests/api/test_server_router.py +++ b/tests/api/test_server_router.py @@ -415,3 +415,59 @@ def test_get_all_response_format(self, mock_handlers, client): data = response.json() assert data["message"] == "Memories retrieved successfully" assert isinstance(data["data"], list) + + +@pytest.mark.parametrize( + ("llm_response", "expected_status"), + [("malformed extraction", 500), ("{}", 200), ('{"memory list": []}', 200)], +) +def test_sync_add_distinguishes_parse_failure_from_empty_result( + mock_init_server, monkeypatch, llm_response, expected_status +): + from types import SimpleNamespace + from unittest.mock import MagicMock + + from memos.api.handlers.add_handler import AddHandler + from memos.api.handlers.base_handler import HandlerDependencies + from memos.mem_reader.simple_struct import SimpleStructMemReader + + reader = SimpleStructMemReader.__new__(SimpleStructMemReader) + reader.config = SimpleNamespace(remove_prompt_example=False) + reader.llm = MagicMock() + reader.llm.generate.return_value = llm_response + reader.chat_window_max_tokens = 1024 + reader._count_tokens = len + reader.save_rawfile = False + reader.memory_version_switch = "off" + cube = MagicMock() + cube.text_mem.add.return_value = [] + scheduler = MagicMock() + handler = AddHandler( + HandlerDependencies( + naive_mem_cube=cube, + mem_reader=reader, + mem_scheduler=scheduler, + feedback_server=MagicMock(), + ) + ) + monkeypatch.setattr("memos.api.routers.server_router.add_handler", handler) + + response = TestClient(mock_init_server, raise_server_exceptions=False).post( + "/product/add", + json={ + "user_id": "synthetic-user", + "writable_cube_ids": ["synthetic-cube"], + "messages": [{"role": "user", "content": "Synthetic input"}], + "async_mode": "sync", + "mode": "fine", + }, + ) + + assert response.status_code == expected_status + assert response.json()["code"] == expected_status + if expected_status == 500: + cube.text_mem.add.assert_not_called() + scheduler.submit_messages.assert_not_called() + else: + assert response.json()["data"] == [] + assert cube.text_mem.add.call_args.args[0] == [] diff --git a/tests/mem_reader/test_extraction_parse_failure.py b/tests/mem_reader/test_extraction_parse_failure.py new file mode 100644 index 000000000..7fbd41240 --- /dev/null +++ b/tests/mem_reader/test_extraction_parse_failure.py @@ -0,0 +1,75 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.exceptions import LLMResponseParseError +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.mem_reader.simple_struct import SimpleStructMemReader +from memos.mem_reader.utils import parse_json_result +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata + + +@pytest.fixture(params=[SimpleStructMemReader, MultiModalStructMemReader]) +def reader(request): + instance = request.param.__new__(request.param) + instance.config = SimpleNamespace(remove_prompt_example=False) + instance.llm = MagicMock() + instance.general_llm = MagicMock() + instance.embedder = MagicMock() + instance.embedder.embed.side_effect = lambda texts: [[0.1] for _ in texts] + instance.chat_window_max_tokens = 1024 + instance.memory_version_switch = "off" + instance.save_rawfile = False + return instance + + +@pytest.mark.parametrize("response", ["", "prose", '{"memory list": [}', "[]", '[{"x": 1}]']) +def test_fine_extraction_propagates_parse_failure(reader, response): + reader.llm.generate.return_value = response + with pytest.raises(LLMResponseParseError): + reader._get_llm_response("Synthetic input", None) + + +@pytest.mark.parametrize("response", ["{}", '{"memory list": []}']) +def test_empty_extraction_is_success_without_fallback(reader, response): + reader.llm.generate.return_value = response + assert reader._get_llm_response("Synthetic input", None) == parse_json_result(response) + + +def test_multimodal_worker_cannot_hide_parse_failure(): + reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + reader.config = SimpleNamespace(remove_prompt_example=False) + reader.llm = MagicMock() + reader.llm.generate.return_value = "malformed extraction" + reader.memory_version_switch = "off" + reader.save_rawfile = False + item = TextualMemoryItem( + memory="Synthetic input", + metadata=TreeNodeTextualMemoryMetadata(sources=[]), + ) + with pytest.raises(LLMResponseParseError): + reader._process_string_fine([item], {}) + + +@pytest.mark.parametrize("reader_cls", [SimpleStructMemReader, MultiModalStructMemReader]) +def test_public_reader_propagates_worker_parse_failure(monkeypatch, reader_cls): + reader = reader_cls.__new__(reader_cls) + error = LLMResponseParseError("Synthetic parse failure") + + def fail(*args, **kwargs): + raise error + + method = ( + "_process_multi_modal_data" + if reader_cls is MultiModalStructMemReader + else "_process_chat_data" + ) + monkeypatch.setattr(reader, method, fail) + with pytest.raises(LLMResponseParseError, match="Synthetic parse failure"): + reader.get_memory( + [[{"role": "user", "content": "Synthetic input"}]], + "chat", + {"user_id": "synthetic-user", "session_id": "synthetic-session"}, + mode="fine", + ) diff --git a/tests/mem_reader/test_parse_json_result_strict.py b/tests/mem_reader/test_parse_json_result_strict.py new file mode 100644 index 000000000..177051ae2 --- /dev/null +++ b/tests/mem_reader/test_parse_json_result_strict.py @@ -0,0 +1,75 @@ +"""Regression tests for issue #2456. + +``parse_json_result`` previously swallowed every JSON-decode failure and +returned ``{}``, so a malformed LLM extraction was indistinguishable from a +successful empty extraction. Downstream callers saw ``{"code": 200, "data": []}`` +for ``/product/add`` requests that actually failed. + +These tests pin the new contract: ``parse_json_result`` raises +``LLMResponseParseError`` on any non-recoverable parse failure, and only +returns the parsed dict on success (including a literal ``{}``). +""" + +from __future__ import annotations + +import json + +import pytest + +from memos.exceptions import LLMResponseParseError +from memos.mem_reader.utils import parse_json_result + + +class TestParseJsonResultSuccess: + def test_returns_parsed_object(self): + raw = '{"memory list": [{"value": "hi"}], "summary": "s"}' + result = parse_json_result(raw) + assert result == {"memory list": [{"value": "hi"}], "summary": "s"} + + def test_returns_empty_object(self): + raw = "{}" + result = parse_json_result(raw) + assert result == {} + + def test_strips_markdown_fences(self): + raw = '```json\n{"summary": "s"}\n```' + result = parse_json_result(raw) + assert result == {"summary": "s"} + + +class TestParseJsonResultRaises: + def test_trailing_comma_raises(self): + # Issue #2456 reproducer: trailing comma is invalid JSON. + raw = '{"memory list": [{"value": "synthetic evidence"}],}' + with pytest.raises(LLMResponseParseError) as exc_info: + parse_json_result(raw) + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError) + + def test_no_json_object_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("not-valid-json") + + def test_unclosed_fence_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("```json\n{unterminated") + + def test_empty_string_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("") + + def test_none_input_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result(None) + + def test_chatter_only_raises(self): + # Mirrors the symptom-level scenario from #1493 / #2456: the LLM + # emits only prose. The function must not silently return ``{}``. + with pytest.raises(LLMResponseParseError): + parse_json_result("Here is what I extracted: ... unfortunately I cannot output JSON.") + + def test_exception_chain_preserves_root_decode_error(self): + with pytest.raises(LLMResponseParseError) as exc_info: + parse_json_result('{"x": "unterminated') + # The trailing warning message and chained cause let operators + # debug the underlying JSON error from the original LLM payload. + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError) diff --git a/tests/mem_reader/test_simple_struct_fallback.py b/tests/mem_reader/test_simple_struct_fallback.py index f4b59d7d5..e83bc0529 100644 --- a/tests/mem_reader/test_simple_struct_fallback.py +++ b/tests/mem_reader/test_simple_struct_fallback.py @@ -1,7 +1,7 @@ """Regression tests for issue #1493. `POST /product/add` returned 200 + "Memory added successfully" but persisted no -memory items. Root cause: when the LLM response fails JSON parsing, +memory items. Root cause: when LLM generation fails, `SimpleStructMemReader._get_llm_response` is supposed to emit a fallback dict with a single salvaged `UserMemory` item so the request still produces at least one stored memory. However, the fallback used the key `"memory_list"` @@ -57,9 +57,8 @@ def test_get_llm_response_fallback_key_matches_consumer(self): (with space). The fallback dict therefore MUST use the same key, or it is effectively dead code and the request stores nothing. """ - # Force `_safe_parse` to behave as if the LLM returned unparseable - # output (the realistic failure mode reported in #1493 / #1355). - self.reader.llm.generate.return_value = "this is not json at all" + # Generation failures preserve the raw-input fallback from #1355. + self.reader.llm.generate.side_effect = OSError("Synthetic generation failure") result = self.reader._get_llm_response("I like strawberry.", custom_tags=None) @@ -74,20 +73,16 @@ def test_get_llm_response_fallback_key_matches_consumer(self): self.assertEqual(item["memory_type"], "UserMemory") self.assertEqual(item["value"], "I like strawberry.") - def test_process_chat_data_fine_yields_node_when_llm_unparseable(self): + def test_process_chat_data_fine_yields_node_when_generation_fails(self): """End-to-end: fine-mode `_process_chat_data` must still produce a `TextualMemoryItem` when the LLM output is unparseable JSON. This is the symptom-level test for #1493: an `/product/add` call carrying `"I like strawberry."` should result in at least one memory - item even when the LLM emits non-JSON chatter. + item even when the LLM provider raises. """ - # Realistic Kimi-K2 style failure: LLM wraps response in chatter so - # `parse_json_result` returns `{}`, `_safe_parse` returns `None`, - # `not response_json` is True and the fallback fires. - self.reader.llm.generate.return_value = ( - "Here is what I extracted: ... unfortunately I cannot output JSON." - ) + # Generation errors remain distinguishable from malformed responses. + self.reader.llm.generate.side_effect = OSError("Synthetic generation failure") scene_data_info = [{"role": "user", "content": "I like strawberry."}] info = {"user_id": "user1", "session_id": "session1"} diff --git a/tests/mem_reader/test_simple_structure.py b/tests/mem_reader/test_simple_structure.py index 41272c0b2..64209d4c3 100644 --- a/tests/mem_reader/test_simple_structure.py +++ b/tests/mem_reader/test_simple_structure.py @@ -112,16 +112,22 @@ def test_parse_json_result_success(self): self.assertIn("summary", result) def test_parse_json_result_failure(self): - """Test failure in JSON parsing.""" - raw_response = "Invalid JSON string" - result = parse_json_result(raw_response) + """Test failure in JSON parsing. - self.assertEqual(result, {}) + As of issue #2456, ``parse_json_result`` no longer swallows malformed + JSON; it raises ``LLMResponseParseError`` so callers can distinguish + "no memories produced" from "couldn't parse the response". + """ + from memos.exceptions import LLMResponseParseError + + raw_response = "Invalid JSON string" + with self.assertRaises(LLMResponseParseError): + parse_json_result(raw_response) def test_get_llm_response_fallback_key_matches_consumer(self): """Regression test for issue #1355. - When the LLM response cannot be parsed as JSON, `_get_llm_response` + When LLM generation fails, `_get_llm_response` is expected to return a fallback dict whose ``"memory list"`` (with a single space) entry contains one salvaged ``UserMemory`` item built from the raw user input. Downstream consumers in @@ -130,8 +136,8 @@ def test_get_llm_response_fallback_key_matches_consumer(self): underscore) the salvaged memory is silently dropped and the request produces zero memories despite returning HTTP 200. """ - # Force `_safe_parse` to return None so the fallback branch is taken. - self.reader.llm.generate.return_value = "not-valid-json" + # A generation error still uses the established raw-input fallback. + self.reader.llm.generate.side_effect = OSError("Synthetic generation failure") self.reader.config.remove_prompt_example = False resp = self.reader._get_llm_response("test memory content", custom_tags=None) @@ -147,10 +153,10 @@ def test_get_llm_response_fallback_key_matches_consumer(self): self.assertEqual(salvaged[0]["value"], "test memory content") self.assertEqual(salvaged[0]["memory_type"], "UserMemory") - def test_process_chat_data_fine_yields_node_when_llm_unparseable(self): + def test_process_chat_data_fine_yields_node_when_generation_fails(self): """Regression test for issue #1355 (end-to-end of the reader stage). - With Kimi-style outputs that fail strict JSON parsing, + When the LLM provider raises a generation error, ``_process_chat_data`` (fine mode) should still emit a fallback ``TextualMemoryItem`` so that the upstream `/product/add` request ultimately writes at least one node to Neo4j instead of returning @@ -158,9 +164,8 @@ def test_process_chat_data_fine_yields_node_when_llm_unparseable(self): """ # Embedder produces a stable embedding for the salvaged item. self.reader.embedder.embed = MagicMock(return_value=[[0.1, 0.2, 0.3]]) - # LLM returns garbage so `_safe_parse` returns None and the fallback - # in `_get_llm_response` is exercised. - self.reader.llm.generate.return_value = "I cannot produce JSON sorry" + # A provider failure exercises the raw-input fallback. + self.reader.llm.generate.side_effect = OSError("Synthetic generation failure") self.reader.config.remove_prompt_example = False scene_data_info = [{"role": "user", "content": "test memory content"}] @@ -173,7 +178,7 @@ def test_process_chat_data_fine_yields_node_when_llm_unparseable(self): len(result), 1, "fine-mode _process_chat_data must produce one salvaged memory " - "item when LLM output is unparseable (bug #1355)", + "item when LLM generation fails (bug #1355)", ) self.assertIsInstance(result[0], TextualMemoryItem) self.assertIn("test memory content", result[0].memory) diff --git a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py index 8cdb873ce..62770e296 100644 --- a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py +++ b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py @@ -235,3 +235,34 @@ def test_playground_legacy_events_align_enhanced_ids_with_filtered_mem_group(mon "updated_at": None, } ] + + +def test_parse_failure_preserves_fast_memory_and_reports_failed_task(monkeypatch): + from memos.exceptions import LLMResponseParseError + + handler, text_mem = _build_handler(memory_version_switch="off") + reader = handler.scheduler_context.get_mem_reader() + reader.fine_transfer_simple_mem.side_effect = LLMResponseParseError("Synthetic parse failure") + handler.scheduler_context.services.submit_web_logs = MagicMock() + monkeypatch.setattr( + "memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler.is_playground_api", + lambda: False, + ) + + handler._process_memories_with_reader( + mem_ids=["raw_1"], + user_id="user_1", + mem_cube_id="cube_1", + text_mem=text_mem, + user_name="cube_1", + task_id="task_1", + info={"trigger_source": "Messages"}, + ) + + assert text_mem.add_calls == [] + assert text_mem.delete_calls == [] + assert text_mem.soft_delete_calls == [] + logs = handler.scheduler_context.services.submit_web_logs.call_args.args[0] + assert logs[0].status == "failed" + assert logs[0].task_id == "task_1" + assert "Synthetic parse failure" in logs[0].log_content diff --git a/tests/memories/textual/test_tree_searcher.py b/tests/memories/textual/test_tree_searcher.py index fe40cbb7d..b9f437fcf 100644 --- a/tests/memories/textual/test_tree_searcher.py +++ b/tests/memories/textual/test_tree_searcher.py @@ -2,9 +2,14 @@ from unittest.mock import MagicMock +import numpy as np import pytest -from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.memories.textual.item import ( + SearchedTreeNodeTextualMemoryMetadata, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) from memos.memories.textual.tree_text_memory.retrieve.searcher import Searcher from memos.reranker.base import BaseReranker @@ -168,3 +173,151 @@ def test_deduplicate_rawfile_results_uses_knowledge_summary_ids(mock_searcher): assert result == [raw_file] mock_searcher.graph_store.get_edges.assert_not_called() + + +def make_searched_item(content: str, score: float, relativity: float = 0.5): + """Build a Searched-type candidate as produced by vector/fulltext recall.""" + return ( + TextualMemoryItem( + id=str(uuid.uuid4()), + memory=content, + metadata=SearchedTreeNodeTextualMemoryMetadata( + memory_type="LongTermMemory", + embedding=[0.1] * 5, + usage=[], + sources=[{"type": "chat", "role": "user", "content": "context"}], + relativity=relativity, + ), + ), + score, + ) + + +def rebuild_like_before(item: TextualMemoryItem, score: float) -> TextualMemoryItem: + """Reference implementation of the pre-optimization dump-and-rebuild behavior.""" + meta_data = item.metadata.model_dump() + meta_data["relativity"] = score + return TextualMemoryItem( + id=item.id, + memory=item.memory, + metadata=SearchedTreeNodeTextualMemoryMetadata(**meta_data), + ) + + +def test_sort_and_trim_hot_path_decoupled_copies(mock_searcher): + """Searched-type items come back as decoupled copies matching the old rebuild.""" + inputs = [ + make_searched_item("m-high", 0.9), + make_searched_item("m-mid", 0.8), + make_searched_item("m-low", 0.7), + ] + input_items = {item.id: item for item, _ in inputs} + + result = mock_searcher._sort_and_trim(inputs, top_k=2) + + assert [item.memory for item in result] == ["m-high", "m-mid"] + for item, score in zip(result, [0.9, 0.8], strict=True): + # Selected items are decoupled copies, not the caller-owned objects. + assert item is not input_items[item.id] + assert item.metadata.relativity == score + assert isinstance(item.metadata.relativity, float) + # Output is exactly what the old dump-and-rebuild path produced. + assert item.to_dict() == rebuild_like_before(input_items[item.id], score).to_dict() + # Caller-owned items keep the relativity injected at recall time. + assert input_items[result[0].id].metadata.relativity == 0.5 + assert input_items[result[1].id].metadata.relativity == 0.5 + + +def test_sort_and_trim_cold_path_type_upgrade(mock_searcher): + """Non-Searched metadata (e.g. graph/bm25 results) is still rebuilt.""" + original = make_item("graph-mem", 0.8) + + result = mock_searcher._sort_and_trim([original], top_k=2) + + assert len(result) == 1 + rebuilt = result[0] + assert rebuilt is not original[0] # cold path produces a new item + assert isinstance(rebuilt.metadata, SearchedTreeNodeTextualMemoryMetadata) + assert rebuilt.metadata.relativity == 0.8 + assert rebuilt.to_dict() == rebuild_like_before(original[0], 0.8).to_dict() + + +def test_sort_and_trim_mixed_types(mock_searcher): + """Mixed Searched/TreeNode candidates all come back as Searched with final scores.""" + inputs = [ + make_searched_item("hot-1", 0.6), + make_item("cold-1", 0.95), + make_searched_item("hot-2", 0.7), + ] + + result = mock_searcher._sort_and_trim(inputs, top_k=3) + + assert [item.memory for item in result] == ["cold-1", "hot-2", "hot-1"] + assert all(isinstance(item.metadata, SearchedTreeNodeTextualMemoryMetadata) for item in result) + assert [item.metadata.relativity for item in result] == [0.95, 0.7, 0.6] + + +def test_sort_and_trim_beyond_top_k_untouched(mock_searcher): + """Candidates trimmed away keep the relativity injected at recall time.""" + inputs = [ + make_searched_item("keep", 0.9), + make_searched_item("drop-1", 0.5), + make_searched_item("drop-2", 0.4), + ] + + result = mock_searcher._sort_and_trim(inputs, top_k=1) + + assert [item.memory for item in result] == ["keep"] + for item, _ in inputs[1:]: + assert item.metadata.relativity == 0.5 + + +def test_sort_and_trim_plugin_filters_zero_scores(mock_searcher): + """plugin=True drops candidates whose score rounds to 0.00 without writing.""" + inputs = [make_searched_item("near-zero", 0.001, relativity=0.42)] + + result = mock_searcher._sort_and_trim(inputs, top_k=5, plugin=True) + + assert result == [] + assert inputs[0][0].metadata.relativity == 0.42 + + +def test_sort_and_trim_reuses_input_list_safely(mock_searcher): + """Calling _sort_and_trim twice on the same candidates is idempotent.""" + inputs = [ + make_searched_item("m-1", 0.9), + make_item("m-2", 0.8), + ] + + first = mock_searcher._sort_and_trim(list(inputs), top_k=2) + second = mock_searcher._sort_and_trim(list(inputs), top_k=2) + + assert [item.to_dict() for item in first] == [item.to_dict() for item in second] + assert [item.metadata.relativity for item in first] == [0.9, 0.8] + + +def test_sort_and_trim_coerces_numpy_score(mock_searcher): + """The in-place write applies the same float() coercion pydantic validation did.""" + item, _score = make_searched_item("np-mem", 0.0) + + result = mock_searcher._sort_and_trim([(item, np.float64(0.9))], top_k=1) + + assert isinstance(result[0].metadata.relativity, float) + assert result[0].metadata.relativity == 0.9 + + +def test_sort_and_trim_isolates_nested_metadata_between_results(mock_searcher): + item, score = make_searched_item("independent", 0.9) + item.metadata.tags = ["original"] + item.metadata.info = {"nested": {"values": ["original"]}} + original = item.model_dump() + + first = mock_searcher._sort_and_trim([(item, score)], top_k=1)[0] + second = mock_searcher._sort_and_trim([(item, score)], top_k=1)[0] + first.metadata.tags.append("changed") + first.metadata.sources[0].content = "changed" + first.metadata.info["nested"]["values"].append("changed") + first.metadata.embedding[0] = 0.9 + + assert item.model_dump() == original + assert second.to_dict() == rebuild_like_before(item, score).to_dict()