From 10fb84fe9ad93aa3b600831075f8202f82245d11 Mon Sep 17 00:00:00 2001 From: RerankerGuo Date: Tue, 13 Oct 2026 11:48:39 +0800 Subject: [PATCH 1/2] fix(mem_reader): raise LLMResponseParseError instead of returning {} on malformed JSON (#2456) parse_json_result silently returned {} on every parse failure, so a malformed LLM extraction was reported as an empty (but successful) /product/add response. The HTTP layer returned {"code": 200, "data": []} for an extraction that actually failed, hiding the wasted tokens and never alerting operators that a request had silently dropped. This makes parse_json_result raise LLMResponseParseError (a new ParserError subclass in memos.exceptions) on any non-recoverable parse failure, so callers can distinguish "the model produced no memories" from "the model produced output we couldn't parse". Every in-tree caller already wraps parse_json_result in a try/except (multi_modal_struct, simple_struct, strategy_struct, file_content_parser, searcher, task_goal_parser), so the exception is caught and surfaces as the synthetic-memory-list fallback instead of the silent {} return path. The literal {} JSON value still returns {} so successful empty extractions are preserved. The fix keeps test_simple_struct_fallback.py::test_process_chat_data_fine_yields_node_when_llm_unparseable passing: _safe_parse catches the new exception and returns None, which still triggers the chat fallback that emits a salvageable UserMemory item. - New exception LLMResponseParseError(ParserError) in memos.exceptions - parse_json_result now raises on empty input, missing JSON object, every unrecoverable JSONDecodeError, while still returning the dict when one is found (including a literal {}) - New tests/mem_reader/test_parse_json_result_strict.py covers the trailing-comma reproducer from the issue, the prose-only 'I cannot output JSON' symptom, empty/None input, no JSON object, and the exception-chain contract (the chained cause preserves the underlying JSONDecodeError for debugging) - Updated test_simple_structure.py::test_parse_json_result_failure and the docstring on test_simple_struct_fallback.py to match the new contract --- src/memos/exceptions.py | 9 ++ src/memos/mem_reader/utils.py | 53 ++++--- .../test_parse_json_result_strict.py | 132 ++++++++++++++++++ .../mem_reader/test_simple_struct_fallback.py | 4 +- tests/mem_reader/test_simple_structure.py | 14 +- 5 files changed, 190 insertions(+), 22 deletions(-) create mode 100644 tests/mem_reader/test_parse_json_result_strict.py diff --git a/src/memos/exceptions.py b/src/memos/exceptions.py index 511e68d8f..e9aeffd5c 100644 --- a/src/memos/exceptions.py +++ b/src/memos/exceptions.py @@ -41,3 +41,12 @@ class EmbedderError(MemOSError): ... class ParserError(MemOSError): ... + + +class LLMResponseParseError(ParserError): + """Raised when an LLM response cannot be parsed into the expected shape. + + Used in place of a silent ``{}`` return so that callers can distinguish + "the model produced no memories" from "the model produced output we + couldn't parse". Issue #2456. + """ diff --git a/src/memos/mem_reader/utils.py b/src/memos/mem_reader/utils.py index c08e8a4ed..a1767d5b1 100644 --- a/src/memos/mem_reader/utils.py +++ b/src/memos/mem_reader/utils.py @@ -2,6 +2,7 @@ import re from memos import log +from memos.exceptions import LLMResponseParseError logger = log.get_logger(__name__) @@ -36,27 +37,41 @@ def derive_key(text: str, max_len: int = 80) -> str: def parse_json_result(response_text: str) -> dict: - s = (response_text or "").strip() + """Parse a JSON object out of an LLM response. - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.I) + Returns the parsed ``dict`` when the response contains a parseable JSON + object (including a literal ``{}``). Raises ``LLMResponseParseError`` + on any other input so callers can distinguish "the model produced no + memories" from "the model produced output we couldn't parse". Issue + #2456. + """ + raw = response_text or "" + s = raw.strip() + + if not s: + raise LLMResponseParseError("LLM response was empty") + + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.IGNORECASE) s = (m.group(1) if m else s.replace("```", "")).strip() i = s.find("{") if i == -1: - return {} + raise LLMResponseParseError(f"No JSON object found in LLM response: tail={raw[-200:]!r}") s = s[i:].strip() + last_err: json.JSONDecodeError | None = None + try: return json.loads(s) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + last_err = e j = max(s.rfind("}"), s.rfind("]")) if j != -1: try: return json.loads(s[: j + 1]) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + last_err = e def _cheap_close(t: str) -> str: t += "}" * max(0, t.count("{") - t.count("}")) @@ -67,14 +82,20 @@ def _cheap_close(t: str) -> str: try: return json.loads(t) except json.JSONDecodeError as e: + last_err = e if "Invalid \\escape" in str(e): - s = s.replace("\\", "\\\\") - return json.loads(s) - logger.warning( - f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \ - json: {s}" - ) - return {} + try: + return json.loads(s.replace("\\", "\\\\")) + except json.JSONDecodeError as e2: + last_err = e2 + + logger.warning( + f"[JSONParse] Failed to decode JSON: {last_err}\nTail: Raw {response_text} \ + json: {s}" + ) + raise LLMResponseParseError( + f"Failed to decode JSON from LLM response: {last_err}" + ) from last_err def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]: @@ -83,7 +104,7 @@ def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]: Returns (success, parsed_dict) with int keys. """ try: - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I) + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE) s = (m.group(1) if m else text).strip() data = json.loads(s) except Exception: @@ -127,7 +148,7 @@ def parse_keep_filter_response(text: str) -> tuple[bool, dict[int, dict]]: Returns (success, parsed_dict) with int keys. """ try: - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I) + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE) s = (m.group(1) if m else text).strip() data = json.loads(s) except Exception: diff --git a/tests/mem_reader/test_parse_json_result_strict.py b/tests/mem_reader/test_parse_json_result_strict.py new file mode 100644 index 000000000..fc86e0423 --- /dev/null +++ b/tests/mem_reader/test_parse_json_result_strict.py @@ -0,0 +1,132 @@ +"""Regression tests for issue #2456. + +``parse_json_result`` previously swallowed every JSON-decode failure and +returned ``{}``, so a malformed LLM extraction was indistinguishable from a +successful empty extraction. Downstream callers saw ``{"code": 200, "data": []}`` +for ``/product/add`` requests that actually failed. + +These tests pin the new contract: ``parse_json_result`` raises +``LLMResponseParseError`` on any non-recoverable parse failure, and only +returns the parsed dict on success (including a literal ``{}``). + +The tests import ``memos.mem_reader.utils`` and ``memos.exceptions`` directly +so they don't need the full ``memos`` import chain (LLM/embedder/reranker +backends) to load. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types + +import pytest + + +# --- minimal package stubs so memos.mem_reader.utils can import memos.log --- +# ``memos.exceptions`` does not depend on the heavy stack, but the test module +# also pulls in ``memos.mem_reader.utils`` which uses ``memos.log.get_logger``. +# We stub the parts we touch to keep this test self-contained. +def _load_memos_submodules(): + pkg_root = types.ModuleType("memos") + pkg_root.__path__ = ["src/memos"] + sys.modules["memos"] = pkg_root + + exc_spec = importlib.util.spec_from_file_location("memos.exceptions", "src/memos/exceptions.py") + exc_mod = importlib.util.module_from_spec(exc_spec) + sys.modules["memos.exceptions"] = exc_mod + exc_spec.loader.exec_module(exc_mod) + + log_mod = types.ModuleType("memos.log") + + class _StubLogger: + def warning(self, *args, **kwargs): + return None + + def error(self, *args, **kwargs): + return None + + def info(self, *args, **kwargs): + return None + + def debug(self, *args, **kwargs): + return None + + def _get_logger(name): + return _StubLogger() + + log_mod.get_logger = _get_logger + sys.modules["memos.log"] = log_mod + + mem_reader_pkg = types.ModuleType("memos.mem_reader") + mem_reader_pkg.__path__ = ["src/memos/mem_reader"] + sys.modules["memos.mem_reader"] = mem_reader_pkg + + util_spec = importlib.util.spec_from_file_location( + "memos.mem_reader.utils", "src/memos/mem_reader/utils.py" + ) + util_mod = importlib.util.module_from_spec(util_spec) + sys.modules["memos.mem_reader.utils"] = util_mod + util_spec.loader.exec_module(util_mod) + + return exc_mod, util_mod + + +_EXC, _UTILS = _load_memos_submodules() +LLMResponseParseError = _EXC.LLMResponseParseError +parse_json_result = _UTILS.parse_json_result + + +class TestParseJsonResultSuccess: + def test_returns_parsed_object(self): + raw = '{"memory list": [{"value": "hi"}], "summary": "s"}' + result = parse_json_result(raw) + assert result == {"memory list": [{"value": "hi"}], "summary": "s"} + + def test_returns_empty_object(self): + raw = "{}" + result = parse_json_result(raw) + assert result == {} + + def test_strips_markdown_fences(self): + raw = '```json\n{"summary": "s"}\n```' + result = parse_json_result(raw) + assert result == {"summary": "s"} + + +class TestParseJsonResultRaises: + def test_trailing_comma_raises(self): + # Issue #2456 reproducer: trailing comma is invalid JSON. + raw = '{"memory list": [{"value": "synthetic evidence"}],}' + with pytest.raises(LLMResponseParseError) as exc_info: + parse_json_result(raw) + assert exc_info.value.__cause__ is not None or "Failed" in str(exc_info.value) + + def test_no_json_object_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("not-valid-json") + + def test_unclosed_fence_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("```json\n{unterminated") + + def test_empty_string_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("") + + def test_none_input_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result(None) + + def test_chatter_only_raises(self): + # Mirrors the symptom-level scenario from #1493 / #2456: the LLM + # emits only prose. The function must not silently return ``{}``. + with pytest.raises(LLMResponseParseError): + parse_json_result("Here is what I extracted: ... unfortunately I cannot output JSON.") + + def test_exception_chain_preserves_root_decode_error(self): + with pytest.raises(LLMResponseParseError) as exc_info: + parse_json_result('{"x": "unterminated') + # The trailing warning message and chained cause let operators + # debug the underlying JSON error from the original LLM payload. + assert exc_info.value.__cause__ is not None diff --git a/tests/mem_reader/test_simple_struct_fallback.py b/tests/mem_reader/test_simple_struct_fallback.py index f4b59d7d5..016413377 100644 --- a/tests/mem_reader/test_simple_struct_fallback.py +++ b/tests/mem_reader/test_simple_struct_fallback.py @@ -83,8 +83,8 @@ def test_process_chat_data_fine_yields_node_when_llm_unparseable(self): item even when the LLM emits non-JSON chatter. """ # Realistic Kimi-K2 style failure: LLM wraps response in chatter so - # `parse_json_result` returns `{}`, `_safe_parse` returns `None`, - # `not response_json` is True and the fallback fires. + # `parse_json_result` raises (issue #2456), `_safe_parse` catches the + # exception and returns `None`, and the fallback fires. self.reader.llm.generate.return_value = ( "Here is what I extracted: ... unfortunately I cannot output JSON." ) diff --git a/tests/mem_reader/test_simple_structure.py b/tests/mem_reader/test_simple_structure.py index 41272c0b2..393d354a6 100644 --- a/tests/mem_reader/test_simple_structure.py +++ b/tests/mem_reader/test_simple_structure.py @@ -112,11 +112,17 @@ def test_parse_json_result_success(self): self.assertIn("summary", result) def test_parse_json_result_failure(self): - """Test failure in JSON parsing.""" - raw_response = "Invalid JSON string" - result = parse_json_result(raw_response) + """Test failure in JSON parsing. - self.assertEqual(result, {}) + As of issue #2456, ``parse_json_result`` no longer swallows malformed + JSON; it raises ``LLMResponseParseError`` so callers can distinguish + "no memories produced" from "couldn't parse the response". + """ + from memos.exceptions import LLMResponseParseError + + raw_response = "Invalid JSON string" + with self.assertRaises(LLMResponseParseError): + parse_json_result(raw_response) def test_get_llm_response_fallback_key_matches_consumer(self): """Regression test for issue #1355. From 4ee17945c75bde27fbca09754344bcf9c06ccdcc Mon Sep 17 00:00:00 2001 From: RerankerGuo <121015044+RerankerGuo@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:04:25 +0800 Subject: [PATCH 2/2] test(mem_reader): import real memos modules instead of sys.modules stubs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test module injected stub packages for memos/memos.log/memos.exceptions/ memos.mem_reader at import time. Those stubs stayed in sys.modules for the whole pytest session and broke every later test that imported the real package: memos.log lost _LOGGING_CONFIGURED_PID/summarize_* helpers, the stub root package shadowed real submodules (memos.chunkers, memos.mem_user), and server_router tests saw 'module memos has no attribute log'. The stub loader also resolved src/... paths against the current working directory, so collecting the file from any other directory raised FileNotFoundError (autotest 'collection' failures). Replace the stub machinery with normal top-level imports — pyproject.toml already sets pythonpath=src and sibling tests import the same modules the same way. Also assert the exception chain directly via exc_info.value.__cause__ being a json.JSONDecodeError, which the current or-clause ('Failed' in message) could never fail. --- .../test_parse_json_result_strict.py | 67 ++----------------- 1 file changed, 5 insertions(+), 62 deletions(-) diff --git a/tests/mem_reader/test_parse_json_result_strict.py b/tests/mem_reader/test_parse_json_result_strict.py index fc86e0423..177051ae2 100644 --- a/tests/mem_reader/test_parse_json_result_strict.py +++ b/tests/mem_reader/test_parse_json_result_strict.py @@ -8,73 +8,16 @@ These tests pin the new contract: ``parse_json_result`` raises ``LLMResponseParseError`` on any non-recoverable parse failure, and only returns the parsed dict on success (including a literal ``{}``). - -The tests import ``memos.mem_reader.utils`` and ``memos.exceptions`` directly -so they don't need the full ``memos`` import chain (LLM/embedder/reranker -backends) to load. """ from __future__ import annotations -import importlib.util -import sys -import types +import json import pytest - -# --- minimal package stubs so memos.mem_reader.utils can import memos.log --- -# ``memos.exceptions`` does not depend on the heavy stack, but the test module -# also pulls in ``memos.mem_reader.utils`` which uses ``memos.log.get_logger``. -# We stub the parts we touch to keep this test self-contained. -def _load_memos_submodules(): - pkg_root = types.ModuleType("memos") - pkg_root.__path__ = ["src/memos"] - sys.modules["memos"] = pkg_root - - exc_spec = importlib.util.spec_from_file_location("memos.exceptions", "src/memos/exceptions.py") - exc_mod = importlib.util.module_from_spec(exc_spec) - sys.modules["memos.exceptions"] = exc_mod - exc_spec.loader.exec_module(exc_mod) - - log_mod = types.ModuleType("memos.log") - - class _StubLogger: - def warning(self, *args, **kwargs): - return None - - def error(self, *args, **kwargs): - return None - - def info(self, *args, **kwargs): - return None - - def debug(self, *args, **kwargs): - return None - - def _get_logger(name): - return _StubLogger() - - log_mod.get_logger = _get_logger - sys.modules["memos.log"] = log_mod - - mem_reader_pkg = types.ModuleType("memos.mem_reader") - mem_reader_pkg.__path__ = ["src/memos/mem_reader"] - sys.modules["memos.mem_reader"] = mem_reader_pkg - - util_spec = importlib.util.spec_from_file_location( - "memos.mem_reader.utils", "src/memos/mem_reader/utils.py" - ) - util_mod = importlib.util.module_from_spec(util_spec) - sys.modules["memos.mem_reader.utils"] = util_mod - util_spec.loader.exec_module(util_mod) - - return exc_mod, util_mod - - -_EXC, _UTILS = _load_memos_submodules() -LLMResponseParseError = _EXC.LLMResponseParseError -parse_json_result = _UTILS.parse_json_result +from memos.exceptions import LLMResponseParseError +from memos.mem_reader.utils import parse_json_result class TestParseJsonResultSuccess: @@ -100,7 +43,7 @@ def test_trailing_comma_raises(self): raw = '{"memory list": [{"value": "synthetic evidence"}],}' with pytest.raises(LLMResponseParseError) as exc_info: parse_json_result(raw) - assert exc_info.value.__cause__ is not None or "Failed" in str(exc_info.value) + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError) def test_no_json_object_raises(self): with pytest.raises(LLMResponseParseError): @@ -129,4 +72,4 @@ def test_exception_chain_preserves_root_decode_error(self): parse_json_result('{"x": "unterminated') # The trailing warning message and chained cause let operators # debug the underlying JSON error from the original LLM payload. - assert exc_info.value.__cause__ is not None + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError)