diff --git a/src/memos/exceptions.py b/src/memos/exceptions.py index 511e68d8f..e9aeffd5c 100644 --- a/src/memos/exceptions.py +++ b/src/memos/exceptions.py @@ -41,3 +41,12 @@ class EmbedderError(MemOSError): ... class ParserError(MemOSError): ... + + +class LLMResponseParseError(ParserError): + """Raised when an LLM response cannot be parsed into the expected shape. + + Used in place of a silent ``{}`` return so that callers can distinguish + "the model produced no memories" from "the model produced output we + couldn't parse". Issue #2456. + """ diff --git a/src/memos/mem_reader/utils.py b/src/memos/mem_reader/utils.py index c08e8a4ed..a1767d5b1 100644 --- a/src/memos/mem_reader/utils.py +++ b/src/memos/mem_reader/utils.py @@ -2,6 +2,7 @@ import re from memos import log +from memos.exceptions import LLMResponseParseError logger = log.get_logger(__name__) @@ -36,27 +37,41 @@ def derive_key(text: str, max_len: int = 80) -> str: def parse_json_result(response_text: str) -> dict: - s = (response_text or "").strip() + """Parse a JSON object out of an LLM response. - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.I) + Returns the parsed ``dict`` when the response contains a parseable JSON + object (including a literal ``{}``). Raises ``LLMResponseParseError`` + on any other input so callers can distinguish "the model produced no + memories" from "the model produced output we couldn't parse". Issue + #2456. + """ + raw = response_text or "" + s = raw.strip() + + if not s: + raise LLMResponseParseError("LLM response was empty") + + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.IGNORECASE) s = (m.group(1) if m else s.replace("```", "")).strip() i = s.find("{") if i == -1: - return {} + raise LLMResponseParseError(f"No JSON object found in LLM response: tail={raw[-200:]!r}") s = s[i:].strip() + last_err: json.JSONDecodeError | None = None + try: return json.loads(s) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + last_err = e j = max(s.rfind("}"), s.rfind("]")) if j != -1: try: return json.loads(s[: j + 1]) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + last_err = e def _cheap_close(t: str) -> str: t += "}" * max(0, t.count("{") - t.count("}")) @@ -67,14 +82,20 @@ def _cheap_close(t: str) -> str: try: return json.loads(t) except json.JSONDecodeError as e: + last_err = e if "Invalid \\escape" in str(e): - s = s.replace("\\", "\\\\") - return json.loads(s) - logger.warning( - f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \ - json: {s}" - ) - return {} + try: + return json.loads(s.replace("\\", "\\\\")) + except json.JSONDecodeError as e2: + last_err = e2 + + logger.warning( + f"[JSONParse] Failed to decode JSON: {last_err}\nTail: Raw {response_text} \ + json: {s}" + ) + raise LLMResponseParseError( + f"Failed to decode JSON from LLM response: {last_err}" + ) from last_err def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]: @@ -83,7 +104,7 @@ def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]: Returns (success, parsed_dict) with int keys. """ try: - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I) + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE) s = (m.group(1) if m else text).strip() data = json.loads(s) except Exception: @@ -127,7 +148,7 @@ def parse_keep_filter_response(text: str) -> tuple[bool, dict[int, dict]]: Returns (success, parsed_dict) with int keys. """ try: - m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I) + m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE) s = (m.group(1) if m else text).strip() data = json.loads(s) except Exception: diff --git a/tests/mem_reader/test_parse_json_result_strict.py b/tests/mem_reader/test_parse_json_result_strict.py new file mode 100644 index 000000000..177051ae2 --- /dev/null +++ b/tests/mem_reader/test_parse_json_result_strict.py @@ -0,0 +1,75 @@ +"""Regression tests for issue #2456. + +``parse_json_result`` previously swallowed every JSON-decode failure and +returned ``{}``, so a malformed LLM extraction was indistinguishable from a +successful empty extraction. Downstream callers saw ``{"code": 200, "data": []}`` +for ``/product/add`` requests that actually failed. + +These tests pin the new contract: ``parse_json_result`` raises +``LLMResponseParseError`` on any non-recoverable parse failure, and only +returns the parsed dict on success (including a literal ``{}``). +""" + +from __future__ import annotations + +import json + +import pytest + +from memos.exceptions import LLMResponseParseError +from memos.mem_reader.utils import parse_json_result + + +class TestParseJsonResultSuccess: + def test_returns_parsed_object(self): + raw = '{"memory list": [{"value": "hi"}], "summary": "s"}' + result = parse_json_result(raw) + assert result == {"memory list": [{"value": "hi"}], "summary": "s"} + + def test_returns_empty_object(self): + raw = "{}" + result = parse_json_result(raw) + assert result == {} + + def test_strips_markdown_fences(self): + raw = '```json\n{"summary": "s"}\n```' + result = parse_json_result(raw) + assert result == {"summary": "s"} + + +class TestParseJsonResultRaises: + def test_trailing_comma_raises(self): + # Issue #2456 reproducer: trailing comma is invalid JSON. + raw = '{"memory list": [{"value": "synthetic evidence"}],}' + with pytest.raises(LLMResponseParseError) as exc_info: + parse_json_result(raw) + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError) + + def test_no_json_object_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("not-valid-json") + + def test_unclosed_fence_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("```json\n{unterminated") + + def test_empty_string_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result("") + + def test_none_input_raises(self): + with pytest.raises(LLMResponseParseError): + parse_json_result(None) + + def test_chatter_only_raises(self): + # Mirrors the symptom-level scenario from #1493 / #2456: the LLM + # emits only prose. The function must not silently return ``{}``. + with pytest.raises(LLMResponseParseError): + parse_json_result("Here is what I extracted: ... unfortunately I cannot output JSON.") + + def test_exception_chain_preserves_root_decode_error(self): + with pytest.raises(LLMResponseParseError) as exc_info: + parse_json_result('{"x": "unterminated') + # The trailing warning message and chained cause let operators + # debug the underlying JSON error from the original LLM payload. + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError) diff --git a/tests/mem_reader/test_simple_struct_fallback.py b/tests/mem_reader/test_simple_struct_fallback.py index f4b59d7d5..016413377 100644 --- a/tests/mem_reader/test_simple_struct_fallback.py +++ b/tests/mem_reader/test_simple_struct_fallback.py @@ -83,8 +83,8 @@ def test_process_chat_data_fine_yields_node_when_llm_unparseable(self): item even when the LLM emits non-JSON chatter. """ # Realistic Kimi-K2 style failure: LLM wraps response in chatter so - # `parse_json_result` returns `{}`, `_safe_parse` returns `None`, - # `not response_json` is True and the fallback fires. + # `parse_json_result` raises (issue #2456), `_safe_parse` catches the + # exception and returns `None`, and the fallback fires. self.reader.llm.generate.return_value = ( "Here is what I extracted: ... unfortunately I cannot output JSON." ) diff --git a/tests/mem_reader/test_simple_structure.py b/tests/mem_reader/test_simple_structure.py index 41272c0b2..393d354a6 100644 --- a/tests/mem_reader/test_simple_structure.py +++ b/tests/mem_reader/test_simple_structure.py @@ -112,11 +112,17 @@ def test_parse_json_result_success(self): self.assertIn("summary", result) def test_parse_json_result_failure(self): - """Test failure in JSON parsing.""" - raw_response = "Invalid JSON string" - result = parse_json_result(raw_response) + """Test failure in JSON parsing. - self.assertEqual(result, {}) + As of issue #2456, ``parse_json_result`` no longer swallows malformed + JSON; it raises ``LLMResponseParseError`` so callers can distinguish + "no memories produced" from "couldn't parse the response". + """ + from memos.exceptions import LLMResponseParseError + + raw_response = "Invalid JSON string" + with self.assertRaises(LLMResponseParseError): + parse_json_result(raw_response) def test_get_llm_response_fallback_key_matches_consumer(self): """Regression test for issue #1355.