Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/memos/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,12 @@ class EmbedderError(MemOSError): ...


class ParserError(MemOSError): ...


class LLMResponseParseError(ParserError):
"""Raised when an LLM response cannot be parsed into the expected shape.

Used in place of a silent ``{}`` return so that callers can distinguish
"the model produced no memories" from "the model produced output we
couldn't parse". Issue #2456.
"""
53 changes: 37 additions & 16 deletions src/memos/mem_reader/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import re

from memos import log
from memos.exceptions import LLMResponseParseError


logger = log.get_logger(__name__)
Expand Down Expand Up @@ -36,27 +37,41 @@ def derive_key(text: str, max_len: int = 80) -> str:


def parse_json_result(response_text: str) -> dict:
s = (response_text or "").strip()
"""Parse a JSON object out of an LLM response.

m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.I)
Returns the parsed ``dict`` when the response contains a parseable JSON
object (including a literal ``{}``). Raises ``LLMResponseParseError``
on any other input so callers can distinguish "the model produced no
memories" from "the model produced output we couldn't parse". Issue
#2456.
"""
raw = response_text or ""
s = raw.strip()

if not s:
raise LLMResponseParseError("LLM response was empty")

m = re.search(r"```(?:json)?\s*([\s\S]*?)```", s, flags=re.IGNORECASE)
s = (m.group(1) if m else s.replace("```", "")).strip()

i = s.find("{")
if i == -1:
return {}
raise LLMResponseParseError(f"No JSON object found in LLM response: tail={raw[-200:]!r}")
s = s[i:].strip()

last_err: json.JSONDecodeError | None = None

try:
return json.loads(s)
except json.JSONDecodeError:
pass
except json.JSONDecodeError as e:
last_err = e

j = max(s.rfind("}"), s.rfind("]"))
if j != -1:
try:
return json.loads(s[: j + 1])
except json.JSONDecodeError:
pass
except json.JSONDecodeError as e:
last_err = e

def _cheap_close(t: str) -> str:
t += "}" * max(0, t.count("{") - t.count("}"))
Expand All @@ -67,14 +82,20 @@ def _cheap_close(t: str) -> str:
try:
return json.loads(t)
except json.JSONDecodeError as e:
last_err = e
if "Invalid \\escape" in str(e):
s = s.replace("\\", "\\\\")
return json.loads(s)
logger.warning(
f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \
json: {s}"
)
return {}
try:
return json.loads(s.replace("\\", "\\\\"))
except json.JSONDecodeError as e2:
last_err = e2

logger.warning(
f"[JSONParse] Failed to decode JSON: {last_err}\nTail: Raw {response_text} \
json: {s}"
)
raise LLMResponseParseError(
f"Failed to decode JSON from LLM response: {last_err}"
) from last_err


def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]:
Expand All @@ -83,7 +104,7 @@ def parse_rewritten_response(text: str) -> tuple[bool, dict[int, dict]]:
Returns (success, parsed_dict) with int keys.
"""
try:
m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I)
m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE)
s = (m.group(1) if m else text).strip()
data = json.loads(s)
except Exception:
Expand Down Expand Up @@ -127,7 +148,7 @@ def parse_keep_filter_response(text: str) -> tuple[bool, dict[int, dict]]:
Returns (success, parsed_dict) with int keys.
"""
try:
m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.I)
m = re.search(r"```(?:json)?\s*([\s\S]*?)```", text, flags=re.IGNORECASE)
s = (m.group(1) if m else text).strip()
data = json.loads(s)
except Exception:
Expand Down
75 changes: 75 additions & 0 deletions tests/mem_reader/test_parse_json_result_strict.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
"""Regression tests for issue #2456.

``parse_json_result`` previously swallowed every JSON-decode failure and
returned ``{}``, so a malformed LLM extraction was indistinguishable from a
successful empty extraction. Downstream callers saw ``{"code": 200, "data": []}``
for ``/product/add`` requests that actually failed.

These tests pin the new contract: ``parse_json_result`` raises
``LLMResponseParseError`` on any non-recoverable parse failure, and only
returns the parsed dict on success (including a literal ``{}``).
"""

from __future__ import annotations

import json

import pytest

from memos.exceptions import LLMResponseParseError
from memos.mem_reader.utils import parse_json_result


class TestParseJsonResultSuccess:
def test_returns_parsed_object(self):
raw = '{"memory list": [{"value": "hi"}], "summary": "s"}'
result = parse_json_result(raw)
assert result == {"memory list": [{"value": "hi"}], "summary": "s"}

def test_returns_empty_object(self):
raw = "{}"
result = parse_json_result(raw)
assert result == {}

def test_strips_markdown_fences(self):
raw = '```json\n{"summary": "s"}\n```'
result = parse_json_result(raw)
assert result == {"summary": "s"}


class TestParseJsonResultRaises:
def test_trailing_comma_raises(self):
# Issue #2456 reproducer: trailing comma is invalid JSON.
raw = '{"memory list": [{"value": "synthetic evidence"}],}'
with pytest.raises(LLMResponseParseError) as exc_info:
parse_json_result(raw)
assert isinstance(exc_info.value.__cause__, json.JSONDecodeError)

def test_no_json_object_raises(self):
with pytest.raises(LLMResponseParseError):
parse_json_result("not-valid-json")

def test_unclosed_fence_raises(self):
with pytest.raises(LLMResponseParseError):
parse_json_result("```json\n{unterminated")

def test_empty_string_raises(self):
with pytest.raises(LLMResponseParseError):
parse_json_result("")

def test_none_input_raises(self):
with pytest.raises(LLMResponseParseError):
parse_json_result(None)

def test_chatter_only_raises(self):
# Mirrors the symptom-level scenario from #1493 / #2456: the LLM
# emits only prose. The function must not silently return ``{}``.
with pytest.raises(LLMResponseParseError):
parse_json_result("Here is what I extracted: ... unfortunately I cannot output JSON.")

def test_exception_chain_preserves_root_decode_error(self):
with pytest.raises(LLMResponseParseError) as exc_info:
parse_json_result('{"x": "unterminated')
# The trailing warning message and chained cause let operators
# debug the underlying JSON error from the original LLM payload.
assert isinstance(exc_info.value.__cause__, json.JSONDecodeError)
4 changes: 2 additions & 2 deletions tests/mem_reader/test_simple_struct_fallback.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ def test_process_chat_data_fine_yields_node_when_llm_unparseable(self):
item even when the LLM emits non-JSON chatter.
"""
# Realistic Kimi-K2 style failure: LLM wraps response in chatter so
# `parse_json_result` returns `{}`, `_safe_parse` returns `None`,
# `not response_json` is True and the fallback fires.
# `parse_json_result` raises (issue #2456), `_safe_parse` catches the
# exception and returns `None`, and the fallback fires.
self.reader.llm.generate.return_value = (
"Here is what I extracted: ... unfortunately I cannot output JSON."
)
Expand Down
14 changes: 10 additions & 4 deletions tests/mem_reader/test_simple_structure.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,17 @@ def test_parse_json_result_success(self):
self.assertIn("summary", result)

def test_parse_json_result_failure(self):
"""Test failure in JSON parsing."""
raw_response = "Invalid JSON string"
result = parse_json_result(raw_response)
"""Test failure in JSON parsing.

self.assertEqual(result, {})
As of issue #2456, ``parse_json_result`` no longer swallows malformed
JSON; it raises ``LLMResponseParseError`` so callers can distinguish
"no memories produced" from "couldn't parse the response".
"""
from memos.exceptions import LLMResponseParseError

raw_response = "Invalid JSON string"
with self.assertRaises(LLMResponseParseError):
parse_json_result(raw_response)

def test_get_llm_response_fallback_key_matches_consumer(self):
"""Regression test for issue #1355.
Expand Down
Loading