Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 97 additions & 0 deletions src/memos/mem_reader/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,75 @@ def derive_key(text: str, max_len: int = 80) -> str:
return (sent[:max_len]).strip()


def _strip_trailing_commas(text: str) -> str:
"""Remove commas that sit immediately before ``}`` or ``]`` (ignoring
whitespace in between), but only when the comma is **outside** a quoted
string. Respects backslash-escaped quotes inside strings.

This repairs the single most common malformed-JSON failure mode from
permissive LLM output (e.g. ``[{"value": "x"},]`` or ``{"a": 1,}``) that
historically caused :func:`parse_json_result` to silently return ``{}``
and the ``/product/add`` endpoint to report a success with no stored
memories (issue #2456).

The scanner is deliberately minimal: it does not try to invent missing
values, close unmatched braces, or repair arbitrary malformed output.
Those jobs stay with the ``_cheap_close`` branch and the final ``{}``
return.
"""
out: list[str] = []
in_string = False
escape = False
# Indices into ``out`` of every comma that is still a candidate for
# removal. A ``}`` or ``]`` closer flushes the whole run; any other
# non-whitespace token invalidates them. We track *all* pending commas
# (not just the latest one) so that runs like ``{"a":1,,}`` collapse to
# ``{"a":1}`` in a single pass instead of leaving an earlier comma behind
# and failing the subsequent ``json.loads`` silently (issue #2456 OCR).
pending_comma_indices: list[int] = []
for ch in text:
if in_string:
out.append(ch)
if escape:
escape = False
elif ch == "\\":
escape = True
elif ch == '"':
in_string = False
continue

# ---- outside string ----
if ch == '"':
pending_comma_indices.clear()
in_string = True
out.append(ch)
continue

if ch == ",":
out.append(ch)
pending_comma_indices.append(len(out) - 1)
continue

if ch in (" ", "\t", "\n", "\r"):
# whitespace does not invalidate a pending trailing-comma repair
out.append(ch)
continue

if ch in ("}", "]") and pending_comma_indices:
# Drop every pending comma in the run (handles ``,,}``, ``, ,}``
# etc.); whitespace between the commas and the closer stays.
for idx in pending_comma_indices:
out[idx] = ""
pending_comma_indices.clear()
out.append(ch)
continue

pending_comma_indices.clear()
out.append(ch)

return "".join(out)


def parse_json_result(response_text: str) -> dict:
s = (response_text or "").strip()

Expand Down Expand Up @@ -70,6 +139,34 @@ def _cheap_close(t: str) -> str:
if "Invalid \\escape" in str(e):
s = s.replace("\\", "\\\\")
return json.loads(s)

# Issue #2456: last-chance narrow repair for trailing commas before
# ``}`` / ``]`` (outside quoted strings). Many LLMs — glm, kimi,
# z.ai gateway — occasionally emit ``[...],]`` which is invalid JSON
# and used to be silently dropped as ``{}``, surfacing as an empty
# successful ``/product/add``.
repaired = _strip_trailing_commas(t)
if repaired != t:
try:
result = json.loads(repaired)
logger.warning(
"[JSONParse] Repaired malformed JSON by stripping "
"trailing comma(s) before closer: %s",
e,
)
return result
except json.JSONDecodeError as repair_err:
# The repair modified the text but the result is still not
# valid JSON (e.g. multiple orthogonal defects beyond
# trailing commas). Surface at DEBUG so future
# investigation has a breadcrumb; the outer WARNING below
# still fires with the original error for the operator.
logger.debug(
"[JSONParse] Trailing-comma repair did not fully fix "
"JSON: %s",
repair_err,
)

logger.warning(
f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \
json: {s}"
Expand Down
128 changes: 128 additions & 0 deletions tests/mem_reader/test_parse_json_result.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
"""Regression tests for issue #2456.

A malformed LLM extraction response with a trailing comma before a closing
brace (``[...],]`` or ``{"a": 1,}``) used to be silently converted to ``{}``
by :func:`memos.mem_reader.utils.parse_json_result`, which downstream was
reported as a successful ``/product/add`` with ``data: []`` and no memories
stored.

The parser now attempts a narrow trailing-comma repair (**outside quoted
strings only**) as a last-chance recovery before giving up with ``{}``. These
tests pin that behavior.
"""

import unittest

from memos.mem_reader.utils import parse_json_result


class TestParseJsonResultTrailingComma(unittest.TestCase):
"""Issue #2456: trailing comma from LLM should be repaired, not dropped."""

# --- the exact reproduction from the bug report ------------------------

def test_issue_2456_reproduction(self):
"""Minimal deterministic reproduction from the bug report must now
recover the memory list instead of returning ``{}``."""
valid = '{"memory list": [{"value": "synthetic evidence"}]}'
malformed = '{"memory list": [{"value": "synthetic evidence"}],}'

expected = {"memory list": [{"value": "synthetic evidence"}]}

self.assertEqual(parse_json_result(valid), expected)
# Was ``{}`` before the fix — now the repair branch recovers the
# original payload so ``/product/add`` can persist the memory.
self.assertEqual(parse_json_result(malformed), expected)

# --- trailing comma in various positions -------------------------------

def test_trailing_comma_before_object_close(self):
"""``{"a": 1,}`` -> ``{"a": 1}``."""
self.assertEqual(parse_json_result('{"a": 1,}'), {"a": 1})

def test_trailing_comma_before_array_close(self):
"""``[1, 2, 3,]`` -> ``[1, 2, 3]`` when wrapped in an object."""
self.assertEqual(
parse_json_result('{"xs": [1, 2, 3,]}'),
{"xs": [1, 2, 3]},
)

def test_trailing_comma_before_newline_and_close(self):
"""Whitespace / newlines between the comma and the closer must not
defeat the repair."""
raw = '{"xs": [\n 1,\n 2,\n],\n}'
self.assertEqual(parse_json_result(raw), {"xs": [1, 2]})

def test_trailing_comma_inside_fenced_code_block(self):
"""``parse_json_result`` already strips ``` fences; the repair must
still fire on the inner payload."""
raw = '```json\n{"a": 1,}\n```'
self.assertEqual(parse_json_result(raw), {"a": 1})

# --- safety: do not touch commas that are legitimate -------------------

def test_comma_inside_string_is_not_stripped(self):
"""A literal ``,}`` *inside* a quoted string must not be touched."""
# The value contains ``foo,}`` as literal data. Repair would corrupt
# the string if the state machine were naive.
raw = '{"msg": "foo,}"}'
self.assertEqual(parse_json_result(raw), {"msg": "foo,}"})

def test_escaped_quote_inside_string_does_not_confuse_scanner(self):
"""``\\"`` inside a string must keep the scanner in-string so that
a trailing comma *outside* the string is still repaired correctly
and commas *inside* the string are preserved."""
# The outer object has a real trailing comma (triggers repair);
# the escaped quotes inside the value must not confuse the scanner
# into leaving in-string and treating one of the inner commas as
# the one to remove. Previously the raw string had the trailing
# ``,}`` *inside* the quoted value, so ``json.loads`` succeeded on
# the first try and the repair branch was never exercised.
raw = '{"msg": "he said \\"hi,\\" then left",}'
self.assertEqual(
parse_json_result(raw),
{"msg": 'he said "hi," then left'},
)

def test_double_trailing_comma_is_fully_repaired(self):
"""OCR follow-up to #2456: a run like ``{"a": 1,,}`` must collapse
to ``{"a": 1}``. Earlier revision only removed the *latest*
pending comma, leaving ``{"a": 1,}`` which then failed
``json.loads`` silently and surfaced as an empty ``/product/add``
response."""
self.assertEqual(parse_json_result('{"a": 1,,}'), {"a": 1})
self.assertEqual(
parse_json_result('{"xs": [1, 2,,,]}'),
{"xs": [1, 2]},
)

def test_valid_json_is_unchanged(self):
"""Valid JSON must round-trip unchanged — no accidental repair."""
raw = '{"a": [1, 2, 3], "b": {"c": "d"}}'
self.assertEqual(
parse_json_result(raw),
{"a": [1, 2, 3], "b": {"c": "d"}},
)

# --- hard-failure contract preserved ------------------------------------

def test_genuinely_malformed_still_returns_empty_dict(self):
"""When the trailing-comma repair cannot help (missing value,
unmatched braces that even ``_cheap_close`` cannot resolve, etc.),
the historical ``{}`` return contract must be preserved so the 15+
callers do not need to change."""
# Missing value for key ``a`` — not a trailing-comma problem.
self.assertEqual(parse_json_result('{"a":}'), {})

def test_non_json_text_still_returns_empty_dict(self):
"""Pure chatter with no JSON must still return ``{}`` — pins the
existing ``test_parse_json_result_failure`` contract."""
self.assertEqual(parse_json_result("this is not json at all"), {})

def test_empty_input_returns_empty_dict(self):
self.assertEqual(parse_json_result(""), {})
self.assertEqual(parse_json_result(None), {}) # type: ignore[arg-type]


if __name__ == "__main__":
unittest.main()
Loading