From 2ea35dfbe66d620e9a0753dc74b4d022ac52d0e3 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Sat, 3 Oct 2026 21:50:47 +0800 Subject: [PATCH 1/2] fix: repair trailing comma in malformed extraction JSON (#2456) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parse_json_result` previously returned `{}` when the LLM emitted a trailing comma before `}` or `]` (e.g. `[{"value": "x"},]`). The empty dict propagated through `multi_modal_struct._process_string_fine` — whose exception-based fallback never fired because no exception was raised — and surfaced as `POST /product/add` returning `{"code":200, "message":"Memory added successfully", "data": []}` with no memories stored. Related to #1355 / fixed-by #1977, but a distinct failure path: the SimpleStructMemReader fallback-key fix does not cover this case. Add a narrow last-chance repair that strips commas sitting immediately before `}` or `]` **outside** quoted strings (respecting backslash-escaped quotes), then reparses with `json.loads`. On success emit a `[JSONParse] Repaired malformed JSON...` warning so the previously-silent failure mode is observable. On repair failure keep returning `{}` to preserve the historical contract for the 15 call sites. Regression tests in `tests/mem_reader/test_parse_json_result.py` cover the exact reproduction from the bug report plus safety cases (literal `,}` inside strings, escaped quotes, genuinely-malformed JSON still returns `{}`). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/memos/mem_reader/utils.py | 80 +++++++++++++++ tests/mem_reader/test_parse_json_result.py | 109 +++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 tests/mem_reader/test_parse_json_result.py diff --git a/src/memos/mem_reader/utils.py b/src/memos/mem_reader/utils.py index c08e8a4ed..26190b995 100644 --- a/src/memos/mem_reader/utils.py +++ b/src/memos/mem_reader/utils.py @@ -35,6 +35,67 @@ def derive_key(text: str, max_len: int = 80) -> str: return (sent[:max_len]).strip() +def _strip_trailing_commas(text: str) -> str: + """Remove commas that sit immediately before ``}`` or ``]`` (ignoring + whitespace in between), but only when the comma is **outside** a quoted + string. Respects backslash-escaped quotes inside strings. + + This repairs the single most common malformed-JSON failure mode from + permissive LLM output (e.g. ``[{"value": "x"},]`` or ``{"a": 1,}``) that + historically caused :func:`parse_json_result` to silently return ``{}`` + and the ``/product/add`` endpoint to report a success with no stored + memories (issue #2456). + + The scanner is deliberately minimal: it does not try to invent missing + values, close unmatched braces, or repair arbitrary malformed output. + Those jobs stay with the ``_cheap_close`` branch and the final ``{}`` + return. + """ + out: list[str] = [] + in_string = False + escape = False + pending_comma_idx: int | None = None # index into ``out`` of a comma eligible for removal + for ch in text: + if in_string: + out.append(ch) + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + + # ---- outside string ---- + if ch == '"': + pending_comma_idx = None + in_string = True + out.append(ch) + continue + + if ch == ",": + out.append(ch) + pending_comma_idx = len(out) - 1 + continue + + if ch in (" ", "\t", "\n", "\r"): + # whitespace does not invalidate a pending trailing-comma repair + out.append(ch) + continue + + if ch in ("}", "]") and pending_comma_idx is not None: + # Drop the pending comma; whitespace between comma and closer stays. + out[pending_comma_idx] = "" + pending_comma_idx = None + out.append(ch) + continue + + pending_comma_idx = None + out.append(ch) + + return "".join(out) + + def parse_json_result(response_text: str) -> dict: s = (response_text or "").strip() @@ -70,6 +131,25 @@ def _cheap_close(t: str) -> str: if "Invalid \\escape" in str(e): s = s.replace("\\", "\\\\") return json.loads(s) + + # Issue #2456: last-chance narrow repair for trailing commas before + # ``}`` / ``]`` (outside quoted strings). Many LLMs — glm, kimi, + # z.ai gateway — occasionally emit ``[...],]`` which is invalid JSON + # and used to be silently dropped as ``{}``, surfacing as an empty + # successful ``/product/add``. + repaired = _strip_trailing_commas(t) + if repaired != t: + try: + result = json.loads(repaired) + logger.warning( + "[JSONParse] Repaired malformed JSON by stripping " + "trailing comma(s) before closer: %s", + e, + ) + return result + except json.JSONDecodeError: + pass + logger.warning( f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \ json: {s}" diff --git a/tests/mem_reader/test_parse_json_result.py b/tests/mem_reader/test_parse_json_result.py new file mode 100644 index 000000000..940da3dac --- /dev/null +++ b/tests/mem_reader/test_parse_json_result.py @@ -0,0 +1,109 @@ +"""Regression tests for issue #2456. + +A malformed LLM extraction response with a trailing comma before a closing +brace (``[...],]`` or ``{"a": 1,}``) used to be silently converted to ``{}`` +by :func:`memos.mem_reader.utils.parse_json_result`, which downstream was +reported as a successful ``/product/add`` with ``data: []`` and no memories +stored. + +The parser now attempts a narrow trailing-comma repair (**outside quoted +strings only**) as a last-chance recovery before giving up with ``{}``. These +tests pin that behavior. +""" + +import unittest + +from memos.mem_reader.utils import parse_json_result + + +class TestParseJsonResultTrailingComma(unittest.TestCase): + """Issue #2456: trailing comma from LLM should be repaired, not dropped.""" + + # --- the exact reproduction from the bug report ------------------------ + + def test_issue_2456_reproduction(self): + """Minimal deterministic reproduction from the bug report must now + recover the memory list instead of returning ``{}``.""" + valid = '{"memory list": [{"value": "synthetic evidence"}]}' + malformed = '{"memory list": [{"value": "synthetic evidence"}],}' + + expected = {"memory list": [{"value": "synthetic evidence"}]} + + self.assertEqual(parse_json_result(valid), expected) + # Was ``{}`` before the fix — now the repair branch recovers the + # original payload so ``/product/add`` can persist the memory. + self.assertEqual(parse_json_result(malformed), expected) + + # --- trailing comma in various positions ------------------------------- + + def test_trailing_comma_before_object_close(self): + """``{"a": 1,}`` -> ``{"a": 1}``.""" + self.assertEqual(parse_json_result('{"a": 1,}'), {"a": 1}) + + def test_trailing_comma_before_array_close(self): + """``[1, 2, 3,]`` -> ``[1, 2, 3]`` when wrapped in an object.""" + self.assertEqual( + parse_json_result('{"xs": [1, 2, 3,]}'), + {"xs": [1, 2, 3]}, + ) + + def test_trailing_comma_before_newline_and_close(self): + """Whitespace / newlines between the comma and the closer must not + defeat the repair.""" + raw = '{"xs": [\n 1,\n 2,\n],\n}' + self.assertEqual(parse_json_result(raw), {"xs": [1, 2]}) + + def test_trailing_comma_inside_fenced_code_block(self): + """``parse_json_result`` already strips ``` fences; the repair must + still fire on the inner payload.""" + raw = '```json\n{"a": 1,}\n```' + self.assertEqual(parse_json_result(raw), {"a": 1}) + + # --- safety: do not touch commas that are legitimate ------------------- + + def test_comma_inside_string_is_not_stripped(self): + """A literal ``,}`` *inside* a quoted string must not be touched.""" + # The value contains ``foo,}`` as literal data. Repair would corrupt + # the string if the state machine were naive. + raw = '{"msg": "foo,}"}' + self.assertEqual(parse_json_result(raw), {"msg": "foo,}"}) + + def test_escaped_quote_inside_string_does_not_confuse_scanner(self): + """``\\"`` inside a string must keep the scanner in-string, so a + trailing comma that follows (still inside the string) stays.""" + raw = '{"msg": "he said \\"hi,\\" then left,}"}' + self.assertEqual( + parse_json_result(raw), + {"msg": 'he said "hi," then left,}'}, + ) + + def test_valid_json_is_unchanged(self): + """Valid JSON must round-trip unchanged — no accidental repair.""" + raw = '{"a": [1, 2, 3], "b": {"c": "d"}}' + self.assertEqual( + parse_json_result(raw), + {"a": [1, 2, 3], "b": {"c": "d"}}, + ) + + # --- hard-failure contract preserved ------------------------------------ + + def test_genuinely_malformed_still_returns_empty_dict(self): + """When the trailing-comma repair cannot help (missing value, + unmatched braces that even ``_cheap_close`` cannot resolve, etc.), + the historical ``{}`` return contract must be preserved so the 15+ + callers do not need to change.""" + # Missing value for key ``a`` — not a trailing-comma problem. + self.assertEqual(parse_json_result('{"a":}'), {}) + + def test_non_json_text_still_returns_empty_dict(self): + """Pure chatter with no JSON must still return ``{}`` — pins the + existing ``test_parse_json_result_failure`` contract.""" + self.assertEqual(parse_json_result("this is not json at all"), {}) + + def test_empty_input_returns_empty_dict(self): + self.assertEqual(parse_json_result(""), {}) + self.assertEqual(parse_json_result(None), {}) # type: ignore[arg-type] + + +if __name__ == "__main__": + unittest.main() From 131b5a39fbdbae764aaa76adc8ad5dba56e83925 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Sat, 3 Oct 2026 22:08:45 +0800 Subject: [PATCH 2/2] fix(mem_reader): address OCR follow-ups on #2456 trailing-comma repair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-up issues from the Open Code Review on PR #2457: 1. tests/mem_reader/test_parse_json_result.py — the escaped-quote regression actually had its trailing ``,}`` *inside* the quoted value, so the raw string was already valid JSON and ``_strip_trailing_commas`` was never exercised. Move the trailing comma outside the string and assert the inner commas (and the escaped quotes around them) survive the repair. Also add a positive case for ``,,}`` runs so the fix below is pinned. 2. src/memos/mem_reader/utils.py — ``_strip_trailing_commas`` only remembered the *latest* candidate comma, so ``{"a":1,,}`` was repaired to ``{"a":1,}`` which still fails ``json.loads``. The outer ``logger.warning`` only fires when ``repaired == t``, so the silent fallthrough re-created the exact failure mode (#2456) the repair was meant to kill. Track *all* pending comma indices and flush the whole run on a closer; a string open or any other non-whitespace token clears them. 3. src/memos/mem_reader/utils.py — when the repair modified the text but the result still failed ``json.loads``, the inner except silently swallowed the second error with no diagnostic. Emit a ``logger.debug`` with the repair-attempt error so future debugging has a breadcrumb; the outer WARNING still fires with the original decode error for operator visibility. Verified: ``PYTHONPATH=src pytest tests/mem_reader/test_parse_json_result.py`` 12 passed (adds 1 new case). The 2 pre-existing failures in ``tests/mem_reader/test_coarse_memory_type.py`` are unrelated (missing optional ``markitdown`` dep). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/memos/mem_reader/utils.py | 37 ++++++++++++++++------ tests/mem_reader/test_parse_json_result.py | 27 +++++++++++++--- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/memos/mem_reader/utils.py b/src/memos/mem_reader/utils.py index 26190b995..bf1de069a 100644 --- a/src/memos/mem_reader/utils.py +++ b/src/memos/mem_reader/utils.py @@ -54,7 +54,13 @@ def _strip_trailing_commas(text: str) -> str: out: list[str] = [] in_string = False escape = False - pending_comma_idx: int | None = None # index into ``out`` of a comma eligible for removal + # Indices into ``out`` of every comma that is still a candidate for + # removal. A ``}`` or ``]`` closer flushes the whole run; any other + # non-whitespace token invalidates them. We track *all* pending commas + # (not just the latest one) so that runs like ``{"a":1,,}`` collapse to + # ``{"a":1}`` in a single pass instead of leaving an earlier comma behind + # and failing the subsequent ``json.loads`` silently (issue #2456 OCR). + pending_comma_indices: list[int] = [] for ch in text: if in_string: out.append(ch) @@ -68,14 +74,14 @@ def _strip_trailing_commas(text: str) -> str: # ---- outside string ---- if ch == '"': - pending_comma_idx = None + pending_comma_indices.clear() in_string = True out.append(ch) continue if ch == ",": out.append(ch) - pending_comma_idx = len(out) - 1 + pending_comma_indices.append(len(out) - 1) continue if ch in (" ", "\t", "\n", "\r"): @@ -83,14 +89,16 @@ def _strip_trailing_commas(text: str) -> str: out.append(ch) continue - if ch in ("}", "]") and pending_comma_idx is not None: - # Drop the pending comma; whitespace between comma and closer stays. - out[pending_comma_idx] = "" - pending_comma_idx = None + if ch in ("}", "]") and pending_comma_indices: + # Drop every pending comma in the run (handles ``,,}``, ``, ,}`` + # etc.); whitespace between the commas and the closer stays. + for idx in pending_comma_indices: + out[idx] = "" + pending_comma_indices.clear() out.append(ch) continue - pending_comma_idx = None + pending_comma_indices.clear() out.append(ch) return "".join(out) @@ -147,8 +155,17 @@ def _cheap_close(t: str) -> str: e, ) return result - except json.JSONDecodeError: - pass + except json.JSONDecodeError as repair_err: + # The repair modified the text but the result is still not + # valid JSON (e.g. multiple orthogonal defects beyond + # trailing commas). Surface at DEBUG so future + # investigation has a breadcrumb; the outer WARNING below + # still fires with the original error for the operator. + logger.debug( + "[JSONParse] Trailing-comma repair did not fully fix " + "JSON: %s", + repair_err, + ) logger.warning( f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \ diff --git a/tests/mem_reader/test_parse_json_result.py b/tests/mem_reader/test_parse_json_result.py index 940da3dac..61dfeabf6 100644 --- a/tests/mem_reader/test_parse_json_result.py +++ b/tests/mem_reader/test_parse_json_result.py @@ -69,12 +69,31 @@ def test_comma_inside_string_is_not_stripped(self): self.assertEqual(parse_json_result(raw), {"msg": "foo,}"}) def test_escaped_quote_inside_string_does_not_confuse_scanner(self): - """``\\"`` inside a string must keep the scanner in-string, so a - trailing comma that follows (still inside the string) stays.""" - raw = '{"msg": "he said \\"hi,\\" then left,}"}' + """``\\"`` inside a string must keep the scanner in-string so that + a trailing comma *outside* the string is still repaired correctly + and commas *inside* the string are preserved.""" + # The outer object has a real trailing comma (triggers repair); + # the escaped quotes inside the value must not confuse the scanner + # into leaving in-string and treating one of the inner commas as + # the one to remove. Previously the raw string had the trailing + # ``,}`` *inside* the quoted value, so ``json.loads`` succeeded on + # the first try and the repair branch was never exercised. + raw = '{"msg": "he said \\"hi,\\" then left",}' self.assertEqual( parse_json_result(raw), - {"msg": 'he said "hi," then left,}'}, + {"msg": 'he said "hi," then left'}, + ) + + def test_double_trailing_comma_is_fully_repaired(self): + """OCR follow-up to #2456: a run like ``{"a": 1,,}`` must collapse + to ``{"a": 1}``. Earlier revision only removed the *latest* + pending comma, leaving ``{"a": 1,}`` which then failed + ``json.loads`` silently and surfaced as an empty ``/product/add`` + response.""" + self.assertEqual(parse_json_result('{"a": 1,,}'), {"a": 1}) + self.assertEqual( + parse_json_result('{"xs": [1, 2,,,]}'), + {"xs": [1, 2]}, ) def test_valid_json_is_unchanged(self):