diff --git a/src/memos/mem_reader/utils.py b/src/memos/mem_reader/utils.py index c08e8a4ed..bf1de069a 100644 --- a/src/memos/mem_reader/utils.py +++ b/src/memos/mem_reader/utils.py @@ -35,6 +35,75 @@ def derive_key(text: str, max_len: int = 80) -> str: return (sent[:max_len]).strip() +def _strip_trailing_commas(text: str) -> str: + """Remove commas that sit immediately before ``}`` or ``]`` (ignoring + whitespace in between), but only when the comma is **outside** a quoted + string. Respects backslash-escaped quotes inside strings. + + This repairs the single most common malformed-JSON failure mode from + permissive LLM output (e.g. ``[{"value": "x"},]`` or ``{"a": 1,}``) that + historically caused :func:`parse_json_result` to silently return ``{}`` + and the ``/product/add`` endpoint to report a success with no stored + memories (issue #2456). + + The scanner is deliberately minimal: it does not try to invent missing + values, close unmatched braces, or repair arbitrary malformed output. + Those jobs stay with the ``_cheap_close`` branch and the final ``{}`` + return. + """ + out: list[str] = [] + in_string = False + escape = False + # Indices into ``out`` of every comma that is still a candidate for + # removal. A ``}`` or ``]`` closer flushes the whole run; any other + # non-whitespace token invalidates them. We track *all* pending commas + # (not just the latest one) so that runs like ``{"a":1,,}`` collapse to + # ``{"a":1}`` in a single pass instead of leaving an earlier comma behind + # and failing the subsequent ``json.loads`` silently (issue #2456 OCR). + pending_comma_indices: list[int] = [] + for ch in text: + if in_string: + out.append(ch) + if escape: + escape = False + elif ch == "\\": + escape = True + elif ch == '"': + in_string = False + continue + + # ---- outside string ---- + if ch == '"': + pending_comma_indices.clear() + in_string = True + out.append(ch) + continue + + if ch == ",": + out.append(ch) + pending_comma_indices.append(len(out) - 1) + continue + + if ch in (" ", "\t", "\n", "\r"): + # whitespace does not invalidate a pending trailing-comma repair + out.append(ch) + continue + + if ch in ("}", "]") and pending_comma_indices: + # Drop every pending comma in the run (handles ``,,}``, ``, ,}`` + # etc.); whitespace between the commas and the closer stays. + for idx in pending_comma_indices: + out[idx] = "" + pending_comma_indices.clear() + out.append(ch) + continue + + pending_comma_indices.clear() + out.append(ch) + + return "".join(out) + + def parse_json_result(response_text: str) -> dict: s = (response_text or "").strip() @@ -70,6 +139,34 @@ def _cheap_close(t: str) -> str: if "Invalid \\escape" in str(e): s = s.replace("\\", "\\\\") return json.loads(s) + + # Issue #2456: last-chance narrow repair for trailing commas before + # ``}`` / ``]`` (outside quoted strings). Many LLMs — glm, kimi, + # z.ai gateway — occasionally emit ``[...],]`` which is invalid JSON + # and used to be silently dropped as ``{}``, surfacing as an empty + # successful ``/product/add``. + repaired = _strip_trailing_commas(t) + if repaired != t: + try: + result = json.loads(repaired) + logger.warning( + "[JSONParse] Repaired malformed JSON by stripping " + "trailing comma(s) before closer: %s", + e, + ) + return result + except json.JSONDecodeError as repair_err: + # The repair modified the text but the result is still not + # valid JSON (e.g. multiple orthogonal defects beyond + # trailing commas). Surface at DEBUG so future + # investigation has a breadcrumb; the outer WARNING below + # still fires with the original error for the operator. + logger.debug( + "[JSONParse] Trailing-comma repair did not fully fix " + "JSON: %s", + repair_err, + ) + logger.warning( f"[JSONParse] Failed to decode JSON: {e}\nTail: Raw {response_text} \ json: {s}" diff --git a/tests/mem_reader/test_parse_json_result.py b/tests/mem_reader/test_parse_json_result.py new file mode 100644 index 000000000..61dfeabf6 --- /dev/null +++ b/tests/mem_reader/test_parse_json_result.py @@ -0,0 +1,128 @@ +"""Regression tests for issue #2456. + +A malformed LLM extraction response with a trailing comma before a closing +brace (``[...],]`` or ``{"a": 1,}``) used to be silently converted to ``{}`` +by :func:`memos.mem_reader.utils.parse_json_result`, which downstream was +reported as a successful ``/product/add`` with ``data: []`` and no memories +stored. + +The parser now attempts a narrow trailing-comma repair (**outside quoted +strings only**) as a last-chance recovery before giving up with ``{}``. These +tests pin that behavior. +""" + +import unittest + +from memos.mem_reader.utils import parse_json_result + + +class TestParseJsonResultTrailingComma(unittest.TestCase): + """Issue #2456: trailing comma from LLM should be repaired, not dropped.""" + + # --- the exact reproduction from the bug report ------------------------ + + def test_issue_2456_reproduction(self): + """Minimal deterministic reproduction from the bug report must now + recover the memory list instead of returning ``{}``.""" + valid = '{"memory list": [{"value": "synthetic evidence"}]}' + malformed = '{"memory list": [{"value": "synthetic evidence"}],}' + + expected = {"memory list": [{"value": "synthetic evidence"}]} + + self.assertEqual(parse_json_result(valid), expected) + # Was ``{}`` before the fix — now the repair branch recovers the + # original payload so ``/product/add`` can persist the memory. + self.assertEqual(parse_json_result(malformed), expected) + + # --- trailing comma in various positions ------------------------------- + + def test_trailing_comma_before_object_close(self): + """``{"a": 1,}`` -> ``{"a": 1}``.""" + self.assertEqual(parse_json_result('{"a": 1,}'), {"a": 1}) + + def test_trailing_comma_before_array_close(self): + """``[1, 2, 3,]`` -> ``[1, 2, 3]`` when wrapped in an object.""" + self.assertEqual( + parse_json_result('{"xs": [1, 2, 3,]}'), + {"xs": [1, 2, 3]}, + ) + + def test_trailing_comma_before_newline_and_close(self): + """Whitespace / newlines between the comma and the closer must not + defeat the repair.""" + raw = '{"xs": [\n 1,\n 2,\n],\n}' + self.assertEqual(parse_json_result(raw), {"xs": [1, 2]}) + + def test_trailing_comma_inside_fenced_code_block(self): + """``parse_json_result`` already strips ``` fences; the repair must + still fire on the inner payload.""" + raw = '```json\n{"a": 1,}\n```' + self.assertEqual(parse_json_result(raw), {"a": 1}) + + # --- safety: do not touch commas that are legitimate ------------------- + + def test_comma_inside_string_is_not_stripped(self): + """A literal ``,}`` *inside* a quoted string must not be touched.""" + # The value contains ``foo,}`` as literal data. Repair would corrupt + # the string if the state machine were naive. + raw = '{"msg": "foo,}"}' + self.assertEqual(parse_json_result(raw), {"msg": "foo,}"}) + + def test_escaped_quote_inside_string_does_not_confuse_scanner(self): + """``\\"`` inside a string must keep the scanner in-string so that + a trailing comma *outside* the string is still repaired correctly + and commas *inside* the string are preserved.""" + # The outer object has a real trailing comma (triggers repair); + # the escaped quotes inside the value must not confuse the scanner + # into leaving in-string and treating one of the inner commas as + # the one to remove. Previously the raw string had the trailing + # ``,}`` *inside* the quoted value, so ``json.loads`` succeeded on + # the first try and the repair branch was never exercised. + raw = '{"msg": "he said \\"hi,\\" then left",}' + self.assertEqual( + parse_json_result(raw), + {"msg": 'he said "hi," then left'}, + ) + + def test_double_trailing_comma_is_fully_repaired(self): + """OCR follow-up to #2456: a run like ``{"a": 1,,}`` must collapse + to ``{"a": 1}``. Earlier revision only removed the *latest* + pending comma, leaving ``{"a": 1,}`` which then failed + ``json.loads`` silently and surfaced as an empty ``/product/add`` + response.""" + self.assertEqual(parse_json_result('{"a": 1,,}'), {"a": 1}) + self.assertEqual( + parse_json_result('{"xs": [1, 2,,,]}'), + {"xs": [1, 2]}, + ) + + def test_valid_json_is_unchanged(self): + """Valid JSON must round-trip unchanged — no accidental repair.""" + raw = '{"a": [1, 2, 3], "b": {"c": "d"}}' + self.assertEqual( + parse_json_result(raw), + {"a": [1, 2, 3], "b": {"c": "d"}}, + ) + + # --- hard-failure contract preserved ------------------------------------ + + def test_genuinely_malformed_still_returns_empty_dict(self): + """When the trailing-comma repair cannot help (missing value, + unmatched braces that even ``_cheap_close`` cannot resolve, etc.), + the historical ``{}`` return contract must be preserved so the 15+ + callers do not need to change.""" + # Missing value for key ``a`` — not a trailing-comma problem. + self.assertEqual(parse_json_result('{"a":}'), {}) + + def test_non_json_text_still_returns_empty_dict(self): + """Pure chatter with no JSON must still return ``{}`` — pins the + existing ``test_parse_json_result_failure`` contract.""" + self.assertEqual(parse_json_result("this is not json at all"), {}) + + def test_empty_input_returns_empty_dict(self): + self.assertEqual(parse_json_result(""), {}) + self.assertEqual(parse_json_result(None), {}) # type: ignore[arg-type] + + +if __name__ == "__main__": + unittest.main()