diff --git a/src/memos/mem_os/utils/format_utils.py b/src/memos/mem_os/utils/format_utils.py index 3991d0667..94c941166 100644 --- a/src/memos/mem_os/utils/format_utils.py +++ b/src/memos/mem_os/utils/format_utils.py @@ -1,5 +1,6 @@ import math import random +import re from typing import Any @@ -10,6 +11,19 @@ logger = get_logger(__name__) +# Strip the outer markdown fences from an LLM JSON response. The opening +# fence may include a short info string (``json`` or otherwise) optionally +# surrounded by blanks; the closing fence is bare. Both patterns are +# anchored so interior triple-backtick sequences inside a JSON string value +# are preserved verbatim (issue #2420 — suggestions/responses that +# contained code snippets were being silently corrupted before +# ``json.loads``). The info string is restricted to a real identifier so a +# single-line fenced payload (````` ```{"a": 1}``` `````, no newline after +# the fence) is not swallowed by a greedy character class. +_JSON_FENCE_OPEN_RE = re.compile(r"^```[ \t]*[A-Za-z0-9_+-]*[ \t]*\n?") +_JSON_FENCE_CLOSE_RE = re.compile(r"\n?```\s*$") + + def extract_node_name(memory: str) -> str: """Extract the first two words from memory as node_name""" if not memory: @@ -533,7 +547,7 @@ def convert_graph_to_tree_forworkmem( node_name = extract_node_name(memory) memory_key = node.get("metadata", {}).get("key", node_name) usage = node.get("metadata", {}).get("usage", []) - frequency = len(usage) if len(usage) < 100 else 100 + frequency = min(100, len(usage)) node_map[node["id"]] = { "id": node["id"], "value": memory, @@ -1394,11 +1408,16 @@ def clean_json_response(response: str) -> str: """ Remove markdown JSON code block formatting from LLM response. + Only an opening fence (with its optional ``json`` info string) at the + very start and a closing fence at the very end are markup; interior + triple-backtick sequences are part of the JSON payload (issue #2420) + and must be preserved verbatim. + Args: response: Raw response string that may contain ```json and ``` Returns: - str: Clean JSON string without markdown formatting + str: Clean JSON string without the outer markdown fences. Raises: ValueError: If ``response`` is None. This is almost always an upstream @@ -1413,4 +1432,11 @@ def clean_json_response(response: str) -> str: "failed silently (check timed_with_status / generate() error " "handling)." ) - return response.replace("```json", "").replace("```", "").strip() + s = response.strip() + # Anchor at the start so interior ```json / ``` markers inside JSON + # string values are left alone. The opening-fence info string ("json" or + # any other word) and the trailing newline are stripped. + s = _JSON_FENCE_OPEN_RE.sub("", s, count=1) + # Anchor at the end so a stray closing fence inside a value is kept. + s = _JSON_FENCE_CLOSE_RE.sub("", s, count=1) + return s.strip() diff --git a/tests/mem_os/test_format_utils.py b/tests/mem_os/test_format_utils.py index b97178784..5bf8e0d47 100644 --- a/tests/mem_os/test_format_utils.py +++ b/tests/mem_os/test_format_utils.py @@ -35,6 +35,17 @@ def test_clean_json_response_removes_plain_code_block(self): expected = '{"key": "value"}' assert clean_json_response(input_str) == expected + def test_clean_json_response_removes_single_line_fence(self): + """A single-line fenced payload (no newline after the opening fence) + must keep its JSON intact. A greedy info-string class swallowed the + whole payload here before the pattern was tightened.""" + assert clean_json_response('```{"a": 1}```') == '{"a": 1}' + + def test_clean_json_response_removes_single_line_fence_with_info_string(self): + """Same regression with an info string followed by a space instead + of a newline.""" + assert clean_json_response('```json {"key": "value"}```') == '{"key": "value"}' + def test_clean_json_response_strips_whitespace(self): """Test that leading/trailing whitespace is stripped.""" input_str = ' \n {"key": "value"} \n ' @@ -73,3 +84,37 @@ def test_clean_json_response_preserves_internal_backticks(self): result = clean_json_response(input_str) assert "`example`" in result assert result.count("`") == 2 # Only internal backticks remain + + def test_clean_json_response_preserves_internal_fenced_code(self): + """Issue #2420 reproducer: a JSON string value that itself contains a + fenced code block must not have its fences stripped. + + Before the fix the global ``response.replace("```", "")`` call + removed *every* triple-backtick sequence, including the one inside + the JSON value, silently corrupting suggestions and chat metadata + that contained code snippets. + """ + input_str = '```json\n{"text": "```py\\ncode```"}\n```' + result = clean_json_response(input_str) + assert result == '{"text": "```py\\ncode```"}' + + def test_clean_json_response_preserves_unclosed_fence_in_value(self): + """A JSON value containing a single unclosed fence (e.g. a partial + snippet the LLM didn't close) must also pass through unchanged. + """ + input_str = '```json\n{"text": "here is some ```partial"}\n```' + result = clean_json_response(input_str) + assert result == '{"text": "here is some ```partial"}' + + def test_clean_json_response_handles_only_opening_fence(self): + """An opening fence with no matching closing fence should still be + removed (best effort; it can never make the JSON valid).""" + input_str = '```json\n{"key": "value"}' + result = clean_json_response(input_str) + assert result == '{"key": "value"}' + + def test_clean_json_response_handles_only_closing_fence(self): + """A closing fence with no opening fence should still be removed.""" + input_str = '{"key": "value"}\n```' + result = clean_json_response(input_str) + assert result == '{"key": "value"}'