From 47a6f23aad5265470550989fe68aea651759990b Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Mon, 12 Oct 2026 02:18:37 +0800 Subject: [PATCH] fix(reorganizer): optimize all graph tenants, run scheduled jobs, and survive non-JSON LLM output - enumerate tenants via new get_all_user_names() (Neo4jGraphDB impl + BaseGraphDB fallback) and route both the _reorganize_needed trigger and the schedule jobs through _optimize_all_users(scope), isolating per-tenant failures; backends without the capability keep the previous single-tenant behavior - call schedule.run_pending() in the structure optimizer loop so the registered 100s jobs actually run instead of being dead code - make _parse_json_result return None for anything that is not a valid JSON object and short-circuit _summarize_cluster/_local_subcluster, so a malformed LLM reply skips the cluster instead of crashing the whole optimization round or silently writing empty parent nodes Fixes #2480 --- src/memos/graph_dbs/base.py | 13 + src/memos/graph_dbs/neo4j.py | 25 ++ .../tree_text_memory/organize/reorganizer.py | 97 +++++- tests/graph_dbs/test_get_all_user_names.py | 67 +++++ tests/memories/test_reorganizer.py | 282 ++++++++++++++++++ 5 files changed, 471 insertions(+), 13 deletions(-) create mode 100644 tests/graph_dbs/test_get_all_user_names.py create mode 100644 tests/memories/test_reorganizer.py diff --git a/src/memos/graph_dbs/base.py b/src/memos/graph_dbs/base.py index 0bc4a54f8..7fb55d223 100644 --- a/src/memos/graph_dbs/base.py +++ b/src/memos/graph_dbs/base.py @@ -302,3 +302,16 @@ def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str | None = N - metadata: dict[str, Any] - Node metadata user_name: Optional user name (will use config default if not provided) """ + + def get_all_user_names(self) -> list[str]: + """ + List every tenant (user_name) that currently has data in the graph store. + + Default fallback for backends that do not support tenant enumeration: + returns only the configured default user_name (if any), preserving the + previous single-tenant behavior. Backends with tenant isolation (e.g. + Neo4j) override this with a real implementation. + """ + config = getattr(self, "config", None) + user_name = getattr(config, "user_name", None) + return [user_name] if user_name else [] diff --git a/src/memos/graph_dbs/neo4j.py b/src/memos/graph_dbs/neo4j.py index 56c3e08a0..aa418777a 100644 --- a/src/memos/graph_dbs/neo4j.py +++ b/src/memos/graph_dbs/neo4j.py @@ -194,6 +194,31 @@ def node_not_exist(self, scope: str, user_name: str | None = None) -> int: ) return result.single() is None + def get_all_user_names(self) -> list[str]: + """List every tenant (user_name) that currently has Memory nodes. + + Used by the structure reorganizer to optimize all tenants instead of + only the configured default one. Falls back to the configured + ``user_name`` when the query fails or the graph has no named tenants, + so callers keep the previous single-tenant behavior. + """ + fallback = [self.config.user_name] if self.config.user_name else [] + try: + query = """ + MATCH (n:Memory) + WHERE n.user_name IS NOT NULL + RETURN DISTINCT n.user_name AS user_name + """ + with self.driver.session(database=self.db_name) as session: + result = session.run(query) + user_names = sorted( + {record["user_name"] for record in result if record["user_name"]} + ) + return user_names or fallback + except Exception as e: + logger.warning(f"[get_all_user_names] Failed to enumerate tenants: {e}") + return fallback + def remove_oldest_memory( self, memory_type: str, keep_latest: int, user_name: str | None = None ) -> None: diff --git a/src/memos/memories/textual/tree_text_memory/organize/reorganizer.py b/src/memos/memories/textual/tree_text_memory/organize/reorganizer.py index b7fb6b1a0..79eadb4d3 100644 --- a/src/memos/memories/textual/tree_text_memory/organize/reorganizer.py +++ b/src/memos/memories/textual/tree_text_memory/organize/reorganizer.py @@ -155,21 +155,56 @@ def _run_structure_organizer_loop(self): """ import schedule - schedule.every(100).seconds.do(self.optimize_structure, scope="LongTermMemory") - schedule.every(100).seconds.do(self.optimize_structure, scope="UserMemory") + schedule.every(100).seconds.do(self._optimize_all_users, scope="LongTermMemory") + schedule.every(100).seconds.do(self._optimize_all_users, scope="UserMemory") logger.info("Structure optimizer schedule started.") while not getattr(self, "_stop_scheduler", False): + schedule.run_pending() if any(self._is_optimizing.values()): time.sleep(1) continue if self._reorganize_needed: logger.info("[Reorganizer] Triggering optimize_structure due to new nodes.") - self.optimize_structure(scope="LongTermMemory") - self.optimize_structure(scope="UserMemory") + self._optimize_all_users(scope="LongTermMemory") + self._optimize_all_users(scope="UserMemory") self._reorganize_needed = False time.sleep(30) + def _optimize_all_users(self, scope: str, **kwargs) -> None: + """ + Run structure optimization for every tenant present in the graph store. + + Without tenant enumeration, only the configured default tenant (e.g. + ``memosdefault``) would ever be optimized, while data written through + the API lives under each request's ``user_name``. Backends that cannot + enumerate tenants (no ``get_all_user_names``) keep the previous + single-tenant behavior. A failure on one tenant is logged and does not + abort the remaining tenants. + """ + get_user_names = getattr(self.graph_store, "get_all_user_names", None) + try: + user_names = get_user_names() if callable(get_user_names) else None + except Exception as e: + logger.warning( + f"[GraphStructureReorganize] Failed to enumerate tenants, " + f"falling back to the configured default: {e}" + ) + user_names = None + + if not user_names: + self.optimize_structure(scope=scope, **kwargs) + return + + for user_name in user_names: + try: + self.optimize_structure(scope=scope, user_name=user_name, **kwargs) + except Exception as e: + logger.warning( + f"[GraphStructureReorganize] optimize_structure failed for " + f"user_name={user_name}: {e}, trace: {traceback.format_exc()}" + ) + def stop(self): """ Stop the reorganizer thread. @@ -330,17 +365,20 @@ def _process_cluster_and_write( if len(sub_nodes) < min_cluster_size: continue # Skip tiny noise sub_parent_node = self._summarize_cluster(sub_nodes, scope) + if sub_parent_node is None: + continue # LLM output was not a valid JSON object; skip this sub-cluster self._create_parent_node(sub_parent_node, user_name=user_name) self._link_cluster_nodes(sub_parent_node, sub_nodes, user_name=user_name) sub_parents.append(sub_parent_node) if sub_parents and len(sub_parents) >= min_cluster_size: cluster_parent_node = self._summarize_cluster(cluster_nodes, scope) - self._create_parent_node(cluster_parent_node, user_name=user_name) - for sub_parent in sub_parents: - self.graph_store.add_edge( - cluster_parent_node.id, sub_parent.id, "PARENT", user_name=user_name - ) + if cluster_parent_node is not None: + self._create_parent_node(cluster_parent_node, user_name=user_name) + for sub_parent in sub_parents: + self.graph_store.add_edge( + cluster_parent_node.id, sub_parent.id, "PARENT", user_name=user_name + ) logger.info("Adding relations/reasons") nodes_to_check = cluster_nodes @@ -436,6 +474,12 @@ def _local_subcluster( messages = [{"role": "user", "content": prompt}] response_text = self.llm.generate(messages) response_json = self._parse_json_result(response_text) + if response_json is None: + logger.warning( + "[GraphStructureReorganize] Skip local sub-clustering: " + "LLM returned no valid JSON object." + ) + return [] assigned_ids = set() result_subclusters = [] @@ -547,9 +591,14 @@ def recursive_clustering(nodes_list, depth=0): return filtered_clusters - def _summarize_cluster(self, cluster_nodes: list[GraphDBNode], scope: str) -> GraphDBNode: + def _summarize_cluster( + self, cluster_nodes: list[GraphDBNode], scope: str + ) -> GraphDBNode | None: """ Generate a cluster label using LLM, based on top keys in the cluster. + + Returns ``None`` when the LLM does not return a valid JSON object, so + the caller can skip the cluster instead of writing an empty parent node. """ if not cluster_nodes: raise ValueError("Cluster nodes cannot be empty.") @@ -567,6 +616,12 @@ def _summarize_cluster(self, cluster_nodes: list[GraphDBNode], scope: str) -> Gr messages = [{"role": "user", "content": prompt}] response_text = self.llm.generate(messages) response_json = self._parse_json_result(response_text) + if response_json is None: + logger.warning( + "[GraphStructureReorganize] Skip cluster summary: " + "LLM returned no valid JSON object." + ) + return None # Extract fields parent_key = response_json.get("key", "").strip() @@ -595,16 +650,32 @@ def _summarize_cluster(self, cluster_nodes: list[GraphDBNode], scope: str) -> Gr ) return parent_node - def _parse_json_result(self, response_text): + def _parse_json_result(self, response_text) -> dict | None: + """ + Parse the first ``{...}`` JSON object found in an LLM response. + + Returns ``None`` for anything that is not a valid JSON object (empty + text, no braces, malformed JSON, or a non-object JSON value such as a + list), so callers can skip gracefully instead of crashing the whole + optimization round or silently writing empty parent nodes. + """ + if not isinstance(response_text, str) or not response_text.strip(): + return None try: response_text = response_text.replace("```", "").replace("json", "") response_json = extract_first_to_last_brace(response_text)[1] - return response_json except json.JSONDecodeError as e: logger.warning( f"Failed to parse LLM response as JSON: {e}\nRaw response:\n{response_text}" ) - return {} + return None + if not isinstance(response_json, dict): + logger.warning( + f"LLM response is not a JSON object (got {type(response_json).__name__}); " + f"raw response:\n{response_text}" + ) + return None + return response_json def _create_parent_node(self, parent_node: GraphDBNode, user_name: str | None = None) -> None: """ diff --git a/tests/graph_dbs/test_get_all_user_names.py b/tests/graph_dbs/test_get_all_user_names.py new file mode 100644 index 000000000..0e7f23441 --- /dev/null +++ b/tests/graph_dbs/test_get_all_user_names.py @@ -0,0 +1,67 @@ +"""Offline tests for Neo4jGraphDB.get_all_user_names (issue #2480). + +The structure reorganizer needs to enumerate every tenant (user_name) present +in the graph so optimization covers all tenants instead of only the configured +default one. These tests stub the Neo4j driver; no server is required. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from memos.configs.graph_db import Neo4jGraphDBConfig + + +@pytest.fixture +def neo4j_db(): + config = Neo4jGraphDBConfig( + uri="bolt://localhost:7687", + user="neo4j", + password="test", + db_name="test_memory_db", + auto_create=False, + embedding_dimension=3, + user_name="memosdefault", + ) + with patch("neo4j.GraphDatabase") as mock_gd: + mock_driver = MagicMock() + mock_gd.driver.return_value = mock_driver + from memos.graph_dbs.neo4j import Neo4jGraphDB + + db = Neo4jGraphDB(config) + db.driver = mock_driver + yield db + + +class TestNeo4jGetAllUserNames: + def test_returns_distinct_user_names(self, neo4j_db): + session_mock = neo4j_db.driver.session.return_value.__enter__.return_value + session_mock.run.return_value = [ + {"user_name": "alice"}, + {"user_name": "bob"}, + {"user_name": "memosdefault"}, + ] + + assert neo4j_db.get_all_user_names() == ["alice", "bob", "memosdefault"] + + def test_ignores_null_and_blank_user_names(self, neo4j_db): + session_mock = neo4j_db.driver.session.return_value.__enter__.return_value + session_mock.run.return_value = [ + {"user_name": "alice"}, + {"user_name": None}, + {"user_name": ""}, + ] + + assert neo4j_db.get_all_user_names() == ["alice"] + + def test_empty_graph_falls_back_to_configured_user_name(self, neo4j_db): + session_mock = neo4j_db.driver.session.return_value.__enter__.return_value + session_mock.run.return_value = [] + + assert neo4j_db.get_all_user_names() == ["memosdefault"] + + def test_query_failure_falls_back_to_configured_user_name(self, neo4j_db): + session_mock = neo4j_db.driver.session.return_value.__enter__.return_value + session_mock.run.side_effect = RuntimeError("driver closed") + + assert neo4j_db.get_all_user_names() == ["memosdefault"] diff --git a/tests/memories/test_reorganizer.py b/tests/memories/test_reorganizer.py new file mode 100644 index 000000000..47e1b207e --- /dev/null +++ b/tests/memories/test_reorganizer.py @@ -0,0 +1,282 @@ +"""Offline unit tests for GraphStructureReorganizer (issue #2480). + +Covers three defects: +1. The periodic/_reorganize_needed optimization only ever touched the default + configured tenant; it must enumerate all tenants present in the graph. +2. The schedule-based jobs were dead code: the loop never called + ``schedule.run_pending()``. +3. A non-JSON LLM reply crashed the whole optimization round via + ``AttributeError`` (and the ``JSONDecodeError`` path silently produced empty + parent nodes). + +Graph store / LLM / embedder are all stubbed; no Neo4j, LLM service, network, +or embedder is required. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +import memos.memories.textual.tree_text_memory.organize.reorganizer as reorganizer_module + +from memos.graph_dbs.base import BaseGraphDB +from memos.memories.textual.tree_text_memory.organize.reorganizer import ( + GraphStructureReorganizer, + extract_first_to_last_brace, +) + + +def make_node(node_id: str, memory: str, key: str = "k", background: str = ""): + return SimpleNamespace( + id=node_id, + memory=memory, + metadata=SimpleNamespace(key=key, background=background), + ) + + +@pytest.fixture() +def reorganizer(): + graph_store = MagicMock() + llm = MagicMock() + embedder = MagicMock() + # Avoid background threads: is_reorganize=False. + return GraphStructureReorganizer(graph_store, llm, embedder, is_reorganize=False) + + +EMPTY_RELATION_RESULTS = { + "relations": [], + "inferred_nodes": [], + "sequence_links": [], + "aggregate_nodes": [], +} + + +# --------------------------------------------------------------------------- +# Defect C: LLM output parsing must never crash the optimization round +# --------------------------------------------------------------------------- + + +def test_extract_first_to_last_brace_without_braces_yields_none(): + """Upstream sentinel: no braces -> ("", None), must not raise.""" + json_str, parsed = extract_first_to_last_brace("sorry, I cannot answer that") + assert json_str == "" + assert parsed is None + + +def test_parse_json_result_plain_text_returns_none(reorganizer): + assert reorganizer._parse_json_result("sorry, I cannot answer that") is None + + +def test_parse_json_result_malformed_json_returns_none(reorganizer): + """Braces present but invalid JSON -> None (previously {}).""" + assert reorganizer._parse_json_result('{"key": "broken') is None + + +def test_parse_json_result_interior_brace_invalid_json_returns_none(reorganizer): + """Braces exist but the enclosed text is invalid JSON -> None (previously {}).""" + assert reorganizer._parse_json_result('noise {"key": "broken} more noise') is None + + +def test_summarize_cluster_returns_none_on_malformed_json_no_empty_parent(reorganizer): + """The JSONDecodeError path must not silently build an empty parent node.""" + reorganizer.llm.generate.return_value = 'noise {"key": "broken} more noise' + + result = reorganizer._summarize_cluster([make_node("n1", "m1")], "LongTermMemory") + + assert result is None + reorganizer.embedder.embed.assert_not_called() + + +def test_parse_json_result_non_object_json_returns_none(reorganizer, monkeypatch): + """A parsed JSON value that is not an object must not be passed through.""" + monkeypatch.setattr( + reorganizer_module, + "extract_first_to_last_brace", + lambda text: (text, [{"key": "a"}]), + ) + assert reorganizer._parse_json_result("whatever") is None + + +def test_parse_json_result_empty_text_returns_none(reorganizer): + assert reorganizer._parse_json_result("") is None + assert reorganizer._parse_json_result(None) is None + + +def test_parse_json_result_object_passthrough(reorganizer): + text = 'Here you go:\n```json\n{"key": "travel", "value": "likes Kyoto"}\n```' + assert reorganizer._parse_json_result(text) == {"key": "travel", "value": "likes Kyoto"} + + +def test_summarize_cluster_returns_none_on_non_json_llm_output(reorganizer): + reorganizer.llm.generate.return_value = "sorry, I cannot answer that" + + result = reorganizer._summarize_cluster([make_node("n1", "m1")], "LongTermMemory") + + assert result is None + reorganizer.embedder.embed.assert_not_called() + + +def test_summarize_cluster_builds_parent_node_from_llm_json(reorganizer): + reorganizer.llm.generate.return_value = ( + '{"key": "travel", "value": "User likes Kyoto", ' + '"tags": ["travel"], "summary": "travel preference"}' + ) + reorganizer.embedder.embed.return_value = [[0.1, 0.2, 0.3]] + + parent = reorganizer._summarize_cluster([make_node("n1", "m1")], "LongTermMemory") + + assert parent is not None + assert parent.memory == "User likes Kyoto" + assert parent.metadata.key == "travel" + reorganizer.embedder.embed.assert_called_once_with(["User likes Kyoto"]) + + +def test_local_subcluster_returns_empty_on_non_json_llm_output(reorganizer): + reorganizer.llm.generate.return_value = "no json for you" + + assert reorganizer._local_subcluster([make_node("n1", "m1")]) == [] + + +def test_local_subcluster_parses_valid_clusters(reorganizer): + reorganizer.llm.generate.return_value = '{"clusters": [{"ids": ["a", "b"]}]}' + nodes = [make_node("a", "ma"), make_node("b", "mb")] + + assert reorganizer._local_subcluster(nodes) == [nodes] + + +def test_process_cluster_skips_failed_summary_without_writing_empty_parents(reorganizer): + reorganizer.llm.generate.return_value = "sorry, I cannot answer that" + reorganizer.embedder.embed.return_value = [[0.1]] + reorganizer.relation_detector.process_node = MagicMock( + return_value=dict(EMPTY_RELATION_RESULTS) + ) + cluster = [make_node(f"n{i}", f"m{i}") for i in range(4)] + + # Must not raise, and must not create any (empty) parent node. + reorganizer._process_cluster_and_write( + cluster, "LongTermMemory", local_tree_threshold=10, min_cluster_size=2 + ) + + reorganizer.graph_store.add_node.assert_not_called() + reorganizer.graph_store.add_edge.assert_not_called() + + +# --------------------------------------------------------------------------- +# Defect A: optimization must cover all graph tenants, not only the default one +# --------------------------------------------------------------------------- + + +def test_optimize_all_users_enumerates_graph_tenants(reorganizer): + reorganizer.graph_store.get_all_user_names.return_value = ["alice", "bob", "memosdefault"] + reorganizer.optimize_structure = MagicMock() + + reorganizer._optimize_all_users(scope="LongTermMemory") + + assert [ + (call.kwargs["scope"], call.kwargs["user_name"]) + for call in reorganizer.optimize_structure.call_args_list + ] == [ + ("LongTermMemory", "alice"), + ("LongTermMemory", "bob"), + ("LongTermMemory", "memosdefault"), + ] + + +def test_optimize_all_users_tolerates_single_tenant_failure(reorganizer): + reorganizer.graph_store.get_all_user_names.return_value = ["alice", "bob"] + reorganizer.optimize_structure = MagicMock( + side_effect=[RuntimeError("neo4j unavailable"), None] + ) + + # The failing first tenant must not abort the second one. + reorganizer._optimize_all_users(scope="UserMemory") + + assert reorganizer.optimize_structure.call_count == 2 + reorganizer.optimize_structure.assert_any_call(scope="UserMemory", user_name="bob") + + +def test_optimize_all_users_falls_back_without_tenant_enumeration(reorganizer): + """Backends without get_all_user_names keep the previous single-tenant call.""" + del reorganizer.graph_store.get_all_user_names + reorganizer.optimize_structure = MagicMock() + + reorganizer._optimize_all_users(scope="LongTermMemory") + + reorganizer.optimize_structure.assert_called_once_with(scope="LongTermMemory") + + +def test_optimize_all_users_falls_back_on_enumeration_error(reorganizer): + reorganizer.graph_store.get_all_user_names.side_effect = RuntimeError("driver closed") + reorganizer.optimize_structure = MagicMock() + + reorganizer._optimize_all_users(scope="UserMemory") + + reorganizer.optimize_structure.assert_called_once_with(scope="UserMemory") + + +def test_base_graph_db_default_returns_configured_user_name(): + """Backends that do not implement enumeration fall back to config.user_name.""" + dummy = SimpleNamespace(config=SimpleNamespace(user_name="tenant-a")) + + assert BaseGraphDB.get_all_user_names(dummy) == ["tenant-a"] + + +def test_base_graph_db_default_handles_missing_config(): + dummy = SimpleNamespace() # no .config at all + + assert BaseGraphDB.get_all_user_names(dummy) == [] + + +# --------------------------------------------------------------------------- +# Defect B: the loop must actually run the scheduled jobs +# --------------------------------------------------------------------------- + + +def _run_one_loop_iteration(monkeypatch, reorganizer): + """Run _run_structure_organizer_loop until its first sleep, then stop.""" + import schedule + + # State normally initialized in __init__ only when is_reorganize=True + # (the loop thread is never started otherwise). + reorganizer._stop_scheduler = False + reorganizer._is_optimizing = {"LongTermMemory": False, "UserMemory": False} + + monkeypatch.setattr(schedule, "every", MagicMock()) + run_pending = MagicMock() + monkeypatch.setattr(schedule, "run_pending", run_pending) + + def fake_sleep(_seconds): + reorganizer._stop_scheduler = True + + monkeypatch.setattr(reorganizer_module.time, "sleep", fake_sleep) + reorganizer._run_structure_organizer_loop() + return run_pending + + +def test_loop_calls_schedule_run_pending(monkeypatch, reorganizer): + reorganizer.graph_store.get_all_user_names.return_value = ["alice"] + reorganizer.optimize_structure = MagicMock() + + run_pending = _run_one_loop_iteration(monkeypatch, reorganizer) + + run_pending.assert_called() + + +def test_reorganize_trigger_covers_all_tenants(monkeypatch, reorganizer): + reorganizer.graph_store.get_all_user_names.return_value = ["alice", "bob"] + reorganizer.optimize_structure = MagicMock() + reorganizer._reorganize_needed = True + + _run_one_loop_iteration(monkeypatch, reorganizer) + + assert not reorganizer._reorganize_needed + assert { + (call.kwargs["scope"], call.kwargs["user_name"]) + for call in reorganizer.optimize_structure.call_args_list + } == { + ("LongTermMemory", "alice"), + ("LongTermMemory", "bob"), + ("UserMemory", "alice"), + ("UserMemory", "bob"), + }