From a318a6eca029891602b87675902c7b14d2e1b7f6 Mon Sep 17 00:00:00 2001 From: Matthew Date: Tue, 22 Sep 2026 16:09:09 +0800 Subject: [PATCH 01/15] docs: align Chinese contribution links with the English docs Co-authored-by: Cursor --- .../open_source/contribution/development_workflow.md | 2 +- docs/cn/open_source/contribution/overview.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/cn/open_source/contribution/development_workflow.md b/docs/cn/open_source/contribution/development_workflow.md index 088784fb8..0f8353998 100644 --- a/docs/cn/open_source/contribution/development_workflow.md +++ b/docs/cn/open_source/contribution/development_workflow.md @@ -48,7 +48,7 @@ git fetch upstream git rebase upstream/dev # 把你的 feat 分支基于最新 dev 重放 ``` -提交更改时请遵循项目的提交规范(参见 [提交规范](commit_guidelines.md))。 +提交更改时请遵循项目的提交规范(参见 [提交规范](./commit_guidelines.md))。 #### 推送到你的 Fork 仓库 diff --git a/docs/cn/open_source/contribution/overview.md b/docs/cn/open_source/contribution/overview.md index 83f58c4bc..61678a8ac 100644 --- a/docs/cn/open_source/contribution/overview.md +++ b/docs/cn/open_source/contribution/overview.md @@ -3,11 +3,11 @@ title: 参与 MemOS 开发 desc: 欢迎阅读 MemOS 贡献指南!了解如何配置开发环境、遵循我们的开发流程、撰写规范的提交信息、完善文档以及添加测试用例。 --- -- **首次贡献者:** 请先阅读 [环境配置指南](setting_up.md),以准备你的开发环境。 -- **准备开始编码了吗?** 请查看 [开发流程](development_workflow.md) 指南,了解我们提交更改的流程。 -- **撰写规范的提交信息:** 请参考我们的 [提交规范](commit_guidelines.md)。 -- **参与文档编写:** 如果你正在帮助我们改进文档,请阅读 [文档编写指南](writing_docs.md)。 -- **添加或完善测试用例:** 请参考 [测试编写指南](writing_tests.md)。 +- **首次贡献者:** 请先阅读 [环境配置指南](./setting_up.md),以准备你的开发环境。 +- **准备开始编码了吗?** 请查看 [开发流程](./development_workflow.md) 指南,了解我们提交更改的流程。 +- **撰写规范的提交信息:** 请参考我们的 [提交规范](./commit_guidelines.md)。 +- **参与文档编写:** 如果你正在帮助我们改进文档,请阅读 [文档编写指南](./writing_docs.md)。 +- **添加或完善测试用例:** 请参考 [测试编写指南](./writing_tests.md)。 你的贡献让这个项目更加优秀!✨ 如果有任何问题,欢迎提交 issue 或参与讨论,也可以扫描下方二维码加入我们的 Discord 或微信社群与我们联系。 From 568e9a1cfe007312732db88476629334c9cf9d9f Mon Sep 17 00:00:00 2001 From: Wang Daoji <75928131+Wang-Daoji@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:09:33 -0700 Subject: [PATCH 02/15] fix: recall playground memories from every searched cube (#2436) Playground only kept the first cube in text_mem, so personal memories were dropped when a knowledge-base cube came first. Co-authored-by: Cursor --- src/memos/api/handlers/chat_handler.py | 24 ++++++++++++------------ tests/api/test_playground_cube_recall.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 tests/api/test_playground_cube_recall.py diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 58a96cd75..31acec6d0 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -482,12 +482,8 @@ def generate_chat_response() -> Generator[str, None, None]: yield f"data: {json.dumps({'type': 'status', 'data': '1'})}\n\n" - # Extract memories from search results (first search) - memories_list = [] - if search_response.data and search_response.data.get("text_mem"): - text_mem_results = search_response.data["text_mem"] - if text_mem_results and text_mem_results[0].get("memories"): - memories_list = text_mem_results[0]["memories"] + # Extract memories from every searched cube (first search) + memories_list = self._memories_from_cube_results(search_response.data) # Filter memories by threshold filtered_memories = self._filter_memories_by_threshold(memories_list)[:5] @@ -577,12 +573,8 @@ def generate_chat_response() -> Generator[str, None, None]: project_id=chat_req.project_id, ) - # Extract memories from search results (second search) - memories_list = [] - if search_response.data and search_response.data.get("text_mem"): - text_mem_results = search_response.data["text_mem"] - if text_mem_results and text_mem_results[0].get("memories"): - memories_list = text_mem_results[0]["memories"] + # Extract memories from every searched cube (second search) + memories_list = self._memories_from_cube_results(search_response.data) # Filter memories by threshold, min_num is the min number of memories for playground second_filtered_memories = self._filter_memories_by_threshold( @@ -1161,6 +1153,14 @@ def _format_mem_block( return "\n".join(lines_o), "\n".join(lines_p) + @staticmethod + def _memories_from_cube_results(search_data: dict[str, Any] | None) -> list[Any]: + """Collect textual memories from every cube in a search response.""" + memories_list: list[Any] = [] + for cube_result in (search_data or {}).get("text_mem") or []: + memories_list.extend(cube_result.get("memories") or []) + return memories_list + def _filter_memories_by_threshold( self, memories: list, diff --git a/tests/api/test_playground_cube_recall.py b/tests/api/test_playground_cube_recall.py new file mode 100644 index 000000000..4b1d8fd40 --- /dev/null +++ b/tests/api/test_playground_cube_recall.py @@ -0,0 +1,24 @@ +"""Playground recall must keep memories from every searched cube.""" + +from memos.api.handlers.chat_handler import ChatHandler + + +def test_playground_recall_keeps_memories_from_later_cubes(): + search_data = { + "text_mem": [ + {"cube_id": "knowledge-base", "memories": []}, + { + "cube_id": "personal", + "memories": [{"id": "mem-1", "memory": "use event_id as the idempotency key"}], + }, + ] + } + + memories = ChatHandler._memories_from_cube_results(search_data) + + assert [item["id"] for item in memories] == ["mem-1"] + + +def test_playground_recall_returns_empty_when_search_data_missing(): + assert ChatHandler._memories_from_cube_results(None) == [] + assert ChatHandler._memories_from_cube_results({}) == [] From aba1391324f5ed1db2da8fe9b74e22be0002c0dc Mon Sep 17 00:00:00 2001 From: Daoji Wang <627665797@qq.com> Date: Thu, 8 Oct 2026 16:30:32 +0800 Subject: [PATCH 03/15] fix: skip vision calls for non-sendable image URLs Relative markdown paths such as images/.jpg were forwarded to the vision API and rejected as invalid base64. Co-authored-by: Cursor --- .../read_multi_modal/image_parser.py | 19 +++++ tests/mem_reader/test_image_parser.py | 76 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 tests/mem_reader/test_image_parser.py diff --git a/src/memos/mem_reader/read_multi_modal/image_parser.py b/src/memos/mem_reader/read_multi_modal/image_parser.py index ec19ba304..40561c430 100644 --- a/src/memos/mem_reader/read_multi_modal/image_parser.py +++ b/src/memos/mem_reader/read_multi_modal/image_parser.py @@ -27,6 +27,18 @@ logger = get_logger(__name__) +def is_sendable_image_url(url: str) -> bool: + """Return whether a vision API can accept this image address. + + Accepted forms are http(s) URLs and ``data:image/...;base64,...`` payloads. + Relative paths such as ``images/.jpg`` are rejected. + """ + candidate = url.strip().lower() + if candidate.startswith(("http://", "https://")): + return True + return candidate.startswith("data:image/") and "base64," in candidate + + class ImageParser(BaseMessageParser): """Parser for image_url content parts.""" @@ -177,6 +189,13 @@ def parse_fine( logger.warning("[ImageParser] No image URL found in message") return [] + if not is_sendable_image_url(url): + logger.warning( + "[ImageParser] Skip image URL that cannot be sent to the vision model: %s", + url[:200], + ) + return [] + # Create source for this image source = self.create_source(message, info) diff --git a/tests/mem_reader/test_image_parser.py b/tests/mem_reader/test_image_parser.py new file mode 100644 index 000000000..374ac17e9 --- /dev/null +++ b/tests/mem_reader/test_image_parser.py @@ -0,0 +1,76 @@ +"""Tests for vision image URL validation in ImageParser.parse_fine.""" + +import json + +from unittest.mock import MagicMock + +import pytest + +from memos.mem_reader.read_multi_modal.image_parser import ImageParser + + +def _parser() -> tuple[ImageParser, MagicMock]: + llm = MagicMock() + llm.generate.return_value = json.dumps( + { + "memory list": [ + { + "key": "visible label", + "memory_type": "LongTermMemory", + "value": "A product label is visible.", + "tags": ["image"], + } + ], + "summary": "A labeled product.", + } + ) + embedder = MagicMock() + embedder.embed.return_value = [[0.1, 0.2]] + return ImageParser(embedder, llm), llm + + +def _image_message(url: str) -> dict: + return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}} + + +INFO = {"user_id": "user-1", "session_id": "session-1"} + + +@pytest.mark.parametrize( + "url", + [ + "images/c502e59b213998cd695d7f57a6e33de73f5493f2d3fbab6c4e9635341aa7e7b7.jpg", + "image/photo.png", + "/tmp/photo.jpg", + "file:///tmp/photo.jpg", + "data:images/jpeg;base64,aaaa", + "data:image/jpeg,not-base64", + ], +) +def test_parse_fine_skips_non_sendable_image_url(url: str) -> None: + parser, llm = _parser() + + result = parser.parse_fine(_image_message(url), INFO) + + assert result == [] + llm.generate.assert_not_called() + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/device.jpg", + "http://example.com/images/device.jpg", + "data:image/jpeg;base64,aaaa", + "DATA:IMAGE/PNG;base64,bbbb", + ], +) +def test_parse_fine_sends_http_and_base64_image_urls(url: str) -> None: + parser, llm = _parser() + + result = parser.parse_fine(_image_message(url), INFO) + + assert len(result) == 1 + llm.generate.assert_called_once() + sent_url = llm.generate.call_args.args[0][0]["content"][1]["image_url"]["url"] + assert sent_url == url From 1729844a4eae03c562ac37804bb6ffd8095ca147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=91=E5=B8=83=E6=9E=97?= <11641432+heiheiyouyou@user.noreply.gitee.com> Date: Sat, 10 Oct 2026 15:38:04 +0800 Subject: [PATCH 04/15] Decouple knowledge memories from graph-edge relationships Persist summary-material and raw-file neighbor relationships as node metadata so knowledge storage, retrieval deduplication, deletion, merging, and feedback locking remain functional on backends without graph edges. Raw file creation now follows file sources independently of prompt selection. Constraint: SUMMARY, MATERIAL, PRECEDING, and FOLLOWING edge operations are unavailable on the replacement graph backend. Rejected: Remove legacy edge helpers | retained unused compatibility code as requested. Confidence: high Scope-risk: moderate Directive: Keep knowledge_material_ids and knowledge_summary_ids bidirectional when adding, merging, or deleting knowledge nodes. Tested: 10 focused test functions executed directly; Ruff passed on all changed files. Not-tested: pytest runner crashes with exit 139 in the existing Anaconda environment; uv environment setup timed out fetching lxml. --- src/memos/mem_feedback/feedback.py | 23 +-- src/memos/mem_reader/multi_modal_struct.py | 70 ++++--- .../handlers/mem_read_handler.py | 2 +- src/memos/memories/textual/item.py | 8 + src/memos/memories/textual/tree.py | 171 +++++++++++++++- .../tree_text_memory/retrieve/searcher.py | 31 +-- src/memos/multi_mem_cube/single_cube.py | 27 ++- .../test_single_cube_knowledge_neighbors.py | 50 +++++ tests/mem_feedback/test_feedback.py | 20 ++ tests/mem_reader/test_multi_modal_rawfile.py | 89 +++++++++ .../textual/test_tree_knowledge_relations.py | 182 ++++++++++++++++++ tests/memories/textual/test_tree_searcher.py | 25 +++ 12 files changed, 622 insertions(+), 76 deletions(-) create mode 100644 tests/mem_cube/test_single_cube_knowledge_neighbors.py create mode 100644 tests/mem_reader/test_multi_modal_rawfile.py create mode 100644 tests/memories/textual/test_tree_knowledge_relations.py diff --git a/src/memos/mem_feedback/feedback.py b/src/memos/mem_feedback/feedback.py index 28f495afb..18ebb0b49 100644 --- a/src/memos/mem_feedback/feedback.py +++ b/src/memos/mem_feedback/feedback.py @@ -657,11 +657,6 @@ def _info_comparison(self, memory: TextualMemoryItem, _info: dict, include_keys: def _retrieve(self, query: str, info=None, top_k=20, user_name=None): """Retrieve memory items""" - def check_has_edges(mem_item: TextualMemoryItem) -> tuple[TextualMemoryItem, bool]: - """Check if a memory item has edges.""" - edges = self.searcher.graph_store.get_edges(mem_item.id, user_name=user_name) - return (mem_item, len(edges) == 0) - logger.info(f"[feedback _retrieve] query: {query}, user_name: {user_name}") text_mems = self.searcher.search( query=query, @@ -686,22 +681,16 @@ def check_has_edges(mem_item: TextualMemoryItem) -> tuple[TextualMemoryItem, boo pref_mems = [item[0] for item in pref_mems if float(item[1]) > 0.01] text_mems.extend(pref_mems) - # Memory with edges is not modified by feedback - retrieved_mems = [] - with ContextThreadPoolExecutor(max_workers=10) as executor: - futures = {executor.submit(check_has_edges, item): item for item in text_mems} - for future in concurrent.futures.as_completed(futures): - try: - mem_item, has_no_edges = future.result() - if has_no_edges: - retrieved_mems.append(mem_item) - except Exception as e: - logger.error(f"[0107 Feedback Core: _retrieve] Error checking edges: {e}") + retrieved_mems = [ + item + for item in text_mems + if not (getattr(item.metadata, "knowledge_material_ids", None) or []) + ] if len(retrieved_mems) < len(text_mems): logger.info( f"[0107 Feedback Core: _retrieve] {len(text_mems) - len(retrieved_mems)} " - f"text memories are not modified by feedback due to edges." + f"knowledge memories are not modified by feedback." ) return retrieved_mems diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index c46331f1d..eb226c27d 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -769,30 +769,52 @@ def _process_one_item( except Exception as e: logger.error(f"[MultiModalFine] parse error: {e}") - # save rawfile node - if self.save_rawfile and prompt_type == "doc" and len(fine_items) > 0: - rawfile_chunk = mem_str - file_info = fine_items[0].metadata.sources[0].file_info - source = self.multi_modal_parser.file_content_parser.create_source( - message={"file": file_info}, - info=info_per_item, - chunk_index=chunk_idx, - chunk_total=total_chunks, - chunk_content="", - ) - rawfile_node = self._make_memory_item( - value=rawfile_chunk, - info=info_per_item, - memory_type="RawFileMemory", - tags=[ - "mode:fine", - "multimodal:file", - f"chunk:{chunk_idx + 1}/{total_chunks}", - ], - sources=[source], - ) - rawfile_node.metadata.summary_ids = [mem_node.id for mem_node in fine_items] - fine_items.append(rawfile_node) + # Save the original file chunk independently from prompt selection. + file_source = next( + ( + source + for source in sources + if ( + source.get("type") + if isinstance(source, dict) + else getattr(source, "type", None) + ) + == "file" + ), + None, + ) + if self.save_rawfile and file_source is not None and fine_items: + if isinstance(file_source, dict): + file_info = file_source.get("file_info") or {} + file_content = file_source.get("content") + else: + file_info = file_source.file_info or {} + file_content = file_source.content + + rawfile_chunk = file_content or file_info.get("file_data", "") + if rawfile_chunk: + source = self.multi_modal_parser.file_content_parser.create_source( + message={"file": file_info}, + info=info_per_item, + chunk_index=chunk_idx, + chunk_total=total_chunks, + chunk_content=rawfile_chunk, + ) + rawfile_node = self._make_memory_item( + value=rawfile_chunk, + info=info_per_item, + memory_type="RawFileMemory", + tags=[ + "mode:fine", + "multimodal:file", + f"chunk:{chunk_idx + 1}/{total_chunks}", + ], + sources=[source], + ) + rawfile_node.metadata.summary_ids = [mem_node.id for mem_node in fine_items] + fine_items.append(rawfile_node) + else: + logger.warning("[RawFile] File source has no content; skipping raw-file node") enriched_items = trigger_hook( H.MEMORY_ITEMS_AFTER_FINE_EXTRACT, items=fine_items, diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index a954794b7..50031b7aa 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -209,7 +209,7 @@ def _process_memories_with_reader( for memory in flattened_memories if memory.metadata.memory_type == "RawFileMemory" ] - text_mem.add_rawfile_nodes_n_edges( + text_mem.add_rawfile_nodes( raw_file_mem_group, enhanced_mem_ids, user_id=user_id, diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index 23474cc20..1ba148d8a 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -212,6 +212,14 @@ class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata): default_factory=list, description="The ids of the files associated with the memory.", ) + knowledge_material_ids: list[str] = Field( + default_factory=list, + description="RawFileMemory ids that support this knowledge summary.", + ) + knowledge_summary_ids: list[str] = Field( + default_factory=list, + description="Knowledge summary ids extracted from this raw file memory.", + ) @field_validator("sources", mode="before") @classmethod diff --git a/src/memos/memories/textual/tree.py b/src/memos/memories/textual/tree.py index 426a2359b..c9d74d43f 100644 --- a/src/memos/memories/textual/tree.py +++ b/src/memos/memories/textual/tree.py @@ -399,10 +399,13 @@ def get_all( ) return graph_output - def delete(self, memory_ids: list[str], user_name: str | None = None) -> None: + def delete(self, memory_ids: str | list[str], user_name: str | None = None) -> None: """Hard delete: permanently remove nodes and their edges from the graph.""" if not memory_ids: return + if isinstance(memory_ids, str): + memory_ids = [memory_ids] + self._unlink_knowledge_relations(memory_ids, user_name=user_name) for mid in memory_ids: try: self.graph_store.delete_node(mid, user_name=user_name) @@ -412,6 +415,7 @@ def delete(self, memory_ids: list[str], user_name: str | None = None) -> None: def delete_by_memory_ids(self, memory_ids: list[str]) -> None: """Delete memories by memory_ids.""" try: + self._unlink_knowledge_relations(memory_ids) self.graph_store.delete_node_by_prams(memory_ids=memory_ids) except Exception as e: logger.error(f"An error occurred while deleting memories by memory_ids: {e}") @@ -432,10 +436,71 @@ def delete_by_filter( filter: dict | None = None, ) -> None: """Delete memories by filter.""" + if file_ids: + user_names = writable_cube_ids or [None] + for user_name in user_names: + memory_ids: set[str] = set() + for file_id in file_ids: + memory_ids.update( + self.graph_store.get_by_metadata( + [{"field": "file_ids", "op": "contains", "value": file_id}], + user_name=user_name, + ) + ) + self._unlink_knowledge_relations(list(memory_ids), user_name=user_name) self.graph_store.delete_node_by_prams( writable_cube_ids=writable_cube_ids, file_ids=file_ids, filter=filter ) + @staticmethod + def _node_metadata(node: dict[str, Any] | TextualMemoryItem) -> dict[str, Any]: + if isinstance(node, TextualMemoryItem): + return node.metadata.model_dump() + return node.get("metadata") or {} + + def _get_node_map( + self, memory_ids: list[str] | set[str], user_name: str | None = None + ) -> dict[str, dict[str, Any] | TextualMemoryItem]: + if not memory_ids: + return {} + nodes = self.graph_store.get_nodes(ids=list(memory_ids), user_name=user_name) or [] + return { + node.id if isinstance(node, TextualMemoryItem) else node["id"]: node for node in nodes + } + + def _unlink_knowledge_relations( + self, memory_ids: list[str], user_name: str | None = None + ) -> None: + """Remove reverse knowledge ids before deleting or deactivating nodes.""" + deleting_ids = set(memory_ids) + deleting_nodes = self._get_node_map(deleting_ids, user_name=user_name) + removals: dict[str, dict[str, set[str]]] = {} + + for memory_id, node in deleting_nodes.items(): + metadata = self._node_metadata(node) + for material_id in metadata.get("knowledge_material_ids") or []: + if material_id not in deleting_ids: + removals.setdefault(material_id, {}).setdefault( + "knowledge_summary_ids", set() + ).add(memory_id) + for summary_id in metadata.get("knowledge_summary_ids") or []: + if summary_id not in deleting_ids: + removals.setdefault(summary_id, {}).setdefault( + "knowledge_material_ids", set() + ).add(memory_id) + + related_nodes = self._get_node_map(set(removals), user_name=user_name) + for related_id, fields in removals.items(): + node = related_nodes.get(related_id) + if node is None: + continue + metadata = self._node_metadata(node) + updates = { + field: [value for value in metadata.get(field) or [] if value not in removed_ids] + for field, removed_ids in fields.items() + } + self.graph_store.update_node(related_id, updates, user_name=user_name) + def load(self, dir: str, user_name: str | None = None) -> None: try: memory_file = os.path.join(dir, self.config.memory_filename) @@ -593,6 +658,109 @@ def add_rawfile_nodes_n_edges( f"[RawFile] Time taken to add edges: {end_time - start_time} seconds for {len(from_ids)} edges" ) + def add_rawfile_nodes( + self, + raw_file_mem_group: list[TextualMemoryItem], + mem_ids: list[str], + user_id: str | None = None, + user_name: str | None = None, + ) -> None: + """Persist raw-file nodes and their knowledge relationships without graph edges.""" + valid_summary_ids = set(mem_ids) + summary_to_material_ids: dict[str, list[str]] = {} + + for raw_file_mem in raw_file_mem_group: + summary_ids = list( + dict.fromkeys( + summary_id + for summary_id in getattr(raw_file_mem.metadata, "summary_ids", []) or [] + if summary_id in valid_summary_ids + ) + ) + raw_file_mem.metadata.knowledge_summary_ids = summary_ids + for summary_id in summary_ids: + summary_to_material_ids.setdefault(summary_id, []).append(raw_file_mem.id) + + self.add(raw_file_mem_group, user_name=user_name) + summary_nodes = self._get_node_map(set(summary_to_material_ids), user_name=user_name) + + merged_from_by_summary: dict[str, list[str]] = {} + old_summary_ids: set[str] = set() + for summary_id, node in summary_nodes.items(): + info = self._node_metadata(node).get("info") or {} + merged_from = info.get("merged_from") or [] + if isinstance(merged_from, str): + merged_from = [merged_from] + merged_from_by_summary[summary_id] = list(dict.fromkeys(merged_from)) + old_summary_ids.update(merged_from) + + old_summary_nodes = self._get_node_map(old_summary_ids, user_name=user_name) + material_relinks: dict[str, dict[str, Any]] = {} + + for summary_id, direct_material_ids in summary_to_material_ids.items(): + summary_node = summary_nodes.get(summary_id) + if summary_node is None: + logger.warning( + "Knowledge summary %s was not found after raw-file write", summary_id + ) + continue + + metadata = self._node_metadata(summary_node) + merged_from = merged_from_by_summary.get(summary_id, []) + inherited_material_ids: list[str] = [] + for old_summary_id in merged_from: + old_summary = old_summary_nodes.get(old_summary_id) + if old_summary is not None: + inherited_material_ids.extend( + self._node_metadata(old_summary).get("knowledge_material_ids") or [] + ) + + material_ids = list( + dict.fromkeys( + [ + *(metadata.get("knowledge_material_ids") or []), + *direct_material_ids, + *inherited_material_ids, + ] + ) + ) + self.graph_store.update_node( + summary_id, + {"knowledge_material_ids": material_ids}, + user_name=user_name, + ) + + for material_id in inherited_material_ids: + relink = material_relinks.setdefault(material_id, {"remove": set(), "add": []}) + relink["remove"].update(merged_from) + relink["add"].append(summary_id) + + material_nodes = self._get_node_map(set(material_relinks), user_name=user_name) + for material_id, relink in material_relinks.items(): + material_node = material_nodes.get(material_id) + if material_node is None: + continue + current_summary_ids = ( + self._node_metadata(material_node).get("knowledge_summary_ids") or [] + ) + updated_summary_ids = [ + summary_id + for summary_id in current_summary_ids + if summary_id not in relink["remove"] + ] + updated_summary_ids.extend(relink["add"]) + self.graph_store.update_node( + material_id, + {"knowledge_summary_ids": list(dict.fromkeys(updated_summary_ids))}, + user_name=user_name, + ) + + logger.info( + "[RawFile] Added %s chunks with ID mappings for user %s", + len(raw_file_mem_group), + user_id, + ) + def add_graph_edges( self, from_ids: list[str], to_ids: list[str], types: list[str], user_name: str | None = None ) -> None: @@ -624,6 +792,7 @@ def soft_delete( user_name: str, evolve_to_ids: list[str] | None = None, ) -> None: + self._unlink_knowledge_relations(memory_ids, user_name=user_name) # for ruff check... if not evolve_to_ids: update_fields = {"status": "deleted"} diff --git a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py index 5c9cce78e..b44d28218 100644 --- a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py +++ b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py @@ -3,8 +3,6 @@ import re import traceback -from concurrent.futures import as_completed - from memos.context.context import ContextThreadPoolExecutor from memos.embedders.factory import OllamaEmbedder from memos.graph_dbs.factory import Neo4jGraphDB @@ -1282,7 +1280,7 @@ def _sort_and_trim( @timed def _deduplicate_rawfile_results(self, results, user_name: str | None = None): """ - Deduplicate rawfile related memories by edge + Deduplicate raw-file results using their persisted summary ids. """ if not results: return results @@ -1292,29 +1290,10 @@ def _deduplicate_rawfile_results(self, results, user_name: str | None = None): if not rawfile_items: return results - with ContextThreadPoolExecutor(max_workers=min(len(rawfile_items), 10)) as executor: - futures = [ - executor.submit( - self.graph_store.get_edges, - rawfile_item.id, - type="SUMMARY", - direction="OUTGOING", - user_name=user_name, - ) - for rawfile_item in rawfile_items - ] - for future in as_completed(futures): - try: - edges = future.result() - for edge in edges: - summary_target_id = edge.get("to") - if summary_target_id: - summary_ids_to_remove.add(summary_target_id) - logger.debug( - f"[DEDUP] Marking summary node {summary_target_id} for removal (pointed by RawFileMemory)" - ) - except Exception as e: - logger.warning(f"[DEDUP] Failed to get summary target ids: {e}") + for rawfile_item in rawfile_items: + summary_ids_to_remove.update( + getattr(rawfile_item.metadata, "knowledge_summary_ids", None) or [] + ) filtered_results = [] for item in results: diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py index 27399259a..0b9829b7f 100644 --- a/src/memos/multi_mem_cube/single_cube.py +++ b/src/memos/multi_mem_cube/single_cube.py @@ -441,22 +441,35 @@ def extract_edge_info(edges_info: list[dict], neighbor_relativity: float): ) return edge_mems + def get_neighbor(chunk_target_id: str | None, neighbor_relativity: float): + if not chunk_target_id: + return None + item_neighbor = self.searcher.graph_store.get_node(chunk_target_id, user_name=user_name) + if not item_neighbor: + return None + item_neighbor_mem = TextualMemoryItem(**item_neighbor) + item_neighbor_mem.metadata.relativity = neighbor_relativity + self.logger.info("Add neighbor chunk by id: %s", chunk_target_id) + return item_neighbor_mem + final_items = [] if neighbor_discovery: for item in search_results: if item.metadata.memory_type == "RawFileMemory": neighbor_relativity = item.metadata.relativity * 0.8 - preceding_info = self.searcher.graph_store.get_edges( - item.id, type="PRECEDING", direction="OUTGOING", user_name=user_name + preceding = get_neighbor( + getattr(item.metadata, "preceding_id", None), neighbor_relativity ) - final_items.extend(extract_edge_info(preceding_info, neighbor_relativity)) + if preceding is not None: + final_items.append(preceding) final_items.append(item) - following_info = self.searcher.graph_store.get_edges( - item.id, type="FOLLOWING", direction="OUTGOING", user_name=user_name + following = get_neighbor( + getattr(item.metadata, "following_id", None), neighbor_relativity ) - final_items.extend(extract_edge_info(following_info, neighbor_relativity)) + if following is not None: + final_items.append(following) else: final_items.append(item) @@ -748,7 +761,7 @@ def _process_text_mem( for memory in flattened_local if memory.metadata.memory_type == "RawFileMemory" ] - self.naive_mem_cube.text_mem.add_rawfile_nodes_n_edges( + self.naive_mem_cube.text_mem.add_rawfile_nodes( raw_file_mem_group, mem_ids_local, user_id=add_req.user_id, diff --git a/tests/mem_cube/test_single_cube_knowledge_neighbors.py b/tests/mem_cube/test_single_cube_knowledge_neighbors.py new file mode 100644 index 000000000..5cb433d77 --- /dev/null +++ b/tests/mem_cube/test_single_cube_knowledge_neighbors.py @@ -0,0 +1,50 @@ +import uuid + +from unittest.mock import Mock + +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata + + +def make_raw(item_id: str, relativity: float, **extra) -> TextualMemoryItem: + return TextualMemoryItem( + id=item_id, + memory=item_id, + metadata=TreeNodeTextualMemoryMetadata( + memory_type="RawFileMemory", + relativity=relativity, + **extra, + ), + ) + + +def test_postformat_discovers_rawfile_neighbors_by_id_without_edges(): + import memos.api.handlers # noqa: F401 - initialize package before known circular import + + from memos.multi_mem_cube.single_cube import SingleCubeView + + previous = make_raw(str(uuid.uuid4()), 0.1) + following = make_raw(str(uuid.uuid4()), 0.1) + current = make_raw(str(uuid.uuid4()), 1.0, preceding_id=previous.id, following_id=following.id) + searcher = Mock() + neighbors = { + previous.id: previous.model_dump(), + following.id: following.model_dump(), + } + searcher.graph_store.get_node.side_effect = lambda node_id, **_: neighbors.get(node_id) + view = SingleCubeView( + cube_id="cube-1", + naive_mem_cube=Mock(), + mem_reader=Mock(), + mem_scheduler=Mock(), + logger=Mock(), + searcher=searcher, + ) + + result = view._postformat_memories( + [current], "cube-1", include_embedding=False, neighbor_discovery=True + ) + + assert [item["id"] for item in result] == [previous.id, current.id, following.id] + assert result[0]["metadata"]["relativity"] == 0.8 + assert result[2]["metadata"]["relativity"] == 0.8 + searcher.graph_store.get_edges.assert_not_called() diff --git a/tests/mem_feedback/test_feedback.py b/tests/mem_feedback/test_feedback.py index bbf30ae94..3f86ac482 100644 --- a/tests/mem_feedback/test_feedback.py +++ b/tests/mem_feedback/test_feedback.py @@ -1,3 +1,5 @@ +import uuid + from unittest.mock import Mock from memos.mem_feedback.feedback import MemFeedback @@ -139,3 +141,21 @@ def test_update_judgement_prompt_omits_memory_text_from_response_schema(): assert '"reason"' in output_section assert '"text"' not in output_section assert '"old_memory"' not in output_section + + +def test_retrieve_locks_only_knowledge_summaries_without_edge_queries(): + feedback = MemFeedback.__new__(MemFeedback) + feedback.pref_feedback = False + feedback.searcher = Mock() + normal = TextualMemoryItem(id=str(uuid.uuid4()), memory="normal") + knowledge = TextualMemoryItem( + id=str(uuid.uuid4()), + memory="knowledge", + metadata={"knowledge_material_ids": [str(uuid.uuid4())]}, + ) + feedback.searcher.search.return_value = [(normal, 0.9), (knowledge, 0.8)] + + result = feedback._retrieve("query", user_name="cube-1") + + assert result == [normal] + feedback.searcher.graph_store.get_edges.assert_not_called() diff --git a/tests/mem_reader/test_multi_modal_rawfile.py b/tests/mem_reader/test_multi_modal_rawfile.py new file mode 100644 index 000000000..f583b7173 --- /dev/null +++ b/tests/mem_reader/test_multi_modal_rawfile.py @@ -0,0 +1,89 @@ +from unittest.mock import MagicMock, patch + +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) + + +def test_mixed_file_source_creates_rawfile_without_changing_prompt_type(): + reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + reader.save_rawfile = True + reader.memory_version_switch = "off" + reader.embedder = MagicMock() + reader.embedder.embed.return_value = [[0.1]] + reader.multi_modal_parser = MagicMock() + + file_info = { + "file_id": "file-1", + "filename": "knowledge.txt", + "file_data": "Complete file content", + } + file_source = SourceMessage( + type="file", + role=None, + content="Current file chunk", + file_info=file_info, + ) + fast_item = TextualMemoryItem( + memory="Please parse this file\nCurrent file chunk\nRecord it completely", + metadata=TreeNodeTextualMemoryMetadata( + user_id="user-1", + session_id="session-1", + memory_type="LongTermMemory", + sources=[ + SourceMessage(type="chat", role="user", content="Please parse this file"), + file_source, + SourceMessage(type="chat", role="user", content="Record it completely"), + ], + file_ids=[file_info["file_id"]], + ), + ) + reader.multi_modal_parser.file_content_parser.create_source.return_value = SourceMessage( + type="file", + content=file_source.content, + doc_path=file_info["filename"], + ) + + llm_response = { + "memory list": [ + { + "key": "Knowledge summary", + "memory_type": "LongTermMemory", + "value": "Summarized knowledge", + "tags": ["knowledge"], + } + ], + "summary": "Summary background", + } + + with ( + patch.object(reader, "_get_llm_response", return_value=llm_response) as get_llm_response, + patch.object( + reader, + "_get_maybe_merged_memory", + side_effect=lambda extracted_memory_dict, **kwargs: extracted_memory_dict, + ), + patch( + "memos.mem_reader.multi_modal_struct.trigger_hook", + side_effect=lambda _hook, **kwargs: kwargs["items"], + ), + ): + result = reader._process_string_fine( + [fast_item], + {"user_id": "user-1", "session_id": "session-1"}, + ) + + summaries = [item for item in result if item.metadata.memory_type != "RawFileMemory"] + rawfiles = [item for item in result if item.metadata.memory_type == "RawFileMemory"] + + assert get_llm_response.call_args.args[3] == "chat" + assert len(summaries) == 1 + assert len(rawfiles) == 1 + assert rawfiles[0].memory == file_source.content + assert rawfiles[0].metadata.summary_ids == [summaries[0].id] + assert reader.multi_modal_parser.file_content_parser.create_source.call_args.kwargs[ + "message" + ] == {"file": file_info} diff --git a/tests/memories/textual/test_tree_knowledge_relations.py b/tests/memories/textual/test_tree_knowledge_relations.py new file mode 100644 index 000000000..79bd80743 --- /dev/null +++ b/tests/memories/textual/test_tree_knowledge_relations.py @@ -0,0 +1,182 @@ +import uuid + +from unittest.mock import Mock + +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.memories.textual.tree import TreeTextMemory + + +class FakeGraphStore: + def __init__(self, items: list[TextualMemoryItem]) -> None: + self.nodes = {item.id: item.model_dump() for item in items} + self.edge_calls: list[tuple] = [] + self.deleted_by_params: list[dict] = [] + + def get_nodes(self, ids, user_name=None): + return [self.nodes[node_id] for node_id in ids if node_id in self.nodes] + + def update_node(self, id, fields, user_name=None): + self.nodes[id]["metadata"].update(fields) + + def delete_node(self, id, user_name=None): + self.nodes.pop(id, None) + + def delete_node_by_prams(self, **kwargs): + self.deleted_by_params.append(kwargs) + for node_id in kwargs.get("memory_ids") or []: + self.nodes.pop(node_id, None) + + def get_by_metadata(self, filters, user_name=None, status=None): + matched = [] + for node_id, node in self.nodes.items(): + metadata = node["metadata"] + if all( + ( + item["value"] in (metadata.get(item["field"]) or []) + if item.get("op") == "contains" + else metadata.get(item["field"]) == item["value"] + ) + for item in filters + ): + matched.append(node_id) + return matched + + def add_edge(self, *args, **kwargs): + self.edge_calls.append((args, kwargs)) + + def get_edges(self, *args, **kwargs): + self.edge_calls.append((args, kwargs)) + return [] + + +def _test_id(value: str) -> str: + try: + return str(uuid.UUID(value)) + except ValueError: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, value)) + + +def make_item( + item_id: str, + memory_type: str, + *, + knowledge_material_ids: list[str] | None = None, + knowledge_summary_ids: list[str] | None = None, + info: dict | None = None, + file_ids: list[str] | None = None, + summary_ids: list[str] | None = None, +) -> TextualMemoryItem: + metadata = TreeNodeTextualMemoryMetadata( + memory_type=memory_type, + knowledge_material_ids=[_test_id(value) for value in knowledge_material_ids or []], + knowledge_summary_ids=[_test_id(value) for value in knowledge_summary_ids or []], + info=( + { + **info, + "merged_from": [_test_id(value) for value in info.get("merged_from", [])], + } + if info and "merged_from" in info + else info + ), + file_ids=file_ids or [], + ) + if summary_ids is not None: + metadata.summary_ids = [_test_id(value) for value in summary_ids] + return TextualMemoryItem(id=_test_id(item_id), memory=item_id, metadata=metadata) + + +def make_tree(items: list[TextualMemoryItem]) -> tuple[TreeTextMemory, FakeGraphStore]: + tree = TreeTextMemory.__new__(TreeTextMemory) + graph_store = FakeGraphStore(items) + tree.graph_store = graph_store + + def add(memories, user_name=None, **kwargs): + for memory in memories: + graph_store.nodes[memory.id] = memory.model_dump() + return [memory.id for memory in memories] + + tree.add = Mock(side_effect=add) + return tree, graph_store + + +def test_add_rawfile_nodes_persists_bidirectional_ids_without_edges(): + summary = make_item("summary-1", "LongTermMemory") + material = make_item("material-1", "RawFileMemory", summary_ids=[summary.id]) + tree, graph_store = make_tree([summary]) + + tree.add_rawfile_nodes([material], [summary.id], user_name="cube-1") + + assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [summary.id] + assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [material.id] + assert graph_store.edge_calls == [] + + +def test_add_rawfile_nodes_reconnects_materials_when_summary_is_merged(): + old_material = make_item("material-old", "RawFileMemory", knowledge_summary_ids=["summary-old"]) + old_summary = make_item( + "summary-old", "LongTermMemory", knowledge_material_ids=[old_material.id] + ) + new_summary = make_item("summary-new", "LongTermMemory", info={"merged_from": [old_summary.id]}) + new_material = make_item("material-new", "RawFileMemory", summary_ids=[new_summary.id]) + tree, graph_store = make_tree([old_material, old_summary, new_summary]) + + tree.add_rawfile_nodes([new_material], [new_summary.id], user_name="cube-1") + + assert graph_store.nodes[new_summary.id]["metadata"]["knowledge_material_ids"] == [ + new_material.id, + old_material.id, + ] + assert graph_store.nodes[old_material.id]["metadata"]["knowledge_summary_ids"] == [ + new_summary.id + ] + + +def test_delete_summary_unlinks_material_without_deleting_it(): + material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"]) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.delete_by_memory_ids([summary.id]) + + assert summary.id not in graph_store.nodes + assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [] + + +def test_delete_material_unlinks_summary_without_deleting_it(): + material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"]) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.delete_by_memory_ids([material.id]) + + assert material.id not in graph_store.nodes + assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [] + + +def test_file_delete_unlinks_knowledge_relations_before_bulk_delete(): + material = make_item( + "material-1", + "RawFileMemory", + knowledge_summary_ids=["summary-1"], + file_ids=["file-1"], + ) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.delete_by_filter(writable_cube_ids=["cube-1"], file_ids=["file-1"]) + + assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [] + assert graph_store.deleted_by_params == [ + {"writable_cube_ids": ["cube-1"], "file_ids": ["file-1"], "filter": None} + ] + + +def test_soft_delete_unlinks_summary_before_marking_it_deleted(): + material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"]) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.soft_delete([summary.id], user_name="cube-1") + + assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [] + assert graph_store.nodes[summary.id]["metadata"]["status"] == "deleted" diff --git a/tests/memories/textual/test_tree_searcher.py b/tests/memories/textual/test_tree_searcher.py index b79958ca1..fe40cbb7d 100644 --- a/tests/memories/textual/test_tree_searcher.py +++ b/tests/memories/textual/test_tree_searcher.py @@ -1,3 +1,5 @@ +import uuid + from unittest.mock import MagicMock import pytest @@ -143,3 +145,26 @@ def test_searcher_respects_memory_type(mock_searcher): ) # WorkingMemory triggers only once path A assert mock_searcher.graph_retriever.retrieve.call_args[1]["memory_scope"] == "WorkingMemory" + + +def test_deduplicate_rawfile_results_uses_knowledge_summary_ids(mock_searcher): + summary_id = str(uuid.uuid4()) + material_id = str(uuid.uuid4()) + summary = TextualMemoryItem( + id=summary_id, + memory="summary", + metadata=TreeNodeTextualMemoryMetadata(memory_type="LongTermMemory"), + ) + raw_file = TextualMemoryItem( + id=material_id, + memory="material", + metadata=TreeNodeTextualMemoryMetadata( + memory_type="RawFileMemory", + knowledge_summary_ids=[summary.id], + ), + ) + + result = mock_searcher._deduplicate_rawfile_results([summary, raw_file], user_name="cube-1") + + assert result == [raw_file] + mock_searcher.graph_store.get_edges.assert_not_called() From 721416d2c49eb09dd1c55dcacb70a6217ec79c8a Mon Sep 17 00:00:00 2001 From: keeponlight Date: Sat, 17 Oct 2026 19:58:04 +0800 Subject: [PATCH 05/15] fix(scheduler): align enhanced_mem_ids with filtered mem_group in mem_read_handler --- .../handlers/mem_read_handler.py | 4 +- .../test_mem_read_handler_fast_cleanup.py | 147 +++++++++++++++++- 2 files changed, 143 insertions(+), 8 deletions(-) diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index 50031b7aa..5e663c99e 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -328,9 +328,7 @@ def _process_memories_with_reader( add_meta_legacy: list[dict] = [] update_content_legacy: list[dict] = [] update_meta_legacy: list[dict] = [] - for item_id, item in zip( - enhanced_mem_ids, flattened_memories, strict=False - ): + for item_id, item in zip(enhanced_mem_ids, mem_group, strict=False): key = getattr(item.metadata, "key", None) or transform_name_to_key( name=item.memory ) diff --git a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py index c54b49de6..8cdb873ce 100644 --- a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py +++ b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py @@ -4,11 +4,18 @@ from unittest.mock import MagicMock -def _make_text_item(item_id: str, memory: str): +def _make_text_item( + item_id: str, + memory: str, + *, + memory_type: str = "LongTermMemory", + key: str | None = None, + info: dict | None = None, +): metadata = SimpleNamespace( - memory_type="LongTermMemory", - info={}, - key=None, + memory_type=memory_type, + info=info if info is not None else {}, + key=key, status="activated", confidence=0.9, tags=[], @@ -17,8 +24,10 @@ def _make_text_item(item_id: str, memory: str): class _FakeTextMem: - def __init__(self, items): + def __init__(self, items, added_ids: list[str] | None = None): self._items = {item.id: item for item in items} + self._added_ids = added_ids + self.add_calls: list = [] self.delete_calls: list = [] self.soft_delete_calls: list = [] self.memory_manager = SimpleNamespace( @@ -29,6 +38,13 @@ def __init__(self, items): def get(self, mem_id, user_name=None): return self._items[mem_id] + def add(self, memories, user_name=None): + mem_list = list(memories) + self.add_calls.append((mem_list, user_name)) + if self._added_ids is not None: + return list(self._added_ids) + return [item.id for item in mem_list] + def delete(self, ids, user_name=None): self.delete_calls.append((list(ids), user_name)) @@ -98,3 +114,124 @@ def test_fine_transfer_error_does_not_soft_delete_fast_memory_with_versions(): assert text_mem.delete_calls == [] assert text_mem.soft_delete_calls == [] + + +def test_playground_legacy_events_align_enhanced_ids_with_filtered_mem_group(monkeypatch): + import memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler as handler_module + + from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import ( + MemReadMessageHandler, + ) + + monkeypatch.setattr(handler_module, "is_playground_api", lambda: True) + + raw_chunk = _make_text_item("raw_1", "raw chunk") + raw_file_mem = _make_text_item( + "raw_file_1", + "raw file content", + memory_type="RawFileMemory", + key="raw-file-key", + ) + added_mem = _make_text_item( + "mem_add_1", + "user prefers dark mode", + memory_type="LongTermMemory", + key="theme-pref", + ) + updated_mem = _make_text_item( + "mem_upd_1", + "user lives in Shanghai", + memory_type="UserMemory", + key="city-pref", + info={"merged_from": ["old_mem_1"]}, + ) + + text_mem = _FakeTextMem( + [raw_chunk], + added_ids=["enhanced_id_add_1", "enhanced_id_upd_1"], + ) + mem_cube = SimpleNamespace(text_mem=text_mem) + + mem_reader = MagicMock() + mem_reader.fine_transfer_simple_mem.return_value = [[raw_file_mem, added_mem, updated_mem]] + mem_reader.memory_version_switch = "off" + mem_reader.graph_db = None + mem_reader.save_rawfile = False + + submitted_events: list = [] + services = SimpleNamespace( + create_event_log=lambda **kwargs: SimpleNamespace(**kwargs), + submit_web_logs=lambda events, **kwargs: submitted_events.extend(events), + map_memcube_name=lambda mem_cube_id: "UserMemCube", + submit_messages=lambda messages: None, + ) + scheduler_context = SimpleNamespace( + get_mem_cube=lambda: mem_cube, + get_mem_reader=lambda: mem_reader, + services=services, + ) + + handler = MemReadMessageHandler.__new__(MemReadMessageHandler) + handler.scheduler_context = scheduler_context + + handler._process_memories_with_reader( + mem_ids=["raw_1"], + user_id="user_1", + mem_cube_id="cube_1", + text_mem=text_mem, + user_name="cube_1", + task_id="task_1", + info={"trigger_source": "Messages"}, + ) + + assert len(text_mem.add_calls) == 1 + assert text_mem.add_calls[0] == ([added_mem, updated_mem], "cube_1") + + assert len(submitted_events) == 2 + add_event, update_event = submitted_events + + assert add_event.label == "addMemory" + assert add_event.task_id == "task_1" + assert add_event.memory_len == 1 + assert add_event.memcube_log_content == [ + { + "content": "theme-pref: user prefers dark mode", + "ref_id": "enhanced_id_add_1", + } + ] + assert add_event.metadata == [ + { + "ref_id": "enhanced_id_add_1", + "id": "enhanced_id_add_1", + "key": "theme-pref", + "memory": "user prefers dark mode", + "memory_type": "LongTermMemory", + "status": "activated", + "confidence": 0.9, + "tags": [], + "updated_at": None, + } + ] + + assert update_event.label == "updateMemory" + assert update_event.task_id == "task_1" + assert update_event.memory_len == 1 + assert update_event.memcube_log_content == [ + { + "content": "city-pref: user lives in Shanghai", + "ref_id": "enhanced_id_upd_1", + } + ] + assert update_event.metadata == [ + { + "ref_id": "enhanced_id_upd_1", + "id": "enhanced_id_upd_1", + "key": "city-pref", + "memory": "user lives in Shanghai", + "memory_type": "UserMemory", + "status": "activated", + "confidence": 0.9, + "tags": [], + "updated_at": None, + } + ] From 9b4fea36cf26102b10f07cec40e8389180e7e2ab Mon Sep 17 00:00:00 2001 From: keeponlight Date: Sat, 17 Oct 2026 19:54:09 +0800 Subject: [PATCH 06/15] fix(llm): prevent IndexError when reasoning items are absent in responses API --- src/memos/llms/openai_new.py | 4 +- tests/llms/test_openai_responses_llm.py | 104 ++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 tests/llms/test_openai_responses_llm.py diff --git a/src/memos/llms/openai_new.py b/src/memos/llms/openai_new.py index 766a17fda..50c87b78f 100644 --- a/src/memos/llms/openai_new.py +++ b/src/memos/llms/openai_new.py @@ -50,7 +50,7 @@ def generate(self, messages: MessageList, **kwargs) -> str: output_reasoning = [ item for item in response.output if isinstance(item, ResponseReasoningItem) ] - summary = output_reasoning[0].summary + summary = output_reasoning[0].summary if output_reasoning else None if self.config.remove_think_prefix: return remove_thinking_tags(output_text) @@ -138,7 +138,7 @@ def generate(self, messages: MessageList, **kwargs) -> str: output_reasoning = [ item for item in response.output if isinstance(item, ResponseReasoningItem) ] - summary = output_reasoning[0].summary + summary = output_reasoning[0].summary if output_reasoning else None if self.config.remove_think_prefix: return remove_thinking_tags(output_text) diff --git a/tests/llms/test_openai_responses_llm.py b/tests/llms/test_openai_responses_llm.py new file mode 100644 index 000000000..d6915b122 --- /dev/null +++ b/tests/llms/test_openai_responses_llm.py @@ -0,0 +1,104 @@ +"""Tests for OpenAIResponsesLLM and AzureResponsesLLM generate reasoning handling.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary + +from memos.configs.llm import AzureLLMConfig, OpenAIResponsesLLMConfig +from memos.llms.openai_new import AzureResponsesLLM, OpenAIResponsesLLM + + +MESSAGES = [{"role": "user", "content": "Hello"}] + + +def _make_openai_llm( + *, enable_thinking: bool = False, remove_think_prefix: bool = False +) -> OpenAIResponsesLLM: + llm = OpenAIResponsesLLM.__new__(OpenAIResponsesLLM) + llm.config = OpenAIResponsesLLMConfig( + api_key="test-key", + model_name_or_path="gpt-4o", + enable_thinking=enable_thinking, + remove_think_prefix=remove_think_prefix, + ) + llm.client = MagicMock() + return llm + + +def _make_azure_llm( + *, enable_thinking: bool = False, remove_think_prefix: bool = False +) -> AzureResponsesLLM: + llm = AzureResponsesLLM.__new__(AzureResponsesLLM) + llm.config = AzureLLMConfig( + api_key="test-key", + model_name_or_path="gpt-4o", + enable_thinking=enable_thinking, + remove_think_prefix=remove_think_prefix, + ) + llm.client = MagicMock() + return llm + + +@pytest.mark.parametrize( + "make_llm", + [_make_openai_llm, _make_azure_llm], + ids=["openai", "azure"], +) +class TestResponsesLLMGenerate: + def test_generate_without_reasoning_item_returns_output_text(self, make_llm): + llm = make_llm(enable_thinking=False) + llm.client.responses.create.return_value = SimpleNamespace( + output=[SimpleNamespace(type="message")], + output_text="plain answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "plain answer" + + def test_generate_with_empty_reasoning_summary_returns_output_text(self, make_llm): + llm = make_llm(enable_thinking=True) + reasoning_item = ResponseReasoningItem(id="rs_1", summary=[], type="reasoning") + llm.client.responses.create.return_value = SimpleNamespace( + output=[reasoning_item], + output_text="plain answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "plain answer" + + def test_generate_with_reasoning_summary_prepends_think_tag(self, make_llm): + llm = make_llm(enable_thinking=True, remove_think_prefix=False) + reasoning_item = ResponseReasoningItem( + id="rs_1", + summary=[Summary(text="step by step", type="summary_text")], + type="reasoning", + ) + llm.client.responses.create.return_value = SimpleNamespace( + output=[reasoning_item], + output_text="final answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "step by stepfinal answer" + + def test_generate_with_remove_think_prefix_strips_thinking(self, make_llm): + llm = make_llm(enable_thinking=True, remove_think_prefix=True) + reasoning_item = ResponseReasoningItem( + id="rs_1", + summary=[Summary(text="step by step", type="summary_text")], + type="reasoning", + ) + llm.client.responses.create.return_value = SimpleNamespace( + output=[reasoning_item], + output_text="inline thoughtfinal answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "final answer" From fe70153bffac15f0e04503510b1cd0d82f90eca7 Mon Sep 17 00:00:00 2001 From: lils <310442317+linsheng-li@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:43:32 +0800 Subject: [PATCH 07/15] fix(memory): decode JSON-encoded internal_info metadata --- src/memos/memories/textual/item.py | 10 ++++ tests/memories/textual/test_item.py | 91 +++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/memories/textual/test_item.py diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index 23474cc20..9ad5369e7 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -159,6 +159,16 @@ class TextualMemoryMetadata(BaseModel): description="Internal algorithm metadata reserved for system use.", ) + @field_validator("internal_info", mode="before") + @classmethod + def _decode_internal_info(cls, value: Any) -> Any: + # Neo4j stores dictionaries as JSON strings; decode them before model validation. + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, dict): + raise ValueError("internal_info JSON must decode to an object") + return value + model_config = ConfigDict(extra="allow") covered_history: Any | None = Field( diff --git a/tests/memories/textual/test_item.py b/tests/memories/textual/test_item.py new file mode 100644 index 000000000..71c206742 --- /dev/null +++ b/tests/memories/textual/test_item.py @@ -0,0 +1,91 @@ +"""Regression tests for JSON-encoded internal metadata read from graph storage.""" + +import copy +import json +import uuid + +import pytest + +from pydantic import ValidationError + +from memos.memories.textual.item import ( + PreferenceTextualMemoryMetadata, + SearchedTreeNodeTextualMemoryMetadata, + TextualMemoryItem, + TextualMemoryMetadata, + TreeNodeTextualMemoryMetadata, +) + + +@pytest.mark.parametrize( + "metadata_cls", + [ + TextualMemoryMetadata, + TreeNodeTextualMemoryMetadata, + SearchedTreeNodeTextualMemoryMetadata, + PreferenceTextualMemoryMetadata, + ], +) +def test_internal_info_decodes_json_object(metadata_cls): + value = {"ingest_batch_id": "test-batch", "nested": {"chunk_index": 0, "values": [True, None]}} + + metadata = metadata_cls(internal_info=json.dumps(value)) + + assert metadata.internal_info == value + assert metadata.model_dump()["internal_info"] == value + + +@pytest.mark.parametrize("value", [None, {}, {"ingest_batch_id": "test-batch", "nested": [1, 2]}]) +def test_internal_info_preserves_existing_values(value): + original = copy.deepcopy(value) + + metadata = TextualMemoryMetadata(internal_info=value) + + assert metadata.internal_info == original + assert value == original + + +@pytest.mark.parametrize( + "value", + ["{", "not-json", "[]", "null", "1", "true", '"text"', json.dumps('{"batch": "test"}')], +) +def test_internal_info_rejects_malformed_json_and_encoded_non_objects(value): + with pytest.raises(ValidationError): + TextualMemoryMetadata(internal_info=value) + + +@pytest.mark.parametrize("value", [[], [["batch", "test"]], 1, False]) +def test_internal_info_preserves_rejection_of_invalid_types(value): + with pytest.raises(ValidationError): + TextualMemoryMetadata(internal_info=value) + + +def test_search_item_from_dict_decodes_internal_info_without_changing_other_fields(): + value = {"ingest_batch_id": "test-batch", "chunk_index": 0, "chunk_total": 2} + record = { + "id": str(uuid.uuid4()), + "memory": "Synthetic memory for a graph storage round trip.", + "metadata": { + "memory_type": "LongTermMemory", + "internal_info": json.dumps(value), + "relativity": 0.75, + "sources": [{"type": "chat", "role": "user", "content": "Synthetic source."}], + "session_id": "test-session", + }, + } + original = copy.deepcopy(record) + + item = TextualMemoryItem.from_dict(record) + + assert item.metadata.internal_info == value + assert item.id == record["id"] + assert item.memory == record["memory"] + assert item.metadata.sources[0].content == "Synthetic source." + assert item.metadata.session_id == "test-session" + assert item.metadata.relativity == 0.75 + assert record == original + + # Graph storage serializes dictionaries as JSON strings again on the next write. + stored = item.to_dict() + stored["metadata"]["internal_info"] = json.dumps(stored["metadata"]["internal_info"]) + assert TextualMemoryItem.from_dict(stored).to_dict() == item.to_dict() From c8aed140ddedef45f10e9aac1e79b741c3366d09 Mon Sep 17 00:00:00 2001 From: Zyntrael <2650876010@qq.com> Date: Wed, 21 Oct 2026 21:26:47 +0800 Subject: [PATCH 08/15] feat: align PostgreSQL graph storage with current v2.0.36 APIs (#2531) * feat: align PostgreSQL graph storage with current v2.0.36 APIs Store the existing graph interface on MD5 tenant shards so this branch can run on PostgreSQL without attribute-tree tables. Co-authored-by: Cursor * style: format PostgreSQL graph files for Ruff The upstream check stops on ruff format before the tests run. Co-authored-by: Cursor * ci: retrigger pull request checks The Windows Python 3.11 dependency install failed once and needs another run. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- src/memos/api/config.py | 6 +- src/memos/configs/graph_db.py | 33 +- src/memos/graph_dbs/postgres.py | 2154 ++++++++++++++++--- tests/graph_dbs/test_postgres_400_compat.py | 391 ++++ tests/graph_dbs/test_postgres_pgvector.py | 461 ++++ 5 files changed, 2712 insertions(+), 333 deletions(-) create mode 100644 tests/graph_dbs/test_postgres_400_compat.py create mode 100644 tests/graph_dbs/test_postgres_pgvector.py diff --git a/src/memos/api/config.py b/src/memos/api/config.py index 0247dfa42..f09e51335 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -906,8 +906,8 @@ def get_polardb_config(user_id: str | None = None) -> dict[str, Any]: def get_postgres_config(user_id: str | None = None) -> dict[str, Any]: """Get PostgreSQL + pgvector configuration for MemOS graph storage. - Uses standard PostgreSQL with pgvector extension. - Schema: memos.memories, memos.edges + Uses standard PostgreSQL with pgvector. Shard count, index, and + full-text configuration keep the backend defaults unless overridden. """ user_name = os.getenv("MEMOS_USER_NAME", "default") if user_id: @@ -922,7 +922,7 @@ def get_postgres_config(user_id: str | None = None) -> dict[str, Any]: "schema_name": os.getenv("MEMOS_SCHEMA", "memos"), "user_name": user_name, "use_multi_db": False, - "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "384")), + "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "1024")), "maxconn": int(os.getenv("POSTGRES_MAX_CONN", "20")), } diff --git a/src/memos/configs/graph_db.py b/src/memos/configs/graph_db.py index 98de09812..984c95060 100644 --- a/src/memos/configs/graph_db.py +++ b/src/memos/configs/graph_db.py @@ -1,4 +1,4 @@ -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from pydantic import BaseModel, Field, field_validator, model_validator @@ -195,8 +195,7 @@ class PostgresGraphDBConfig(BaseConfig): Does NOT require Apache AGE or other graph extensions. Schema: - - memos_memories: Main table for memory nodes (id, memory, properties JSONB, embedding vector) - - memos_edges: Edge table for relationships (source_id, target_id, type) + - {schema}_graph_{n}."Memory" and "Edges", one physical shard per tenant Example: --- @@ -223,13 +222,33 @@ class PostgresGraphDBConfig(BaseConfig): default=False, description="If False: use single database with logical isolation by user_name", ) - embedding_dimension: int = Field( - default=768, description="Dimension of vector embedding (768 for all-mpnet-base-v2)" + embedding_dimension: int = Field(default=1024, description="Dimension of vector embedding") + shard_count: int = Field( + default=400, + ge=1, + le=4096, + description="Number of MD5 tenant shards", ) maxconn: int = Field( default=20, description="Maximum number of connections in the connection pool", ) + minconn: int = Field( + default=8, + ge=1, + description="Warm connections kept in the pool", + ) + vector_index_type: Literal["hnsw", "ivfflat"] = Field( + default="hnsw", + description="pgvector approximate index type", + ) + hnsw_m: int = Field(default=16, ge=2, le=100) + hnsw_ef_construction: int = Field(default=64, ge=4, le=1000) + ivfflat_lists: int = Field(default=100, ge=1) + fulltext_config: str = Field( + default="jiebacfg", + description="PostgreSQL text search configuration used to build search_vector", + ) @model_validator(mode="after") def validate_config(self): @@ -238,6 +257,10 @@ def validate_config(self): raise ValueError("`db_name` must be provided") if not self.use_multi_db and not self.user_name: raise ValueError("In single-database mode, `user_name` must be provided") + if self.minconn > self.maxconn: + raise ValueError(f"`minconn` ({self.minconn}) must be <= `maxconn` ({self.maxconn})") + if not self.fulltext_config.replace(".", "_").isidentifier(): + raise ValueError("`fulltext_config` must be a valid PostgreSQL configuration name") return self diff --git a/src/memos/graph_dbs/postgres.py b/src/memos/graph_dbs/postgres.py index 594f7e695..2820a8b54 100644 --- a/src/memos/graph_dbs/postgres.py +++ b/src/memos/graph_dbs/postgres.py @@ -5,16 +5,18 @@ No Apache AGE or other graph extensions required. Tables: -- {schema}.memories: Memory nodes with JSONB properties and vector embeddings -- {schema}.edges: Relationships between memory nodes +- {schema}_graph_{n}."Memory": one tenant shard of memory nodes +- {schema}_graph_{n}."Edges": relationships for that same shard """ +import hashlib import json +import random import re import time from contextlib import suppress -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Literal from memos.configs.graph_db import PostgresGraphDBConfig @@ -28,7 +30,7 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: """Ensure metadata has proper datetime fields and normalized types.""" - now = datetime.utcnow().isoformat() + now = datetime.now(timezone.utc).isoformat() metadata.setdefault("created_at", now) metadata.setdefault("updated_at", now) @@ -43,6 +45,9 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: class PostgresGraphDB(BaseGraphDB): """PostgreSQL + pgvector implementation of a graph memory store.""" + supports_lightweight_vector_search = True + prefer_lightweight_vector_search = True + @require_python_package( import_name="psycopg2", install_command="pip install psycopg2-binary", @@ -50,26 +55,29 @@ class PostgresGraphDB(BaseGraphDB): ) def __init__(self, config: PostgresGraphDBConfig): """Initialize PostgreSQL connection pool.""" - import psycopg2 import psycopg2.pool self.config = config + if not self._is_safe_field_name(config.schema_name): + raise ValueError(f"Invalid PostgreSQL schema name: {config.schema_name}") self.schema = config.schema_name + self.db_name = config.db_name self.user_name = config.user_name self._pool_closed = False + self._partition_remainders: dict[str, int] = {} logger.info(f"Connecting to PostgreSQL: {config.host}:{config.port}/{config.db_name}") # Create connection pool self.pool = psycopg2.pool.ThreadedConnectionPool( - minconn=2, + minconn=config.minconn, maxconn=config.maxconn, host=config.host, port=config.port, user=config.user, password=config.password, dbname=config.db_name, - connect_timeout=30, + connect_timeout=10, keepalives_idle=30, keepalives_interval=10, keepalives_count=5, @@ -78,8 +86,39 @@ def __init__(self, config: PostgresGraphDBConfig): # Initialize schema and tables self._init_schema() + def warm_up_search_connections_by_full(self, user_name: str | None = None) -> None: + """Warm one pooled PostgreSQL connection; tenant routing needs no fan-out.""" + conn = self._get_conn() + self._put_conn(conn) + + def get_memory_graph_table_name(self, user_name: str | None) -> str: + """Return the one partition table for this tenant, matching PolarDB shard routing.""" + target = user_name or self.user_name + return self._physical_table("memories", target) + + def create_extension(self) -> None: + self._init_schema() + + def create_graph(self) -> None: + self._init_schema() + + def create_edge(self) -> None: + self._init_schema() + + def create_index( + self, + label: str = "Memory", + vector_property: str = "embedding", + dimensions: int = 1024, + index_name: str = "memory_vector_index", + ) -> None: + self._init_schema() + + def format_param_value(self, value: str | None) -> str: + return "null" if value is None else json.dumps(value, ensure_ascii=False) + def _get_conn(self): - """Get connection from pool with health check.""" + """Get a pooled connection without an extra round trip.""" if self._pool_closed: raise RuntimeError("Connection pool is closed") @@ -91,9 +130,6 @@ def _get_conn(self): self.pool.putconn(conn, close=True) continue conn.autocommit = True - # Health check - with conn.cursor() as cur: - cur.execute("SELECT 1") return conn except Exception as e: if conn: @@ -114,66 +150,145 @@ def _put_conn(self, conn): conn.close() def _init_schema(self): - """Create schema and tables if they don't exist.""" + """Create PolarDB-style MD5 shards with a halfvec Memory table in each.""" + fulltext_config = getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_.]*$", fulltext_config): + raise ValueError(f"Invalid PostgreSQL text search config: {fulltext_config}") + vector_index_type = getattr(self.config, "vector_index_type", "hnsw") + shard_count = int(getattr(self.config, "shard_count", 400)) + if shard_count < 1 or shard_count > 4096: + raise ValueError("PostgreSQL shard_count must be between 1 and 4096") + conn = self._get_conn() try: with conn.cursor() as cur: - # Create schema cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}") - - # Enable pgvector cur.execute("CREATE EXTENSION IF NOT EXISTS vector") - # Create memories table - dim = self.config.embedding_dimension - cur.execute(f""" - CREATE TABLE IF NOT EXISTS {self.schema}.memories ( - id TEXT PRIMARY KEY, - memory TEXT NOT NULL DEFAULT '', - properties JSONB NOT NULL DEFAULT '{{}}', - embedding vector({dim}), - user_name TEXT, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - """) - - # Create edges table - cur.execute(f""" - CREATE TABLE IF NOT EXISTS {self.schema}.edges ( - id SERIAL PRIMARY KEY, - source_id TEXT NOT NULL, - target_id TEXT NOT NULL, - edge_type TEXT NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(source_id, target_id, edge_type) + cur.execute( + """ + SELECT c.relname, c.relkind + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = %s + AND c.relname = ANY(%s) + """, + ( + self.schema, + ["memories", "edges"], + ), + ) + legacy_rows = cur.fetchall() + if isinstance(legacy_rows, list) and legacy_rows: + raise RuntimeError( + f"{self.schema} still has the old memories layout; " + "configure a new MEMOS_SCHEMA for the MD5 shard layout " + "or migrate the existing data" ) - """) - - # Create indexes - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_user - ON {self.schema}.memories(user_name) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_props - ON {self.schema}.memories USING GIN(properties) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_embedding - ON {self.schema}.memories USING ivfflat(embedding vector_cosine_ops) - WITH (lists = 100) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_edges_source - ON {self.schema}.edges(source_id) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_edges_target - ON {self.schema}.edges(target_id) - """) - - logger.info(f"Schema {self.schema} initialized successfully") + + dim = int(self.config.embedding_dimension) + cur.execute( + """ + SELECT COUNT(*) + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE c.relname = 'Memory' + AND c.relkind = 'r' + AND n.nspname ~ %s + """, + (rf"^{re.escape(self.schema)}_graph_[0-9]+$",), + ) + count_row = cur.fetchone() + memory_tables = count_row[0] if count_row and isinstance(count_row[0], int) else -1 + if memory_tables != shard_count: + hnsw_m = int(getattr(self.config, "hnsw_m", 16)) + ef_construction = int(getattr(self.config, "hnsw_ef_construction", 64)) + lists = int(getattr(self.config, "ivfflat_lists", 100)) + statements: list[str] = [] + for remainder in range(shard_count): + shard_schema = f"{self.schema}_graph_{remainder}" + memory = f'"{shard_schema}"."Memory"' + edges = f'"{shard_schema}"."Edges"' + if vector_index_type == "ivfflat": + vector_index = ( + f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec " + f"ON {memory} USING ivfflat (embedding halfvec_cosine_ops) " + f"WITH (lists = {lists})" + ) + else: + vector_index = ( + f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec " + f"ON {memory} USING hnsw (embedding halfvec_cosine_ops) " + f"WITH (m = {hnsw_m}, ef_construction = {ef_construction})" + ) + statements.extend( + [ + f'CREATE SCHEMA IF NOT EXISTS "{shard_schema}"', + f""" + CREATE TABLE IF NOT EXISTS {memory} ( + id TEXT NOT NULL, + memory TEXT NOT NULL DEFAULT '', + properties JSONB NOT NULL DEFAULT '{{}}', + embedding halfvec({dim}), + user_name TEXT NOT NULL, + search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector( + '{fulltext_config}', + coalesce(memory, '') || ' ' || coalesce(properties::text, '') + ) + ) STORED, + deleted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (user_name, id) + ) + """, + f""" + CREATE TABLE IF NOT EXISTS {edges} ( + id BIGSERIAL NOT NULL, + user_name TEXT NOT NULL, + source_id TEXT NOT NULL, + target_id TEXT NOT NULL, + edge_type TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (user_name, id), + UNIQUE(user_name, source_id, target_id, edge_type) + ) + """, + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_user " + f"ON {memory}(user_name)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_props " + f"ON {memory} USING GIN(properties)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_search_vector " + f"ON {memory} USING GIN(search_vector)" + ), + vector_index, + ( + f"CREATE INDEX IF NOT EXISTS idx_edges_source " + f"ON {edges}(user_name, source_id)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_edges_target " + f"ON {edges}(user_name, target_id)" + ), + ( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_tenant_unique " + f"ON {edges}(user_name, source_id, target_id, edge_type)" + ), + ] + ) + cur.execute(";\n".join(statements)) + + logger.info( + "Schema %s initialized successfully with %d MD5 shards", + self.schema, + shard_count, + ) except Exception as e: logger.error(f"Failed to init schema: {e}") raise @@ -197,6 +312,8 @@ def remove_oldest_memory( """ user_name = user_name or self.user_name keep_latest = int(keep_latest) + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: @@ -206,7 +323,7 @@ def remove_oldest_memory( f""" WITH ranked AS ( SELECT id, ROW_NUMBER() OVER (ORDER BY updated_at DESC) as rn - FROM {self.schema}.memories + FROM {memories} WHERE user_name = %s AND properties->>'memory_type' = %s ) @@ -221,101 +338,140 @@ def remove_oldest_memory( # Delete edges first cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) + DELETE FROM {edges} + WHERE user_name = %s + AND (source_id = ANY(%s) OR target_id = ANY(%s)) """, - (ids_to_delete, ids_to_delete), + (user_name, ids_to_delete, ids_to_delete), ) # Delete nodes cur.execute( f""" - DELETE FROM {self.schema}.memories - WHERE id = ANY(%s) + DELETE FROM {memories} + WHERE id = ANY(%s) AND user_name = %s """, - (ids_to_delete,), + (ids_to_delete, user_name), ) logger.info( - f"Removed {len(ids_to_delete)} oldest {memory_type} memories for user {user_name}" + "Removed %d oldest %s memories for user %s", + len(ids_to_delete), + memory_type, + user_name, ) finally: self._put_conn(conn) - def add_node( - self, id: str, memory: str, metadata: dict[str, Any], user_name: str | None = None - ) -> None: - """Add a memory node.""" - user_name = user_name or self.user_name - metadata = _prepare_node_metadata(metadata.copy()) - - # Extract embedding - embedding = metadata.pop("embedding", None) - created_at = metadata.pop("created_at", datetime.utcnow().isoformat()) - updated_at = metadata.pop("updated_at", datetime.utcnow().isoformat()) + def _prepare_node_for_insert( + self, + node_id: str, + memory: str, + metadata: dict[str, Any] | None, + user_name: str, + ) -> tuple[Any, ...]: + """Build one write row with the same property shape PolarDB stores.""" + fields = _prepare_node_metadata(dict(metadata or {})) + fields["user_name"] = user_name + fields.setdefault("delete_time", "") + fields.setdefault("delete_record_id", "") + now = datetime.now(timezone.utc).isoformat() + created_at = fields.pop("created_at", now) + updated_at = fields.pop("updated_at", now) + for field_name in ("sources", "usage"): + value = fields.get(field_name) + if isinstance(value, list): + fields[field_name] = [ + item if isinstance(item, str) else json.dumps(item) for item in value + ] + embedding = fields.pop("embedding", None) + if not isinstance(embedding, list) or not embedding: + embedding = [round(random.uniform(-0.2, 0.2), 6) for _ in range(self._vector_dim())] + else: + embedding = [float(item) for item in embedding] + properties = { + "id": node_id, + "memory": memory, + "created_at": created_at, + "updated_at": updated_at, + "delete_time": fields.get("delete_time", ""), + "delete_record_id": fields.get("delete_record_id", ""), + **fields, + } + return ( + node_id, + memory, + json.dumps(properties, ensure_ascii=False), + embedding, + user_name, + created_at, + updated_at, + ) - # Serialize sources if present - if metadata.get("sources"): - metadata["sources"] = [ - json.dumps(s) if not isinstance(s, str) else s for s in metadata["sources"] - ] + def add_node(self, id: str, memory: str, metadata: dict[str, Any], user_name: str) -> None: + """Add a memory node.""" + if not user_name: + raise ValueError("add_node requires user_name") + self.add_nodes_batch( + [{"id": id, "memory": memory, "metadata": metadata}], + user_name, + ) + def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None: + """Insert every node for one tenant in a single statement.""" + if not nodes: + return + if not user_name: + raise ValueError("add_nodes_batch requires user_name") + rows: list[tuple[Any, ...]] = [] + for node in nodes: + try: + rows.append( + self._prepare_node_for_insert( + node["id"], + node["memory"], + node.get("metadata", {}), + user_name, + ) + ) + except Exception: + logger.error( + "add_nodes_batch failed to prepare node %s", + node.get("id", "unknown"), + exc_info=True, + ) + if not rows: + return + memories = self._physical_table("memories", user_name) + dim = self._vector_dim() + values_sql = ",".join([f"(%s, %s, %s, %s::halfvec({dim}), %s, %s, %s)"] * len(rows)) + params: list[Any] = [value for row in rows for value in row] conn = self._get_conn() try: with conn.cursor() as cur: - if embedding: - cur.execute( - f""" - INSERT INTO {self.schema}.memories - (id, memory, properties, embedding, user_name, created_at, updated_at) - VALUES (%s, %s, %s, %s::vector, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - memory = EXCLUDED.memory, - properties = EXCLUDED.properties, - embedding = EXCLUDED.embedding, - updated_at = EXCLUDED.updated_at - """, - ( - id, - memory, - json.dumps(metadata), - embedding, - user_name, - created_at, - updated_at, - ), - ) - else: - cur.execute( - f""" - INSERT INTO {self.schema}.memories - (id, memory, properties, user_name, created_at, updated_at) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - memory = EXCLUDED.memory, - properties = EXCLUDED.properties, - updated_at = EXCLUDED.updated_at + cur.execute( + f""" + INSERT INTO {memories} + (id, memory, properties, embedding, user_name, created_at, updated_at) + VALUES {values_sql} + ON CONFLICT (user_name, id) DO UPDATE SET + memory = EXCLUDED.memory, + properties = EXCLUDED.properties, + embedding = EXCLUDED.embedding, + updated_at = EXCLUDED.updated_at + WHERE "Memory".user_name = EXCLUDED.user_name """, - (id, memory, json.dumps(metadata), user_name, created_at, updated_at), - ) + params, + ) finally: self._put_conn(conn) - def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str | None = None) -> None: - """Batch add memory nodes.""" - for node in nodes: - self.add_node( - id=node["id"], - memory=node["memory"], - metadata=node.get("metadata", {}), - user_name=user_name, - ) - def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = None) -> None: """Update node fields.""" user_name = user_name or self.user_name if not fields: return + fields = fields.copy() # Get current node current = self.get_node(id, user_name=user_name) @@ -327,7 +483,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N embedding = fields.pop("embedding", None) memory = fields.pop("memory", current.get("memory", "")) props.update(fields) - props["updated_at"] = datetime.utcnow().isoformat() + props["updated_at"] = datetime.now(timezone.utc).isoformat() + memories = self._physical_table("memories", user_name) conn = self._get_conn() try: @@ -335,8 +492,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N if embedding: cur.execute( f""" - UPDATE {self.schema}.memories - SET memory = %s, properties = %s, embedding = %s::vector, updated_at = NOW() + UPDATE {memories} + SET memory = %s, properties = %s, embedding = %s::halfvec({self._vector_dim()}), updated_at = NOW() WHERE id = %s AND user_name = %s """, (memory, json.dumps(props), embedding, id, user_name), @@ -344,7 +501,7 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N else: cur.execute( f""" - UPDATE {self.schema}.memories + UPDATE {memories} SET memory = %s, properties = %s, updated_at = NOW() WHERE id = %s AND user_name = %s """, @@ -356,21 +513,23 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N def delete_node(self, id: str, user_name: str | None = None) -> None: """Delete a node and its edges.""" user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: # Delete edges cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = %s OR target_id = %s + DELETE FROM {edges} + WHERE user_name = %s AND (source_id = %s OR target_id = %s) """, - (id, id), + (user_name, id, id), ) # Delete node cur.execute( f""" - DELETE FROM {self.schema}.memories + DELETE FROM {memories} WHERE id = %s AND user_name = %s """, (id, user_name), @@ -378,22 +537,34 @@ def delete_node(self, id: str, user_name: str | None = None) -> None: finally: self._put_conn(conn) - def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[str, Any] | None: + def get_node( + self, id: str, include_embedding: bool = False, user_name: str | None = None + ) -> dict[str, Any] | None: """Get a single node by ID.""" - user_name = kwargs.get("user_name") or self.user_name + relation = self._tenant_relation("memories", [user_name] if user_name else []) conn = self._get_conn() try: with conn.cursor() as cur: cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" - cur.execute( - f""" - SELECT {cols} FROM {self.schema}.memories - WHERE id = %s AND user_name = %s - """, - (id, user_name), - ) + cols += ", user_name" + if user_name: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = %s AND user_name = %s + """, + (id, user_name), + ) + else: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = %s + """, + (id,), + ) row = cur.fetchone() if not row: return None @@ -402,32 +573,77 @@ def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[s self._put_conn(conn) def get_nodes( - self, ids: list, include_embedding: bool = False, **kwargs + self, ids: list[str], user_name: str | None = None, **kwargs ) -> list[dict[str, Any]]: """Get multiple nodes by IDs.""" if not ids: return [] - user_name = kwargs.get("user_name") or self.user_name + include_embedding = bool(kwargs.get("include_embedding", False)) + relation = self._tenant_relation("memories", [user_name] if user_name else []) conn = self._get_conn() try: with conn.cursor() as cur: cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" - cur.execute( - f""" - SELECT {cols} FROM {self.schema}.memories - WHERE id = ANY(%s) AND user_name = %s - """, - (ids, user_name), - ) + cols += ", user_name" + if user_name: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = ANY(%s) AND user_name = %s + AND COALESCE(properties->>'status', 'activated') <> 'archived' + """, + (ids, user_name), + ) + else: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = ANY(%s) + AND COALESCE(properties->>'status', 'activated') <> 'archived' + """, + (ids,), + ) return [self._parse_row(row, include_embedding) for row in cur.fetchall()] finally: self._put_conn(conn) + @staticmethod + def _normalize_node_properties( + raw_properties: dict[str, Any], user_name: str | None = None + ) -> dict[str, Any]: + """Restore the logical PolarDB metadata shape from relational storage.""" + properties = dict(raw_properties) + if user_name: + properties["user_name"] = user_name + + sources = properties.get("sources") + if isinstance(sources, list): + normalized_sources: list[Any] = [] + for source in sources: + if isinstance(source, str): + try: + parsed_source = json.loads(source) + except (json.JSONDecodeError, TypeError): + parsed_source = {"type": "doc", "content": source} + normalized_sources.append(parsed_source) + elif isinstance(source, dict): + normalized_sources.append(source) + else: + normalized_sources.append({"type": "doc", "content": str(source)}) + properties["sources"] = normalized_sources + + return properties + def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]: """Parse database row to node dict.""" - props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + raw_props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + user_name_index = 6 if include_embedding else 5 + user_name = row[user_name_index] if len(row) > user_name_index else None + props = self._normalize_node_properties(raw_props, user_name=user_name) + props["id"] = row[0] + props["memory"] = row[1] or "" props["created_at"] = row[3].isoformat() if row[3] else None props["updated_at"] = row[4].isoformat() if row[4] else None result = { @@ -436,9 +652,21 @@ def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]: "metadata": props, } if include_embedding and len(row) > 5: - result["metadata"]["embedding"] = row[5] + result["metadata"]["embedding"] = self._coerce_embedding(row[5]) return result + @staticmethod + def _coerce_embedding(value: Any) -> list[float]: + """Normalize pgvector values returned as lists, tuples, or text.""" + if value is None: + return [] + if isinstance(value, str): + value = value.strip().strip("[]") + if not value: + return [] + return [float(item) for item in value.split(",")] + return [float(item) for item in value] + @staticmethod def _is_safe_field_name(field: str) -> bool: """Validate field names used in dynamic SQL fragments.""" @@ -486,31 +714,40 @@ def _build_single_filter_condition( op_map = {"gt": ">", "lt": "<", "gte": ">=", "lte": "<="} sql_op = op_map[op] if raw_key in timestamp_fields or raw_key.endswith("_at"): - parts.append(f"({text_expr})::timestamptz {sql_op} %s::timestamptz") + parts.append(f"({text_expr})::timestamp {sql_op} %s::timestamp") params.append(op_value) else: parts.append(f"NULLIF({text_expr}, '')::numeric {sql_op} %s") params.append(op_value) elif op == "contains": - if raw_key in array_fields: - parts.append(f"{json_expr} @> %s::jsonb") - params.append(json.dumps([op_value])) - else: - parts.append(f"{text_expr} ILIKE %s") - params.append(f"%{op_value}%") + values = op_value if isinstance(op_value, list) else [op_value] + or_parts = [] + for item in values: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([item], ensure_ascii=False)) + parts.append("(" + " OR ".join(or_parts) + ")") elif op == "in": if not isinstance(op_value, list): raise ValueError( - f"in operator expects list for '{key}', got {type(op_value).__name__}" + f"in operator expects list for '{key}', " + f"got {type(op_value).__name__}" ) - if raw_key in array_fields: - parts.append(f"{json_expr} ?| %s") - params.append([str(v) for v in op_value]) + if len(op_value) == 0: + parts.append("FALSE") + elif raw_key in array_fields or raw_key == "related_id": + or_parts = [] + for item in op_value: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([str(item)], ensure_ascii=False)) + parts.append("(" + " OR ".join(or_parts) + ")") else: parts.append(f"{text_expr} = ANY(%s)") params.append([str(v) for v in op_value]) elif op == "like": - parts.append(f"{text_expr} ILIKE %s") + parts.append(f"{text_expr} LIKE %s") + params.append(f"%{op_value}%") + elif op == "nolike": + parts.append(f"{text_expr} NOT LIKE %s") params.append(f"%{op_value}%") else: raise ValueError(f"Unsupported filter operator: {op}") @@ -532,6 +769,7 @@ def _build_single_filter_condition( def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[Any]) -> str: """Build SQL WHERE fragment from filter dict.""" + filter_dict = self.parse_filter(filter_dict) if not filter_dict: return "" @@ -542,7 +780,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A parts: list[str] = [] for cond in and_conditions: if isinstance(cond, dict): - cond_sql = self._build_single_filter_condition(cond, params) + cond_sql = self._build_filter_where_clause(cond, params) if cond_sql: parts.append(f"({cond_sql})") return " AND ".join(parts) @@ -554,7 +792,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A parts: list[str] = [] for cond in or_conditions: if isinstance(cond, dict): - cond_sql = self._build_single_filter_condition(cond, params) + cond_sql = self._build_filter_where_clause(cond, params) if cond_sql: parts.append(f"({cond_sql})") return f"({' OR '.join(parts)})" if parts else "" @@ -562,6 +800,55 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A cond_sql = self._build_single_filter_condition(filter_dict, params) return cond_sql or "" + def parse_filter(self, filter_dict: dict | None = None): + """Apply the same implicit info.* field routing as the PolarDB backend.""" + if filter_dict is None: + return None + full_fields = { + "id", + "key", + "tags", + "type", + "usage", + "memory", + "status", + "sources", + "user_id", + "graph_id", + "user_name", + "background", + "confidence", + "created_at", + "session_id", + "updated_at", + "memory_type", + "node_type", + "info", + "source", + "file_ids", + "project_id", + "manager_user_id", + "delete_time", + "related_id", + "message_ids", + } + + def normalize(condition: Any) -> Any: + if not isinstance(condition, dict): + return condition + normalized: dict[str, Any] = {} + for key, value in condition.items(): + if key.lower() in {"and", "or"} and isinstance(value, list): + normalized[key] = [normalize(item) for item in value] + continue + normalized_key = ( + key if key in full_fields or key.startswith("info.") else f"info.{key}" + ) + normalized[normalized_key] = value + return normalized + + return normalize(filter_dict) + def delete_node_by_prams( self, writable_cube_ids: list[str] | None = None, @@ -571,7 +858,8 @@ def delete_node_by_prams( ) -> int: """Delete nodes by memory_ids, file_ids, or filter.""" logger.info( - "[delete_node_by_prams] memory_ids: %s, file_ids: %s, filter: %s, writable_cube_ids: %s", + "[delete_node_by_prams] memory_ids: %s, file_ids: %s, " + "filter: %s, writable_cube_ids: %s", memory_ids, file_ids, filter, @@ -600,7 +888,8 @@ def delete_node_by_prams( if not where_conditions: logger.warning( - "[delete_node_by_prams] No nodes to delete (no memory_ids, file_ids, or filter provided)" + "[delete_node_by_prams] No nodes to delete " + "(no memory_ids, file_ids, or filter provided)" ) return 0 @@ -609,32 +898,148 @@ def delete_node_by_prams( params.append(writable_cube_ids) where_clause = " AND ".join(where_conditions) + memory_tables = self._physical_tables("memories", writable_cube_ids) + edge_tables = self._physical_tables("edges", writable_cube_ids) conn = self._get_conn() try: with conn.cursor() as cur: - query = f""" - WITH to_delete AS ( - SELECT id - FROM {self.schema}.memories - WHERE {where_clause} - ), - deleted_edges AS ( - DELETE FROM {self.schema}.edges e + deleted_count = 0 + for memories, edges in zip(memory_tables, edge_tables, strict=True): + query = f""" + WITH to_delete AS ( + SELECT id, user_name + FROM {memories} + WHERE {where_clause} + ), + deleted_edges AS ( + DELETE FROM {edges} e + USING to_delete d + WHERE e.user_name = d.user_name + AND (e.source_id = d.id OR e.target_id = d.id) + ) + DELETE FROM {memories} m USING to_delete d - WHERE e.source_id = d.id OR e.target_id = d.id - ) - DELETE FROM {self.schema}.memories m - USING to_delete d - WHERE m.id = d.id - """ - cur.execute(query, params) - deleted_count = cur.rowcount if cur.rowcount is not None else 0 + WHERE m.id = d.id + """ + cur.execute(query, params) + deleted_count += cur.rowcount if cur.rowcount is not None else 0 logger.info("[delete_node_by_prams] Deleted %s nodes", deleted_count) return deleted_count finally: self._put_conn(conn) + @staticmethod + def _normalize_cube_memory_groups( + cube_memory_groups: list[dict[str, Any]], + ) -> list[tuple[str, str]]: + """Normalize and de-duplicate (tenant, memory-id) requests.""" + pairs: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for group in cube_memory_groups: + cube_id = str(group.get("cube_id") or "").strip() + if not cube_id: + continue + for raw_memory_id in group.get("memory_ids") or []: + memory_id = str(raw_memory_id).strip() + pair = (cube_id, memory_id) + if memory_id and pair not in seen: + seen.add(pair) + pairs.append(pair) + return pairs + + def delete_node_by_cube_memory_groups(self, cube_memory_groups: list[dict[str, Any]]) -> int: + """Delete exactly the requested (cube, memory-id) pairs. + + PolarDB locates these pairs in one of 400 schemas. PostgreSQL keeps the + same isolation rule in a shared table by joining both user_name and id. + """ + pairs = self._normalize_cube_memory_groups(cube_memory_groups) + if not pairs: + return 0 + user_names = [pair[0] for pair in pairs] + memory_ids = [pair[1] for pair in pairs] + tenant_names = list(dict.fromkeys(user_names)) + memory_tables = self._physical_tables("memories", tenant_names) + edge_tables = self._physical_tables("edges", tenant_names) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + deleted_count = 0 + for memories, edges in zip(memory_tables, edge_tables, strict=True): + cur.execute( + f""" + WITH requested(user_name, memory_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ), to_delete AS ( + SELECT m.id, m.user_name + FROM {memories} m + JOIN requested + ON requested.user_name = m.user_name + AND requested.memory_id = m.id + ), deleted_edges AS ( + DELETE FROM {edges} e + USING to_delete d + WHERE e.user_name = d.user_name + AND (e.source_id = d.id OR e.target_id = d.id) + ), deleted AS ( + DELETE FROM {memories} m + USING to_delete d + WHERE m.id = d.id AND m.user_name = d.user_name + RETURNING 1 + ) + SELECT COUNT(*) FROM deleted + """, + (user_names, memory_ids), + ) + row = cur.fetchone() + deleted_count += int(row[0]) if row and row[0] is not None else 0 + return deleted_count + finally: + self._put_conn(conn) + + def get_user_names_by_cube_memory_groups( + self, cube_memory_groups: list[dict[str, Any]] + ) -> dict[str, str | None]: + """Resolve memory ownership without scanning unrelated tenants.""" + raw_ids = [ + str(memory_id).strip() + for group in cube_memory_groups + for memory_id in (group.get("memory_ids") or []) + if str(memory_id).strip() + ] + result: dict[str, str | None] = dict.fromkeys(raw_ids) + pairs = self._normalize_cube_memory_groups(cube_memory_groups) + if not pairs: + return result + + user_names = [pair[0] for pair in pairs] + memory_ids = [pair[1] for pair in pairs] + memories = self._tenant_relation("memories", list(dict.fromkeys(user_names)), alias="m") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH requested(user_name, memory_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ) + SELECT requested.memory_id, m.user_name + FROM requested + LEFT JOIN {memories} + ON m.id = requested.memory_id + AND m.user_name = requested.user_name + """, + (user_names, memory_ids), + ) + for memory_id, owner in cur.fetchall(): + if owner: + result[str(memory_id)] = str(owner) + return result + finally: + self._put_conn(conn) + # ========================================================================= # Edge Management # ========================================================================= @@ -643,16 +1048,36 @@ def add_edge( self, source_id: str, target_id: str, type: str, user_name: str | None = None ) -> None: """Create an edge between nodes.""" + user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - INSERT INTO {self.schema}.edges (source_id, target_id, edge_type) - VALUES (%s, %s, %s) - ON CONFLICT (source_id, target_id, edge_type) DO NOTHING + INSERT INTO {edges} + (user_name, source_id, target_id, edge_type) + SELECT %s, %s, %s, %s + WHERE EXISTS ( + SELECT 1 FROM {memories} + WHERE id = %s AND user_name = %s + ) AND EXISTS ( + SELECT 1 FROM {memories} + WHERE id = %s AND user_name = %s + ) + ON CONFLICT (user_name, source_id, target_id, edge_type) DO NOTHING """, - (source_id, target_id, type), + ( + user_name, + source_id, + target_id, + type, + source_id, + user_name, + target_id, + user_name, + ), ) finally: self._put_conn(conn) @@ -661,31 +1086,62 @@ def delete_edge( self, source_id: str, target_id: str, type: str, user_name: str | None = None ) -> None: """Delete an edge.""" + user_name = user_name or self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = %s AND target_id = %s AND edge_type = %s + DELETE FROM {edges} + WHERE user_name = %s + AND source_id = %s AND target_id = %s AND edge_type = %s """, - (source_id, target_id, type), + (user_name, source_id, target_id, type), ) finally: self._put_conn(conn) - def edge_exists(self, source_id: str, target_id: str, type: str) -> bool: + def edge_exists( + self, + source_id: str, + target_id: str, + type: str = "ANY", + direction: str = "OUTGOING", + user_name: str | None = None, + ) -> bool: """Check if edge exists.""" + user_name = user_name or self.user_name + direction = direction.upper() + if direction not in {"OUTGOING", "INCOMING", "ANY"}: + raise ValueError("direction must be OUTGOING, INCOMING, or ANY") + if direction == "INCOMING": + source_id, target_id = target_id, source_id + + direction_sql = "source_id = %s AND target_id = %s" + params: list[Any] = [user_name, source_id, target_id] + if direction == "ANY": + direction_sql = ( + "((source_id = %s AND target_id = %s) OR (source_id = %s AND target_id = %s))" + ) + params = [user_name, source_id, target_id, target_id, source_id] + + type_sql = "" + if type != "ANY": + type_sql = " AND edge_type = %s" + params.append(type) + + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - SELECT 1 FROM {self.schema}.edges - WHERE source_id = %s AND target_id = %s AND edge_type = %s + SELECT 1 FROM {edges} + WHERE user_name = %s AND {direction_sql}{type_sql} LIMIT 1 """, - (source_id, target_id, type), + params, ) return cur.fetchone() is not None finally: @@ -696,43 +1152,132 @@ def edge_exists(self, source_id: str, target_id: str, type: str) -> bool: # ========================================================================= def get_neighbors( - self, id: str, type: str, direction: Literal["in", "out", "both"] = "out" + self, + id: str, + type: str, + direction: Literal["in", "out", "both"] = "out", ) -> list[str]: """Get neighboring node IDs.""" + user_name = self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: if direction == "out": cur.execute( f""" - SELECT target_id FROM {self.schema}.edges - WHERE source_id = %s AND edge_type = %s + SELECT target_id FROM {edges} + WHERE user_name = %s AND source_id = %s AND edge_type = %s """, - (id, type), + (user_name, id, type), ) elif direction == "in": cur.execute( f""" - SELECT source_id FROM {self.schema}.edges - WHERE target_id = %s AND edge_type = %s + SELECT source_id FROM {edges} + WHERE user_name = %s AND target_id = %s AND edge_type = %s """, - (id, type), + (user_name, id, type), ) else: # both cur.execute( f""" - SELECT target_id FROM {self.schema}.edges WHERE source_id = %s AND edge_type = %s + SELECT target_id FROM {edges} + WHERE user_name = %s AND source_id = %s AND edge_type = %s UNION - SELECT source_id FROM {self.schema}.edges WHERE target_id = %s AND edge_type = %s + SELECT source_id FROM {edges} + WHERE user_name = %s AND target_id = %s AND edge_type = %s """, - (id, type, id, type), + (user_name, id, type, user_name, id, type), ) return [row[0] for row in cur.fetchall()] finally: self._put_conn(conn) + def get_children_with_embeddings( + self, id: str, user_name: str | None = None + ) -> list[dict[str, Any]]: + """Return PARENT-edge children with vectors for structure optimization.""" + user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT child.id, child.embedding, child.memory + FROM {edges} e + JOIN {memories} child + ON child.id = e.target_id AND child.user_name = e.user_name + WHERE e.user_name = %s + AND e.source_id = %s + AND e.edge_type = 'PARENT' + AND child.deleted_at IS NULL + ORDER BY e.id + """, + (user_name, id), + ) + return [ + { + "id": row[0], + "embedding": self._coerce_embedding(row[1]), + "memory": row[2] or "", + } + for row in cur.fetchall() + ] + finally: + self._put_conn(conn) + + def get_edges( + self, + id: str, + type: str = "ANY", + direction: str = "ANY", + user_name: str | None = None, + ) -> list[dict[str, str]]: + """Return tenant-scoped edges connected to a node.""" + user_name = user_name or self.user_name + direction = direction.upper() + if direction not in {"OUTGOING", "INCOMING", "ANY"}: + raise ValueError("direction must be OUTGOING, INCOMING, or ANY") + + conditions = ["user_name = %s"] + params: list[Any] = [user_name] + if direction == "OUTGOING": + conditions.append("source_id = %s") + params.append(id) + elif direction == "INCOMING": + conditions.append("target_id = %s") + params.append(id) + else: + conditions.append("(source_id = %s OR target_id = %s)") + params.extend([id, id]) + if type != "ANY": + conditions.append("edge_type = %s") + params.append(type) + + edges = self._physical_table("edges", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT source_id, target_id, edge_type + FROM {edges} + WHERE {" AND ".join(conditions)} + ORDER BY id + """, + params, + ) + return [{"from": row[0], "to": row[1], "type": row[2]} for row in cur.fetchall()] + finally: + self._put_conn(conn) + def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[str]: """Get path between nodes using recursive CTE.""" + user_name = self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: @@ -740,13 +1285,14 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s f""" WITH RECURSIVE path AS ( SELECT source_id, target_id, ARRAY[source_id] as nodes, 1 as depth - FROM {self.schema}.edges - WHERE source_id = %s + FROM {edges} + WHERE user_name = %s AND source_id = %s UNION ALL SELECT e.source_id, e.target_id, p.nodes || e.source_id, p.depth + 1 - FROM {self.schema}.edges e + FROM {edges} e JOIN path p ON e.source_id = p.target_id - WHERE p.depth < %s AND NOT e.source_id = ANY(p.nodes) + WHERE e.user_name = %s + AND p.depth < %s AND NOT e.source_id = ANY(p.nodes) ) SELECT nodes || target_id as full_path FROM path @@ -754,34 +1300,95 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s ORDER BY depth LIMIT 1 """, - (source_id, max_depth, target_id), + (user_name, source_id, user_name, max_depth, target_id), ) row = cur.fetchone() return row[0] if row else [] finally: self._put_conn(conn) - def get_subgraph(self, center_id: str, depth: int = 2) -> list[str]: - """Get subgraph around center node.""" + def get_subgraph( + self, + center_id: str, + depth: int = 2, + center_status: str = "activated", + user_name: str | None = None, + ) -> dict[str, Any]: + """Return the PolarDB-compatible local subgraph structure.""" + if not 1 <= depth <= 5: + raise ValueError("depth must be 1-5") + center_id = center_id.strip('"') + user_name = user_name or self.user_name + core_node = self.get_node(center_id, user_name=user_name) + if core_node is None: + return {"core_node": None, "neighbors": [], "edges": []} + actual_status = core_node.get("metadata", {}).get("status", "activated") + if center_status and actual_status != center_status: + return {"core_node": None, "neighbors": [], "edges": []} + + edges_table = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - WITH RECURSIVE subgraph AS ( - SELECT %s::text as node_id, 0 as level - UNION - SELECT CASE WHEN e.source_id = s.node_id THEN e.target_id ELSE e.source_id END, - s.level + 1 - FROM {self.schema}.edges e + WITH RECURSIVE subgraph(node_id, level, path) AS ( + SELECT %s::text, 0, ARRAY[%s::text] + UNION ALL + SELECT + CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END, + s.level + 1, + s.path || CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END + FROM {edges_table} e JOIN subgraph s ON (e.source_id = s.node_id OR e.target_id = s.node_id) - WHERE s.level < %s + WHERE e.user_name = %s + AND s.level < %s + AND NOT ( + CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END + ) = ANY(s.path) ) - SELECT DISTINCT node_id FROM subgraph - """, - (center_id, depth), + SELECT DISTINCT node_id FROM subgraph ORDER BY node_id + """, + (center_id, center_id, user_name, depth), ) - return [row[0] for row in cur.fetchall()] + node_ids = [str(row[0]) for row in cur.fetchall()] + neighbor_ids = [node_id for node_id in node_ids if node_id != center_id] + neighbors = ( + self.get_nodes(neighbor_ids, user_name=user_name) if neighbor_ids else [] + ) + + if node_ids: + cur.execute( + f""" + SELECT source_id, target_id, edge_type + FROM {edges_table} + WHERE user_name = %s + AND source_id = ANY(%s) + AND target_id = ANY(%s) + ORDER BY id + """, + (user_name, node_ids, node_ids), + ) + edges = [ + {"source": row[0], "target": row[1], "type": row[2]} + for row in cur.fetchall() + ] + else: + edges = [] + return { + "core_node": core_node, + "neighbors": neighbors, + "edges": edges, + } finally: self._put_conn(conn) @@ -789,95 +1396,638 @@ def get_context_chain(self, id: str, type: str = "FOLLOWS") -> list[str]: """Get ordered chain following relationship type.""" return self.get_neighbors(id, type, "out") + def get_neighbors_by_tag( + self, + tags: list[str], + exclude_ids: list[str], + top_k: int = 5, + min_overlap: int = 1, + include_embedding: bool = False, + user_name: str | None = None, + ) -> list[dict[str, Any]]: + """Find active tenant memories ranked by tag overlap.""" + if not tags or top_k <= 0: + return [] + user_name = user_name or self.user_name + params: list[Any] = [user_name] + exclude_sql = "" + if exclude_ids: + exclude_sql = "AND id <> ALL(%s)" + params.append(exclude_ids) + + cols = "id, memory, properties, created_at, updated_at" + if include_embedding: + cols += ", embedding" + cols += ", user_name" + memories = self._physical_table("memories", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT {cols} + FROM {memories} + WHERE user_name = %s + AND COALESCE(properties->>'status', 'activated') = 'activated' + AND COALESCE(properties->>'node_type', '') <> 'reasoning' + AND COALESCE(properties->>'memory_type', '') <> 'WorkingMemory' + {exclude_sql} + """, + params, + ) + ranked: list[tuple[int, dict[str, Any]]] = [] + requested_tags = set(tags) + for row in cur.fetchall(): + node = self._parse_row(row, include_embedding) + node_tags = node.get("metadata", {}).get("tags", []) + if isinstance(node_tags, str): + try: + node_tags = json.loads(node_tags) + except (TypeError, json.JSONDecodeError): + node_tags = [] + overlap = len(requested_tags.intersection(node_tags or [])) + if overlap >= min_overlap: + ranked.append((overlap, node)) + ranked.sort(key=lambda item: item[0], reverse=True) + return [node for _, node in ranked[:top_k]] + finally: + self._put_conn(conn) + # ========================================================================= # Search Operations # ========================================================================= - def search_by_embedding( + def _effective_user_names( self, - vector: list[float], - top_k: int = 5, - scope: str | None = None, - status: str | None = None, - threshold: float | None = None, - search_filter: dict | None = None, user_name: str | None = None, - filter: dict | None = None, knowledgebase_ids: list[str] | None = None, - **kwargs, - ) -> list[dict]: - """Search nodes by vector similarity using pgvector.""" - user_name = user_name or self.user_name + ) -> list[str]: + """Resolve the memory cubes that a query is allowed to read.""" + names: list[str] = [] + primary = user_name or self.user_name + if primary: + names.append(primary) + for knowledgebase_id in knowledgebase_ids or []: + if knowledgebase_id and knowledgebase_id not in names: + names.append(knowledgebase_id) + return names + + def _shard_count(self) -> int: + return int(getattr(self.config, "shard_count", 400)) + + def _remainder_cache(self) -> dict[str, int]: + cache = getattr(self, "_partition_remainders", None) + if cache is None: + cache = {} + self._partition_remainders = cache + return cache + + def _partition_remainder(self, user_name: str, conn=None) -> int: + """Return the PolarDB MD5 shard number for one tenant.""" + del conn + cache = self._remainder_cache() + cached = cache.get(user_name) + if cached is not None: + return cached + remainder = ( + int(hashlib.md5(user_name.encode("utf-8")).hexdigest(), 16) % self._shard_count() + ) + cache[user_name] = remainder + return remainder - # Build WHERE clause - conditions = ["embedding IS NOT NULL"] - params = [] + def _qualified_partition(self, table: str, remainder: int) -> str: + physical_names = { + "memories": "Memory", + "edges": "Edges", + } + physical = physical_names.get(table) + if physical is None: + raise ValueError(f"Invalid partitioned table: {table}") + schema = f"{self.schema}_graph_{remainder}" + if not self._is_safe_field_name(schema): + raise ValueError(f"Invalid partition schema: {schema}") + return f'"{schema}"."{physical}"' + + def _all_partitions_relation(self, table: str, alias: str | None = None) -> str: + """Read every hash partition explicitly, as PolarDB unions every shard.""" + branches = " UNION ALL ".join( + f"SELECT * FROM {self._qualified_partition(table, remainder)}" + for remainder in range(self._shard_count()) + ) + return f"({branches}) {alias or table}" - if user_name: + def _tenant_relation( + self, + table: str, + user_names: list[str] | None, + alias: str | None = None, + conn=None, + ) -> str: + """Return one child table, or a union of only the touched partitions.""" + names = [name for name in (user_names or []) if name] + if not names: + return self._all_partitions_relation(table, alias) + remainders = sorted({self._partition_remainder(name, conn) for name in names}) + if len(remainders) == 1: + relation = self._qualified_partition(table, remainders[0]) + else: + branches = " UNION ALL ".join( + f"SELECT * FROM {self._qualified_partition(table, remainder)}" + for remainder in remainders + ) + relation = f"({branches})" + if alias is None: + alias = table + return f"{relation} {alias}" if alias else relation + + def _physical_tables(self, table: str, user_names: list[str] | None, conn=None) -> list[str]: + """Real tables a write can target. A union subquery cannot be updated.""" + names = [name for name in (user_names or []) if name] + if not names: + return [ + self._qualified_partition(table, remainder) + for remainder in range(self._shard_count()) + ] + remainders = sorted({self._partition_remainder(name, conn) for name in names}) + return [self._qualified_partition(table, remainder) for remainder in remainders] + + def _physical_table(self, table: str, user_name: str | None, conn=None) -> str: + """Return the one partition table PolarDB would address for this tenant.""" + if not user_name: + return f"{self.schema}.{table}" + return self._qualified_partition(table, self._partition_remainder(user_name, conn)) + + def _vector_dim(self) -> int: + dim = int(getattr(self.config, "embedding_dimension", 1024)) + if dim < 1 or dim > 4096: + raise ValueError(f"Invalid embedding dimension: {dim}") + return dim + + def _ensure_hnsw_search_settings(self, conn) -> None: + """Match PolarDB's HNSW session settings once for each pooled connection.""" + ready = getattr(self, "_hnsw_ready_ids", None) + if ready is None: + ready = set() + self._hnsw_ready_ids = ready + marker = id(conn) + if marker in ready: + return + with conn.cursor() as cur: + cur.execute( + """ + SELECT set_config('hnsw.ef_search', '100', false), + set_config('hnsw.iterative_scan', 'relaxed_order', false) + """ + ) + ready.add(marker) + + def _projected_field_exprs(self, return_fields: list[str] | None) -> list[tuple[str, str]]: + projected: list[tuple[str, str]] = [] + for field in self._validate_return_fields(return_fields): + if field in {"id", "memory", "score", "user_name", "created_at", "updated_at"}: + continue + if field == "embedding": + projected.append((field, "embedding")) + else: + projected.append((field, f"properties->'{field}'")) + return projected + + @staticmethod + def _decode_projected_value(value: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return value + + def _build_projected_search_result( + self, row: tuple[Any, ...], extra_fields: list[str] + ) -> dict[str, Any]: + created_at, updated_at = row[3], row[4] + result: dict[str, Any] = { + "id": row[0], + "score": float(row[1]), + "memory": row[2] or "", + "user_name": row[5], + } + if created_at is not None: + result["created_at"] = ( + created_at.isoformat() if hasattr(created_at, "isoformat") else created_at + ) + if updated_at is not None: + result["updated_at"] = ( + updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at + ) + for offset, field in enumerate(extra_fields): + result[field] = self._decode_projected_value(row[6 + offset]) + return result + + def _append_user_name_condition( + self, conditions: list[str], params: list[Any], user_names: list[str] + ) -> None: + if len(user_names) == 1: conditions.append("user_name = %s") - params.append(user_name) + params.append(user_names[0]) + elif user_names: + conditions.append("user_name = ANY(%s)") + params.append(user_names) + def _build_search_conditions( + self, + params: list[Any], + *, + user_name: str | None = None, + knowledgebase_ids: list[str] | None = None, + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + filter: dict | None = None, + default_activated: bool = True, + ) -> list[str]: + conditions: list[str] = [] + user_names = self._effective_user_names(user_name, knowledgebase_ids) + self._append_user_name_condition(conditions, params, user_names) if scope: conditions.append("properties->>'memory_type' = %s") params.append(scope) - if status: conditions.append("properties->>'status' = %s") params.append(status) - else: - conditions.append( - "(properties->>'status' = 'activated' OR properties->>'status' IS NULL)" - ) + elif default_activated: + conditions.append("properties->>'status' = 'activated'") + + for filter_dict in (search_filter, filter): + if filter_dict: + filter_sql = self._build_filter_where_clause(filter_dict, params) + if filter_sql: + conditions.append(f"({filter_sql})") + return conditions + + def _build_search_result( + self, row: tuple[Any, ...], return_fields: list[str] | None + ) -> dict[str, Any]: + result: dict[str, Any] = {"id": row[0], "score": float(row[1])} + fields = self._validate_return_fields(return_fields) + if not fields: + return result + + memory = row[2] or "" + created_at, updated_at, user_name = row[4], row[5], row[6] + raw_properties = row[3] if isinstance(row[3], dict) else json.loads(row[3] or "{}") + properties = dict(raw_properties) + properties["user_name"] = user_name + properties.update( + { + "id": row[0], + "memory": memory, + "created_at": created_at.isoformat() if created_at else None, + "updated_at": updated_at.isoformat() if updated_at else None, + "user_name": user_name, + } + ) + for field in fields: + if field != "id" and field in properties: + result[field] = properties[field] + return result - if search_filter: - for k, v in search_filter.items(): - conditions.append(f"properties->>'{k}' = %s") - params.append(str(v)) + def _build_keyword_result( + self, row: tuple[Any, ...], return_fields: list[str] | None + ) -> dict[str, Any]: + result: dict[str, Any] = {"id": row[0]} + fields = self._validate_return_fields(return_fields) + if not fields: + return result + + raw_properties = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + properties = dict(raw_properties) + properties["user_name"] = row[5] + properties.update( + { + "id": row[0], + "memory": row[1] or "", + "created_at": row[3].isoformat() if row[3] else None, + "updated_at": row[4].isoformat() if row[4] else None, + "user_name": row[5], + } + ) + for field in fields: + if field != "id" and field in properties: + result[field] = properties[field] + return result + def search_by_embedding( + self, + vector: list[float], + user_name: str, + top_k: int = 5, + scope: str | None = None, + status: str | None = None, + threshold: float | None = None, + search_filter: dict | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + return_fields: list[str] | None = None, + light_weight_mode: bool = False, + **kwargs, + ) -> list[dict]: + """Search nodes by vector similarity using pgvector.""" + if light_weight_mode and not return_fields: + return_fields = [ + "memory", + "memory_type", + "user_name", + "status", + "key", + "tags", + "confidence", + "source", + ] + params: list[Any] = [] + conditions = ["embedding IS NOT NULL"] + conditions.extend( + self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + ) where_clause = " AND ".join(conditions) + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) - # pgvector cosine distance: 1 - (a <=> b) gives similarity score + # Match PolarDB: one distance, index-ordered LIMIT, then drop rows at or past 0.9. + dim = self._vector_dim() + distance_sql = "embedding <=> (SELECT query_vector FROM q)" conn = self._get_conn() try: + self._ensure_hnsw_search_settings(conn) with conn.cursor() as cur: + projected = self._projected_field_exprs(return_fields) if light_weight_mode else [] + if light_weight_mode: + inner_columns = "memory, created_at, updated_at, user_name" + extra_inner = "".join( + f', {expression} AS "{field}"' for field, expression in projected + ) + extra_outer = "".join(f', "{field}"' for field, _ in projected) + else: + inner_columns = "memory, properties, created_at, updated_at, user_name" + extra_inner = "" + extra_outer = "" cur.execute( f""" - SELECT id, 1 - (embedding <=> %s::vector) as score - FROM {self.schema}.memories - WHERE {where_clause} - ORDER BY embedding <=> %s::vector - LIMIT %s - """, - (vector, *params, vector, top_k), + WITH q AS ( + SELECT %s::halfvec({dim}) AS query_vector + ), + t AS MATERIALIZED ( + SELECT COALESCE(properties->>'id', id) AS id, + ({distance_sql}) AS scope_distance, + {inner_columns} + {extra_inner} + FROM {relation} + WHERE {where_clause} + ORDER BY {distance_sql} + LIMIT %s + ) + SELECT id, + ((1 - scope_distance) + 1) / 2.0 AS score, + {inner_columns} + {extra_outer} + FROM t + WHERE scope_distance < 0.9 + """, + (vector, *params, top_k), ) results = [] for row in cur.fetchall(): score = float(row[1]) if threshold is None or score >= threshold: - results.append({"id": row[0], "score": score}) + if light_weight_mode: + results.append( + self._build_projected_search_result( + row, [field for field, _ in projected] + ) + ) + else: + results.append(self._build_search_result(row, return_fields)) return results finally: self._put_conn(conn) - def get_by_metadata( + def search_by_fulltext( self, - filters: list[dict[str, Any]], + query_words: list[str], + top_k: int = 10, + scope: str | None = None, + status: str | None = None, + threshold: float | None = None, + search_filter: dict | None = None, + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + tsvector_field: str = "properties_tsvector_zh", + tsquery_config: str = "jiebacfg", + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Search with PolarDB's AND tsquery on one tenant partition.""" + terms = [str(word).strip() for word in query_words if str(word).strip()] + if not terms: + return [] + + query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config): + raise ValueError(f"Invalid PostgreSQL text search config: {query_config}") + quoted_terms = [] + for term in terms: + if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}: + term = term[1:-1] + quoted_terms.append("'" + term.replace("'", "''") + "'") + params: list[Any] = [" & ".join(quoted_terms)] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH q AS ( + SELECT to_tsquery('{query_config}', %s) AS query + ) + SELECT COALESCE(properties->>'id', id) AS id, + ts_rank(search_vector, q.query) AS score, + memory, properties, created_at, updated_at, user_name + FROM {relation} + CROSS JOIN q + WHERE {where_clause} + AND search_vector @@ q.query + ORDER BY score DESC, updated_at DESC + LIMIT %s + """, + (*params, top_k), + ) + output = [] + for row in cur.fetchall(): + result = self._build_search_result(row, return_fields) + if threshold is None or result["score"] >= threshold: + output.append(result) + return output + finally: + self._put_conn(conn) + + def search_by_keywords_tfidf( + self, + query_words: list[str], + scope: str | None = None, status: str | None = None, + search_filter: dict | None = None, user_name: str | None = None, filter: dict | None = None, knowledgebase_ids: list[str] | None = None, + tsvector_field: str = "properties_tsvector_zh", + tsquery_config: str = "jiebaqry", + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Return keyword matches with PolarDB's OR tsquery and no score.""" + if not user_name: + return [] + terms = [str(word).strip() for word in query_words if str(word).strip()] + if not terms: + return [] + + query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config): + raise ValueError(f"Invalid PostgreSQL text search config: {query_config}") + quoted_terms = [] + for term in terms: + if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}: + term = term[1:-1] + quoted_terms.append("'" + term.replace("'", "''") + "'") + params: list[Any] = [" | ".join(quoted_terms)] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH q AS ( + SELECT to_tsquery('{query_config}', %s) AS query + ) + SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name + FROM {relation} + CROSS JOIN q + WHERE {where_clause} + AND search_vector @@ q.query + """, + (*params,), + ) + return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()] + finally: + self._put_conn(conn) + + def search_by_keywords_like( + self, + query_word: str, + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Fallback substring search compatible with the PolarDB adapter.""" + if not user_name: + return [] + pattern = f"%{query_word}%" + params: list[Any] = [] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + search_filter=search_filter, + filter=filter, + status=status, + scope=scope, + ) + conditions.append("COALESCE(properties->>'memory', '') LIKE %s") + params.append(pattern) + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name + FROM {relation} + WHERE {" AND ".join(conditions)} + """, + params, + ) + return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()] + finally: + self._put_conn(conn) + + def get_by_metadata( + self, + filters: list[dict[str, Any]], + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list | None = None, user_name_flag: bool = True, + status: str | None = None, + **kwargs, ) -> list[str]: """Get node IDs matching metadata filters.""" - user_name = user_name or self.user_name + if not user_name: + raise ValueError("get_by_metadata requires user_name") + conditions: list[str] = [] + params: list[Any] = [] + user_names: list[str] = [] - conditions = [] - params = [] - - if user_name_flag and user_name: - conditions.append("user_name = %s") - params.append(user_name) + if user_name_flag: + user_names = self._effective_user_names(user_name, knowledgebase_ids) + self._append_user_name_condition(conditions, params, user_names) if status: conditions.append("properties->>'status' = %s") @@ -887,29 +2037,52 @@ def get_by_metadata( field = f["field"] op = f.get("op", "=") value = f["value"] + text_expr, json_expr = self._field_expr(field) if op == "=": - conditions.append(f"properties->>'{field}' = %s") + conditions.append(f"{text_expr} = %s") params.append(str(value)) elif op == "in": - placeholders = ",".join(["%s"] * len(value)) - conditions.append(f"properties->>'{field}' IN ({placeholders})") - params.extend([str(v) for v in value]) + if not isinstance(value, list): + raise ValueError("in operator expects a list") + conditions.append(f"{text_expr} = ANY(%s)") + params.append([str(item) for item in value]) elif op in (">", ">=", "<", "<="): - conditions.append(f"(properties->>'{field}')::numeric {op} %s") + conditions.append(f"NULLIF({text_expr}, '')::numeric {op} %s") params.append(value) elif op == "contains": - conditions.append(f"properties->'{field}' @> %s::jsonb") - params.append(json.dumps([value])) + values = value if isinstance(value, list) else [value] + or_parts = [] + for item in values: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([item], ensure_ascii=False)) + conditions.append("(" + " OR ".join(or_parts) + ")") + elif op == "like": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"%{value}%") + elif op == "starts_with": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"{value}%") + elif op == "ends_with": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"%{value}") + else: + raise ValueError(f"Unsupported operator: {op}") + + if filter: + filter_sql = self._build_filter_where_clause(filter, params) + if filter_sql: + conditions.append(f"({filter_sql})") where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation("memories", user_names) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - SELECT id FROM {self.schema}.memories + SELECT id FROM {relation} WHERE {where_clause} """, params, @@ -921,23 +2094,34 @@ def get_by_metadata( def get_all_memory_items( self, scope: str, + user_name: str | None = None, include_embedding: bool = False, - status: str | None = None, filter: dict | None = None, - knowledgebase_ids: list[str] | None = None, - **kwargs, + knowledgebase_ids: list | None = None, + status: str | None = None, ) -> list[dict]: """Get all memory items of a specific type.""" - user_name = kwargs.get("user_name") or self.user_name - - conditions = ["properties->>'memory_type' = %s", "user_name = %s"] - params = [scope, user_name] + if not user_name: + return [] + if scope not in {"WorkingMemory", "LongTermMemory", "UserMemory", "OuterMemory"}: + raise ValueError(f"Unsupported memory type scope: {scope}") - if status: - conditions.append("properties->>'status' = %s") - params.append(status) + params: list[Any] = [] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + filter=filter, + default_activated=False, + ) - where_clause = " AND ".join(conditions) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) conn = self._get_conn() try: @@ -945,10 +2129,12 @@ def get_all_memory_items( cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" + cols += ", user_name" cur.execute( f""" - SELECT {cols} FROM {self.schema}.memories + SELECT {cols} FROM {relation} WHERE {where_clause} + LIMIT 100 """, params, ) @@ -957,29 +2143,44 @@ def get_all_memory_items( self._put_conn(conn) def get_structure_optimization_candidates( - self, scope: str, include_embedding: bool = False + self, + scope: str, + include_embedding: bool = False, + user_name: str | None = None, ) -> list[dict]: """Find isolated nodes (no edges).""" - user_name = self.user_name + if not user_name: + raise ValueError("get_structure_optimization_candidates requires user_name") + memories = self._tenant_relation("memories", [user_name]) + edges = self._tenant_relation("edges", [user_name]) conn = self._get_conn() try: with conn.cursor() as cur: cols = "m.id, m.memory, m.properties, m.created_at, m.updated_at" + if include_embedding: + cols += ", m.embedding" + cols += ", m.user_name" cur.execute( f""" SELECT {cols} - FROM {self.schema}.memories m - LEFT JOIN {self.schema}.edges e1 ON m.id = e1.source_id - LEFT JOIN {self.schema}.edges e2 ON m.id = e2.target_id + FROM {memories} m + LEFT JOIN {edges} e1 + ON m.id = e1.source_id + AND e1.user_name = m.user_name + AND e1.edge_type = 'PARENT' + LEFT JOIN {edges} e2 + ON m.id = e2.target_id + AND e2.user_name = m.user_name + AND e2.edge_type = 'PARENT' WHERE m.properties->>'memory_type' = %s AND m.user_name = %s AND m.properties->>'status' = 'activated' AND e1.id IS NULL AND e2.id IS NULL - """, + """, (scope, user_name), ) - return [self._parse_row(row, False) for row in cur.fetchall()] + return [self._parse_row(row, include_embedding) for row in cur.fetchall()] finally: self._put_conn(conn) @@ -987,6 +2188,190 @@ def get_structure_optimization_candidates( # Maintenance # ========================================================================= + def get_memory_count(self, memory_type: str, user_name: str | None = None) -> int: + """Count active memories of one type for a tenant.""" + user_name = user_name or self.user_name + relation = self._tenant_relation("memories", [user_name] if user_name else []) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COUNT(*) + FROM {relation} + WHERE user_name = %s + AND properties->>'memory_type' = %s + AND COALESCE(properties->>'status', 'activated') = 'activated' + """, + (user_name, memory_type), + ) + row = cur.fetchone() + return int(row[0]) if row else 0 + finally: + self._put_conn(conn) + + def node_not_exist(self, scope: str, user_name: str | None = None) -> int: + """Return 1 when no active node exists for the requested scope.""" + return int(self.get_memory_count(scope, user_name=user_name) == 0) + + def count_nodes(self, scope: str, user_name: str | None = None) -> int: + """Compatibility alias for memory counts.""" + return self.get_memory_count(scope, user_name=user_name) + + def get_user_names_by_memory_ids(self, memory_ids: list[str]) -> dict[str, str | None]: + normalized_ids = list( + dict.fromkeys( + str(memory_id).strip() for memory_id in memory_ids if str(memory_id).strip() + ) + ) + if not normalized_ids: + return {} + result: dict[str, str | None] = dict.fromkeys(normalized_ids) + memories = self._all_partitions_relation("memories") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, user_name + FROM {memories} + WHERE id = ANY(%s) + """, + (normalized_ids,), + ) + result.update({str(row[0]): row[1] for row in cur.fetchall()}) + return result + finally: + self._put_conn(conn) + + def exist_user_name(self, user_name: str) -> dict[str, bool]: + memories = self._physical_table("memories", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f"SELECT EXISTS(SELECT 1 FROM {memories} WHERE user_name = %s)", + (user_name,), + ) + row = cur.fetchone() + return {user_name: bool(row and row[0])} + finally: + self._put_conn(conn) + + def delete_node_by_mem_cube_id( + self, + mem_cube_id: str, + delete_record_id: str, + hard_delete: bool = False, + ) -> int: + """Soft-delete a tenant's memories, or purge a previously deleted batch.""" + if not mem_cube_id: + raise ValueError("delete_node_by_mem_cube_id mem_cube_id is required") + if not delete_record_id: + raise ValueError("delete_node_by_mem_cube_id delete_record_id is required") + + memories = self._physical_table("memories", mem_cube_id) + edges = self._physical_table("edges", mem_cube_id) + conn = self._get_conn() + try: + with conn.cursor() as cur: + if hard_delete: + cur.execute( + f""" + WITH deleted AS ( + DELETE FROM {memories} + WHERE user_name = %s + AND properties->>'delete_record_id' = %s + RETURNING id + ), deleted_edges AS ( + DELETE FROM {edges} e + USING deleted d + WHERE e.user_name = %s + AND (e.source_id = d.id OR e.target_id = d.id) + RETURNING e.id + ) + SELECT COUNT(*) FROM deleted + """, + (mem_cube_id, delete_record_id, mem_cube_id), + ) + row = cur.fetchone() + return int(row[0]) if row else 0 + else: + deleted_at = datetime.now(timezone.utc).isoformat() + cur.execute( + f""" + UPDATE {memories} + SET properties = properties || %s::jsonb, + deleted_at = %s::timestamptz, + updated_at = NOW() + WHERE user_name = %s + AND COALESCE(properties->>'status', 'activated') <> 'deleted' + """, + ( + json.dumps( + { + "status": "deleted", + "delete_time": deleted_at, + "delete_record_id": delete_record_id, + } + ), + deleted_at, + mem_cube_id, + ), + ) + return max(int(cur.rowcount or 0), 0) + finally: + self._put_conn(conn) + + def recover_memory_by_mem_cube_id( + self, + mem_cube_id: str, + delete_record_id: str, + ) -> int: + """Recover one tenant soft-delete batch.""" + if not mem_cube_id: + raise ValueError("recover_memory_by_mem_cube_id mem_cube_id is required") + if not delete_record_id: + raise ValueError("recover_memory_by_mem_cube_id delete_record_id is required") + + memories = self._physical_table("memories", mem_cube_id) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + UPDATE {memories} + SET properties = properties || %s::jsonb, + deleted_at = NULL, + updated_at = NOW() + WHERE user_name = %s + AND properties->>'delete_record_id' = %s + AND properties->>'status' = 'deleted' + """, + ( + json.dumps( + { + "status": "activated", + "delete_time": "", + "delete_record_id": "", + } + ), + mem_cube_id, + delete_record_id, + ), + ) + return max(int(cur.rowcount or 0), 0) + finally: + self._put_conn(conn) + + def drop_database(self) -> None: + """Compatibility operation: clear only the configured tenant. + + The standard PostgreSQL backend is shared, so dropping the physical + database or schema would delete other tenants. + """ + self.clear(user_name=self.user_name) + def deduplicate_nodes(self) -> None: """Not implemented - handled at application level.""" @@ -1012,6 +2397,8 @@ def get_grouped_counts( user_name = user_name or self.user_name if not group_fields: raise ValueError("group_fields cannot be empty") + if any(not self._is_safe_field_name(field) for field in group_fields): + raise ValueError("group_fields contains an invalid field name") # Build SELECT and GROUP BY clauses # Fields come from JSONB properties column @@ -1033,10 +2420,11 @@ def get_grouped_counts( query_params.extend(params.values()) where_sql = " AND ".join(conditions) + memories = self._physical_table("memories", user_name) query = f""" SELECT {select_fields}, COUNT(*) AS count - FROM {self.schema}.memories + FROM {memories} WHERE {where_sql} GROUP BY {group_by} """ @@ -1067,13 +2455,15 @@ def merge_nodes(self, id1: str, id2: str) -> str: def clear(self, user_name: str | None = None) -> None: """Clear all data for user.""" user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: # Get all node IDs for user cur.execute( f""" - SELECT id FROM {self.schema}.memories WHERE user_name = %s + SELECT id FROM {memories} WHERE user_name = %s """, (user_name,), ) @@ -1083,16 +2473,17 @@ def clear(self, user_name: str | None = None) -> None: # Delete edges cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) + DELETE FROM {edges} + WHERE user_name = %s + AND (source_id = ANY(%s) OR target_id = ANY(%s)) """, - (ids, ids), + (user_name, ids, ids), ) # Delete nodes cur.execute( f""" - DELETE FROM {self.schema}.memories WHERE user_name = %s + DELETE FROM {memories} WHERE user_name = %s """, (user_name,), ) @@ -1100,36 +2491,148 @@ def clear(self, user_name: str | None = None) -> None: finally: self._put_conn(conn) - def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, Any]: - """Export all data.""" - user_name = kwargs.get("user_name") or self.user_name + def export_graph( + self, + user_name: str | None = None, + include_embedding: bool = False, + user_id: str | None = None, + page: int | None = None, + page_size: int | None = None, + filter: dict | None = None, + memory_type: list[str] | None = None, + status: list[str] | None = None, + **kwargs, + ) -> dict[str, Any]: + """Export memories with the same filters and pagination as PolarDB.""" + + extracted_object_type: str | None = None + extracted_mem_cube_id: str | None = None + + def extract_special_values(value: Any) -> Any: + nonlocal extracted_object_type, extracted_mem_cube_id + if not isinstance(value, dict): + return value + cleaned: dict[str, Any] = {} + for key, item in value.items(): + if key in {"and", "or"} and isinstance(item, list): + children = [extract_special_values(child) for child in item] + children = [child for child in children if child not in (None, {}, [])] + if children: + cleaned[key] = children + elif key == "object_type" and isinstance(item, str): + extracted_object_type = extracted_object_type or item + elif key == "mem_cube_id" and isinstance(item, str): + extracted_mem_cube_id = extracted_mem_cube_id or item + else: + cleaned[key] = item + return cleaned or None + + filter_for_sql = extract_special_values(filter) + conditions: list[str] = [] + params: list[Any] = [] + has_object_filter = bool(extracted_object_type and extracted_mem_cube_id) + object_type = (extracted_object_type or "").strip().lower() + + if has_object_filter: + if object_type == "user": + conditions.append("user_name <> %s") + params.append(extracted_mem_cube_id) + elif object_type == "public": + conditions.append("user_name = %s") + params.append(extracted_mem_cube_id) + elif user_name: + conditions.append("user_name = %s") + params.append(user_name) + + effective_user_id = user_id or getattr(self.config, "user_id", None) + if effective_user_id: + conditions.append("properties->>'user_id' = %s") + params.append(effective_user_id) + if memory_type: + conditions.append("properties->>'memory_type' = ANY(%s)") + params.append([str(item) for item in memory_type]) + if status: + conditions.append("properties->>'status' = ANY(%s)") + params.append([str(item) for item in status]) + else: + conditions.append("COALESCE(properties->>'status', 'activated') <> 'deleted'") + conditions.append("deleted_at IS NULL") + if filter_for_sql: + filter_sql = self._build_filter_where_clause(filter_for_sql, params) + if filter_sql: + conditions.append(f"({filter_sql})") + + where_clause = " AND ".join(conditions) if conditions else "TRUE" + target_user = user_name + if has_object_filter and object_type == "public": + target_user = extracted_mem_cube_id + elif has_object_filter and object_type == "user": + target_user = None + memories = self._physical_table("memories", target_user) + use_pagination = page is not None and page_size is not None + if use_pagination: + page = max(int(page), 1) + page_size = max(int(page_size), 1) + offset = (page - 1) * page_size + conn = self._get_conn() try: with conn.cursor() as cur: - # Get nodes + cur.execute( + f"SELECT COUNT(*) FROM {memories} WHERE {where_clause}", + params, + ) + count_row = cur.fetchone() + total_nodes = int(count_row[0]) if count_row else 0 + cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" + cols += ", user_name" + pagination_sql = "" + data_params = list(params) + if use_pagination: + pagination_sql = "LIMIT %s OFFSET %s" + data_params.extend([page_size, offset]) cur.execute( f""" - SELECT {cols} FROM {self.schema}.memories - WHERE user_name = %s - ORDER BY created_at DESC - """, - (user_name,), + SELECT {cols} + FROM {memories} + WHERE {where_clause} + ORDER BY created_at DESC NULLS LAST, id DESC + {pagination_sql} + """, + data_params, ) nodes = [self._parse_row(row, include_embedding) for row in cur.fetchall()] - # Get edges node_ids = [n["id"] for n in nodes] - if node_ids: + node_user_names = [ + n.get("metadata", {}).get("user_name") or user_name for n in nodes + ] + if node_ids and all(node_user_names): + edges_table = self._tenant_relation( + "edges", + list(dict.fromkeys(str(name) for name in node_user_names if name)), + alias="e", + conn=conn, + ) cur.execute( f""" - SELECT source_id, target_id, edge_type - FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) - """, - (node_ids, node_ids), + WITH selected_nodes(user_name, node_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ) + SELECT e.source_id, e.target_id, e.edge_type + FROM {edges_table} + JOIN selected_nodes AS source_node + ON source_node.user_name = e.user_name + AND source_node.node_id = e.source_id + JOIN selected_nodes AS target_node + ON target_node.user_name = e.user_name + AND target_node.node_id = e.target_id + ORDER BY e.id + """, + (node_user_names, node_ids), ) edges = [ {"source": row[0], "target": row[1], "type": row[2]} @@ -1141,7 +2644,7 @@ def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, A return { "nodes": nodes, "edges": edges, - "total_nodes": len(nodes), + "total_nodes": total_nodes, "total_edges": len(edges), } finally: @@ -1164,6 +2667,7 @@ def import_graph(self, data: dict[str, Any], user_name: str | None = None) -> No source_id=edge["source"], target_id=edge["target"], type=edge["type"], + user_name=user_name, ) def close(self): diff --git a/tests/graph_dbs/test_postgres_400_compat.py b/tests/graph_dbs/test_postgres_400_compat.py new file mode 100644 index 000000000..bd23fd8ff --- /dev/null +++ b/tests/graph_dbs/test_postgres_400_compat.py @@ -0,0 +1,391 @@ +import json + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def postgres_db(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db.schema = "memos" + db.db_name = "memos" + db.user_name = "tenant-a" + db._pool_closed = False + db.config = SimpleNamespace( + embedding_dimension=3, + fulltext_config="simple", + vector_index_type="hnsw", + hnsw_m=16, + hnsw_ef_construction=64, + ivfflat_lists=100, + shard_count=400, + ) + + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + db._get_conn = MagicMock(return_value=connection) + db._put_conn = MagicMock() + db._partition_remainders = { + "tenant-a": 17, + "tenant-b": 17, + "kb-1": 17, + } + return db, cursor + + +def _executed_sql(cursor: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_schema_does_not_create_profile_storage(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "profile_templates" not in sql + assert "profile_instances" not in sql + + +def test_schema_creates_400_md5_memory_shards(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "PARTITION BY HASH" not in sql + assert '"Memory"' in sql + assert "embedding halfvec(" in sql + assert "memos_graph_0" in sql + assert "memos_graph_399" in sql + assert "MODULUS" not in sql + assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 401 + assert "PRIMARY KEY (user_name, id)" in sql + + +def test_schema_rejects_legacy_nonpartitioned_parent(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("memories", "r")] + + with pytest.raises(RuntimeError, match="configure a new MEMOS_SCHEMA"): + db._init_schema() + + +def test_schema_skips_partition_ddl_when_layout_is_complete(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (400,) + + db._init_schema() + + sql = _executed_sql(cursor) + assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 1 + assert "memos_graph_0" not in sql + + +def test_node_upsert_targets_composite_partition_key(postgres_db): + db, cursor = postgres_db + + db.add_node( + "memory-1", + "memory", + {"embedding": [1.0, 0.0, 0.0]}, + user_name="tenant-a", + ) + + assert "ON CONFLICT (user_name, id)" in _executed_sql(cursor) + + +def test_batch_insert_uses_one_statement_and_polardb_properties(postgres_db): + db, cursor = postgres_db + + db.add_nodes_batch( + [ + {"id": "m1", "memory": "one", "metadata": {"embedding": [1.0, 0.0, 0.0]}}, + {"id": "m2", "memory": "two", "metadata": {"embedding": [0.0, 1.0, 0.0]}}, + ], + user_name="tenant-a", + ) + + assert cursor.execute.call_count == 1 + query, params = cursor.execute.call_args.args + assert query.count("::halfvec(") == 2 + assert "ON CONFLICT (user_name, id)" in query + first_properties = json.loads(params[2]) + second_properties = json.loads(params[9]) + assert first_properties["id"] == "m1" + assert first_properties["memory"] == "one" + assert second_properties["id"] == "m2" + assert second_properties["memory"] == "two" + assert "embedding" not in first_properties + + +def test_node_upsert_persists_tenant_in_metadata(postgres_db): + db, cursor = postgres_db + + db.add_node( + "memory-1", + "memory", + {"embedding": [1.0, 0.0, 0.0]}, + user_name="tenant-a", + ) + + _, params = cursor.execute.call_args.args + properties = json.loads(params[2]) + assert properties["user_name"] == "tenant-a" + assert properties["delete_time"] == "" + assert properties["delete_record_id"] == "" + + +def test_batch_node_metadata_matches_polardb_shape(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + + node = db._parse_row( + ( + "memory-1", + "service HTTP comparison memory", + {"status": "activated"}, + now, + now, + ) + ) + + assert node["metadata"]["id"] == "memory-1" + assert node["metadata"]["memory"] == "service HTTP comparison memory" + + +def test_parse_row_restores_tenant_and_deserializes_sources(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + source = {"type": "chat", "role": "user", "content": "hello"} + + node = db._parse_row( + ( + "memory-1", + "memory", + {"sources": [json.dumps(source)]}, + now, + now, + "tenant-a", + ) + ) + + assert node["metadata"]["user_name"] == "tenant-a" + assert node["metadata"]["sources"] == [source] + + +def test_new_polardb_business_methods_are_available(): + required_methods = { + "delete_node_by_cube_memory_groups", + "get_user_names_by_cube_memory_groups", + "get_children_with_embeddings", + "get_neighbors_by_tag", + } + + missing = sorted(name for name in required_methods if not hasattr(PostgresGraphDB, name)) + assert missing == [] + + +def test_get_subgraph_matches_polardb_return_shape(postgres_db): + db, cursor = postgres_db + core = { + "id": "core", + "memory": "core memory", + "metadata": {"status": "activated"}, + } + neighbor = { + "id": "neighbor", + "memory": "neighbor memory", + "metadata": {"status": "activated"}, + } + db.get_node = MagicMock(return_value=core) + db.get_nodes = MagicMock(return_value=[neighbor]) + cursor.fetchall.side_effect = [ + [("core",), ("neighbor",)], + [("core", "neighbor", "RELATE_TO")], + ] + + result = db.get_subgraph( + "core", + depth=2, + center_status="activated", + user_name="tenant-a", + ) + + assert result == { + "core_node": core, + "neighbors": [neighbor], + "edges": [{"source": "core", "target": "neighbor", "type": "RELATE_TO"}], + } + assert "user_name = %s" in _executed_sql(cursor) + db.get_node.assert_called_once_with("core", user_name="tenant-a") + db.get_nodes.assert_called_once_with(["neighbor"], user_name="tenant-a") + + +def test_cube_group_delete_pairs_ids_with_tenants(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (2,) + + deleted = db.delete_node_by_cube_memory_groups( + [ + {"cube_id": "tenant-a", "memory_ids": ["m1"]}, + {"cube_id": "tenant-b", "memory_ids": ["m2"]}, + ] + ) + + query, params = cursor.execute.call_args.args + assert deleted == 2 + assert "unnest" in query.lower() + assert "requested.user_name" in query + assert params == (["tenant-a", "tenant-b"], ["m1", "m2"]) + + +def test_memory_owner_lookup_preserves_missing_ids(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("m1", "tenant-a")] + + owners = db.get_user_names_by_memory_ids(["m1", "missing"]) + + assert owners == {"m1": "tenant-a", "missing": None} + + +def test_export_graph_supports_latest_filters_and_pagination(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchone.return_value = (3,) + cursor.fetchall.side_effect = [ + [ + ( + "m1", + "memory", + {"memory_type": "LongTermMemory", "status": "activated"}, + now, + now, + ) + ], + [("m1", "m2", "RELATE_TO")], + ] + + result = db.export_graph( + user_name="tenant-a", + page=2, + page_size=1, + filter={"agent_id": "agent-1"}, + memory_type=["LongTermMemory"], + status=["activated"], + ) + + sql = _executed_sql(cursor) + assert result["total_nodes"] == 3 + assert result["nodes"][0]["id"] == "m1" + assert "properties->'info'->>'agent_id'" in sql + assert "LIMIT %s OFFSET %s" in sql + assert "properties->>'memory_type' = ANY(%s)" in sql + assert "JOIN selected_nodes AS source_node" in sql + assert "source_node.user_name = e.user_name" in sql + + +def test_keyword_like_matches_polardb_filters_and_parameter_name(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "memory-1", + "needle memory", + {"source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + result = db.search_by_keywords_like( + query_word="needle", + scope="LongTermMemory", + status="activated", + search_filter={"source": "conversation"}, + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + return_fields=["memory", "source"], + ) + + query, params = cursor.execute.call_args.args + assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}] + assert "user_name = ANY(%s)" in query + assert ["tenant-a", "kb-1"] in params + assert "properties->>'source'" in query + assert "properties->>'memory', '') LIKE %s" in query + + +def test_keyword_tfidf_matches_polardb_result_shape(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "memory-1", + "needle memory", + {"source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + result = db.search_by_keywords_tfidf( + ["needle", "memory"], + scope="LongTermMemory", + user_name="tenant-a", + return_fields=["memory", "source"], + ) + + assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}] + query, params = cursor.execute.call_args.args + assert params[0] == "'needle' | 'memory'" + assert "to_tsquery('jiebaqry'" in query + assert "ILIKE" not in query + + +def test_keyword_searches_require_explicit_user_name(postgres_db): + db, cursor = postgres_db + + assert db.search_by_keywords_like("needle") == [] + assert db.search_by_keywords_tfidf(["needle"]) == [] + cursor.execute.assert_not_called() + + +def test_get_all_memory_items_matches_polardb_defaults(postgres_db): + db, cursor = postgres_db + + assert db.get_all_memory_items("LongTermMemory") == [] + cursor.execute.assert_not_called() + + with pytest.raises(ValueError, match="Unsupported memory type scope"): + db.get_all_memory_items("Context", user_name="tenant-a") + + cursor.fetchall.return_value = [] + db.get_all_memory_items("LongTermMemory", user_name="tenant-a") + query, _ = cursor.execute.call_args.args + assert "properties->>'status'" not in query + + +def test_metadata_contains_list_uses_json_array_overlap(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("m1",)] + + result = db.get_by_metadata( + [{"field": "tags", "op": "contains", "value": ["blue", "green"]}], + user_name="tenant-a", + ) + + query, params = cursor.execute.call_args.args + assert result == ["m1"] + assert "properties->'tags' @> %s::jsonb" in query + assert " OR " in query + assert json.dumps(["blue"], ensure_ascii=False) in params + assert json.dumps(["green"], ensure_ascii=False) in params diff --git a/tests/graph_dbs/test_postgres_pgvector.py b/tests/graph_dbs/test_postgres_pgvector.py new file mode 100644 index 000000000..d404cf230 --- /dev/null +++ b/tests/graph_dbs/test_postgres_pgvector.py @@ -0,0 +1,461 @@ +import hashlib +import json + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.configs.graph_db import PostgresGraphDBConfig +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def postgres_db(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db.schema = "memos" + db.db_name = "memos" + db.user_name = "tenant-a" + db._pool_closed = False + db.config = SimpleNamespace( + embedding_dimension=3, + fulltext_config="simple", + vector_index_type="hnsw", + hnsw_m=16, + hnsw_ef_construction=64, + ivfflat_lists=100, + shard_count=400, + ) + db._partition_remainders = { + "tenant-a": 17, + "tenant-b": 17, + "kb-1": 17, + } + + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + db._get_conn = MagicMock(return_value=connection) + db._put_conn = MagicMock() + return db, cursor + + +def _executed_sql(cursor: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_postgres_pool_keeps_eight_warm_connections(): + config = PostgresGraphDBConfig( + host="postgres", + user="postgres", + password="secret", + db_name="memos", + user_name="tenant-a", + ) + assert config.minconn == 8 + assert config.maxconn == 20 + + +def test_get_conn_does_not_ping_before_the_query(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db._pool_closed = False + cursor = MagicMock() + connection = MagicMock() + connection.closed = 0 + connection.cursor.return_value.__enter__.return_value = cursor + db.pool = MagicMock() + db.pool.getconn.return_value = connection + + assert db._get_conn() is connection + cursor.execute.assert_not_called() + + +def test_postgres_config_defaults_to_current_embedding_dimension(): + config = PostgresGraphDBConfig( + host="postgres", + user="memos", + password="secret", + db_name="memos", + user_name="tenant-a", + ) + + assert config.embedding_dimension == 1024 + assert config.shard_count == 400 + + +def test_init_schema_creates_tenant_edges_fulltext_and_hnsw(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "user_name TEXT NOT NULL" in sql + assert "UNIQUE(user_name, source_id, target_id, edge_type)" in sql + assert "search_vector tsvector" in sql + assert "properties::text" in sql + assert "USING GIN(search_vector)" in sql + assert "halfvec_cosine_ops" in sql + assert "idx_memory_embedding_halfvec" in sql + assert "embedding halfvec(" in sql + assert '"Memory"' in sql + assert "USING ivfflat" not in sql + + +def test_edge_operations_are_tenant_scoped_and_support_any_direction(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (1,) + cursor.fetchall.return_value = [("source", "target", "RELATE_TO")] + + db.add_edge("source", "target", "RELATE_TO", user_name="tenant-a") + assert db.edge_exists( + "source", + "target", + "RELATE_TO", + direction="ANY", + user_name="tenant-a", + ) + edges = db.get_edges("source", type="ANY", direction="ANY", user_name="tenant-a") + + sql = _executed_sql(cursor) + assert "user_name" in sql + assert edges == [{"from": "source", "to": "target", "type": "RELATE_TO"}] + assert any( + "tenant-a" in call.args[1] for call in cursor.execute.call_args_list if len(call.args) > 1 + ) + + db.edge_exists("source", "target", type="ANY", user_name="tenant-a") + query, params = cursor.execute.call_args.args + assert "edge_type = %s" not in query + assert params == ["tenant-a", "source", "target"] + + +def test_vector_search_applies_filters_kb_ids_and_return_fields(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.95, + "remember this", + {"status": "activated", "source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + results = db.search_by_embedding( + vector=[0.1, 0.2, 0.3], + top_k=5, + scope="LongTermMemory", + status="activated", + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + filter={"and": [{"info.agent_id": "agent-1"}, {"tags": {"contains": "blue"}}]}, + return_fields=["memory", "source", "created_at"], + ) + + query, params = cursor.execute.call_args.args + assert "user_name = ANY(%s)" in query + assert "COALESCE(properties->>'id', id)" in query + assert "properties->'info'->>'agent_id'" in query + assert "properties->'tags' @> %s::jsonb" in query + assert ["tenant-a", "kb-1"] in params + assert results[0]["memory"] == "remember this" + assert results[0]["source"] == "conversation" + assert results[0]["created_at"] == now.isoformat() + + +def test_lightweight_vector_search_returns_memory_without_extra_lookup(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.95, + "remember this", + now, + now, + "tenant-a", + '"LongTermMemory"', + '"activated"', + None, + None, + None, + None, + ) + ] + + results = db.search_by_embedding( + vector=[0.1, 0.2, 0.3], + top_k=5, + scope="LongTermMemory", + user_name="tenant-a", + light_weight_mode=True, + ) + + assert results[0]["memory"] == "remember this" + assert results[0]["memory_type"] == "LongTermMemory" + assert PostgresGraphDB.supports_lightweight_vector_search is True + assert PostgresGraphDB.prefer_lightweight_vector_search is True + + +def test_search_result_does_not_add_postgres_only_property_aliases(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + row = ( + "node-1", + 0.95, + "remember this", + {"status": "activated"}, + now, + now, + "tenant-a", + ) + + result = db._build_search_result(row, ["metadata", "properties"]) + + assert result == {"id": "node-1", "score": 0.95} + + +def test_search_result_keeps_polardb_serialized_source_shape(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + source = json.dumps({"type": "chat", "role": "user", "content": "hello"}) + row = ( + "node-1", + 0.95, + "remember this", + {"sources": [source]}, + now, + now, + "tenant-a", + ) + + result = db._build_search_result(row, ["sources"]) + + assert result["sources"] == [source] + + +def test_fulltext_search_uses_polardb_and_tsquery(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.8, + "中文记忆内容", + {"status": "activated"}, + now, + now, + "tenant-a", + ) + ] + + results = db.search_by_fulltext( + query_words=["中文", "记忆"], + top_k=10, + scope="LongTermMemory", + user_name="tenant-a", + filter={"info.agent_id": "agent-1"}, + return_fields=["memory"], + ) + + query, params = cursor.execute.call_args.args + assert "to_tsquery('jiebacfg'" in query + assert "search_vector @@" in query + assert "ILIKE" not in query + assert "properties->'info'->>'agent_id'" in query + assert params[0] == "'中文' & '记忆'" + assert results == [{"id": "node-1", "score": 0.8, "memory": "中文记忆内容"}] + + +def test_get_node_without_tenant_searches_all_partitions(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchone.return_value = ( + "node-1", + "remember this", + {"status": "activated", "user_name": "tenant-b"}, + now, + now, + ) + + result = db.get_node("node-1") + + query, params = cursor.execute.call_args.args + assert "WHERE id = %s" in query + assert "user_name = %s" not in query + assert '"memos_graph_0"."Memory"' in query + assert '"memos_graph_399"."Memory"' in query + assert "FROM memos.memories" not in query + assert params == ("node-1",) + assert result["id"] == "node-1" + + +def test_get_node_with_tenant_remains_tenant_scoped(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = None + + assert db.get_node("node-1", user_name="tenant-b") is None + + query, params = cursor.execute.call_args.args + assert "WHERE id = %s AND user_name = %s" in query + assert params == ("node-1", "tenant-b") + + +def test_get_nodes_without_tenant_searches_all_partitions(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + assert db.get_nodes(["node-1", "node-2"]) == [] + + query, params = cursor.execute.call_args.args + assert "WHERE id = ANY(%s)" in query + assert "user_name = %s" not in query + assert '"memos_graph_0"."Memory"' in query + assert '"memos_graph_399"."Memory"' in query + assert "FROM memos.memories" not in query + assert params == (["node-1", "node-2"],) + + +def test_cube_soft_delete_recover_and_counts_are_available(postgres_db): + db, cursor = postgres_db + cursor.rowcount = 3 + cursor.fetchone.side_effect = [(3,), (0,)] + + assert db.delete_node_by_mem_cube_id("tenant-a", "delete-1") == 3 + assert db.recover_memory_by_mem_cube_id("tenant-a", "delete-1") == 3 + assert db.get_memory_count("LongTermMemory", user_name="tenant-a") == 3 + assert db.node_not_exist("UserMemory", user_name="tenant-a") == 1 + + sql = _executed_sql(cursor) + assert "delete_record_id" in sql + assert "status" in sql + assert "user_name = %s" in sql + + +def test_drop_database_is_scoped_to_current_tenant(postgres_db): + db, _cursor = postgres_db + db.clear = MagicMock() + + db.drop_database() + + db.clear.assert_called_once_with(user_name="tenant-a") + + +def test_structure_candidates_accept_tenant_and_optional_embedding(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + "remember this", + {"status": "activated", "memory_type": "LongTermMemory"}, + now, + now, + [0.1, 0.2, 0.3], + ) + ] + + results = db.get_structure_optimization_candidates( + "LongTermMemory", + include_embedding=True, + user_name="tenant-b", + ) + + query, params = cursor.execute.call_args.args + assert "m.embedding" in query + assert params == ("LongTermMemory", "tenant-b") + assert results[0]["metadata"]["embedding"] == [0.1, 0.2, 0.3] + + +def test_structure_candidates_only_exclude_parent_edges(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + db.get_structure_optimization_candidates( + "LongTermMemory", + user_name="tenant-a", + ) + + query, _ = cursor.execute.call_args.args + assert "e1.edge_type = 'PARENT'" in query + assert "e2.edge_type = 'PARENT'" in query + + +def test_time_filter_uses_timestamp_comparison(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + db.search_by_embedding( + [0.1, 0.2, 0.3], + user_name="tenant-a", + top_k=5, + filter={"created_at": {"gt": "2026-01-01T00:00:00"}}, + ) + + query, _params = cursor.execute.call_args.args + assert "::timestamp >" in query + assert "timestamptz" not in query + + +def test_single_tenant_vector_search_reads_one_hash_partition(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {"tenant-a": 17} + cursor.fetchall.return_value = [] + + db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5) + + query, params = cursor.execute.call_args.args + assert 'FROM "memos_graph_17"."Memory"' in query + assert "FROM memos.memories" not in query + assert query.count("%s::halfvec(3)") == 1 + assert "AS MATERIALIZED" in query + assert "ORDER BY embedding <=> (SELECT query_vector FROM q)" in query + materialized_and_score, outer_filter = query.split("FROM t", 1) + materialized_body = materialized_and_score.split("AS MATERIALIZED", 1)[1] + assert "< 0.9" not in materialized_body + assert "scope_distance < 0.9" in outer_filter + assert "((1 - scope_distance) + 1) / 2.0" in materialized_and_score + assert "embedding::halfvec" not in query + assert "user_name = %s" in query + assert "user_name = ANY(%s)" not in query + assert params[0] == [0.1, 0.2, 0.3] + assert params[1] == "tenant-a" + assert params[-1] == 5 + assert any("hnsw.iterative_scan" in str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_cross_shard_vector_search_reads_only_those_partitions(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {"tenant-a": 17, "kb-1": 19} + cursor.fetchall.return_value = [] + + db.search_by_embedding( + [0.1, 0.2, 0.3], + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + top_k=5, + ) + + query, _params = cursor.execute.call_args.args + assert '"memos_graph_17"."Memory"' in query + assert '"memos_graph_19"."Memory"' in query + assert "UNION ALL" in query + assert "FROM memos.memories" not in query + + +def test_partition_remainder_uses_polardb_md5(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {} + cursor.fetchall.return_value = [] + shard = int(hashlib.md5(b"tenant-a").hexdigest(), 16) % 400 + + db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5) + + query = cursor.execute.call_args.args[0] + assert f'"memos_graph_{shard}"."Memory"' in query + assert "satisfies_hash_partition" not in _executed_sql(cursor) + assert db._partition_remainder("tenant-a") == shard From 9b35357f1d5957886cb23562e6c07e289be45758 Mon Sep 17 00:00:00 2001 From: lils <310442317+linsheng-li@users.noreply.github.com> Date: Wed, 21 Oct 2026 22:54:22 +0800 Subject: [PATCH 09/15] fix(scheduler): initialize RabbitMQ IO thread before shutdown --- .../webservice_modules/rabbitmq_service.py | 2 +- tests/mem_scheduler/test_rabbitmq_shutdown.py | 110 ++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 tests/mem_scheduler/test_rabbitmq_shutdown.py diff --git a/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py b/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py index 3123bfdee..e39f4bbd9 100644 --- a/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py +++ b/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py @@ -61,7 +61,7 @@ def __init__(self): self.rabbitmq_conn_sleep_seconds = 1 # Thread management - self._rabbitmq_io_loop_thread = None # For IOLoop execution + self._io_loop_thread = None # For IOLoop execution self._rabbitmq_stop_flag = False # Graceful shutdown flag # Use RLock because publishing may trigger initialization, which also grabs the lock. self._rabbitmq_lock = threading.RLock() diff --git a/tests/mem_scheduler/test_rabbitmq_shutdown.py b/tests/mem_scheduler/test_rabbitmq_shutdown.py new file mode 100644 index 000000000..5cb594c43 --- /dev/null +++ b/tests/mem_scheduler/test_rabbitmq_shutdown.py @@ -0,0 +1,110 @@ +"""Native scheduler cleanup only; no models, broker or business data.""" + +import threading +import time + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from memos.api import lifecycle +from memos.configs.mem_scheduler import GeneralSchedulerConfig +from memos.mem_scheduler.optimized_scheduler import OptimizedScheduler +from memos.mem_scheduler.webservice_modules.rabbitmq_service import RabbitMQSchedulerModule + + +def test_close_without_rabbitmq_initialization_is_safe_and_repeatable(): + scheduler = RabbitMQSchedulerModule() + scheduler.rabbitmq_close() + scheduler.rabbitmq_close() + assert scheduler._io_loop_thread is None + assert scheduler._rabbitmq_stop_flag is True + + +def test_early_initialization_skip_still_closes(monkeypatch): + monkeypatch.setenv("CI", "true") + monkeypatch.delenv("MEMOS_ENABLE_RABBITMQ", raising=False) + scheduler = RabbitMQSchedulerModule() + scheduler.initialize_rabbitmq() + assert scheduler._rabbitmq_initializing is False + scheduler.rabbitmq_close() + assert scheduler._io_loop_thread is None + + +def test_real_io_thread_and_connection_cleanup_still_execute(): + scheduler = RabbitMQSchedulerModule() + stopped = threading.Event() + calls = [] + scheduler.rabbitmq_channel = SimpleNamespace( + is_open=True, close=lambda: calls.append("channel") + ) + scheduler.rabbitmq_connection = SimpleNamespace( + is_open=True, + close=lambda: calls.append("connection"), + ioloop=SimpleNamespace(stop=lambda: (calls.append("ioloop"), stopped.set())), + ) + worker = threading.Thread(target=stopped.wait, daemon=True) + scheduler._io_loop_thread = worker + worker.start() + try: + scheduler.rabbitmq_close() + assert not worker.is_alive() + assert calls == ["channel", "connection", "ioloop"] + assert scheduler._rabbitmq_stop_flag is True + finally: + stopped.set() + worker.join(timeout=1) + + +def test_actual_optimized_scheduler_lifecycle_joins_threads_and_shuts_executor(monkeypatch): + scheduler = OptimizedScheduler(GeneralSchedulerConfig(use_redis_queue=False)) + scheduler._running = True + + def native_running_loop(): + while scheduler._running: + time.sleep(0.005) + + consumer = threading.Thread(target=native_running_loop, daemon=True) + monitor = threading.Thread(target=native_running_loop, daemon=True) + scheduler._consumer_thread = consumer + scheduler._monitor_thread = monitor + consumer.start() + monitor.start() + task = scheduler.dispatcher.dispatcher_executor.submit(lambda: 42) + assert task.result(timeout=2) == 42 + shutdown = Mock(wraps=scheduler.dispatcher.shutdown) + monkeypatch.setattr(scheduler.dispatcher, "shutdown", shutdown) + error = Mock() + monkeypatch.setattr(lifecycle.logger, "exception", error) + try: + lifecycle.shutdown_components({"mem_scheduler": scheduler}) + assert not consumer.is_alive() and not monitor.is_alive() + assert scheduler._running is False + assert task.done() and scheduler.dispatcher.dispatcher_executor._shutdown is True + deadline = time.monotonic() + 2 + while ( + any(t.is_alive() for t in scheduler.dispatcher.dispatcher_executor._threads) + and time.monotonic() < deadline + ): + time.sleep(0.005) + assert not any(t.is_alive() for t in scheduler.dispatcher.dispatcher_executor._threads) + shutdown.assert_called_once() + assert scheduler._rabbitmq_stop_flag is True + error.assert_not_called() + finally: + scheduler._running = False + consumer.join(timeout=1) + monitor.join(timeout=1) + scheduler.dispatcher.shutdown() + + +def test_unexpected_join_error_is_not_hidden(): + scheduler = RabbitMQSchedulerModule() + + def fail_join(**kwargs): + raise RuntimeError("fixture join failed") + + scheduler._io_loop_thread = SimpleNamespace(is_alive=lambda: True, join=fail_join) + with pytest.raises(RuntimeError, match="fixture join failed"): + scheduler.rabbitmq_close() From f6266bef7fd1ff51019c95f5f1c911ce0d52efeb Mon Sep 17 00:00:00 2001 From: Chengzhen Jia Date: Fri, 23 Oct 2026 19:58:44 +0800 Subject: [PATCH 10/15] fix: pin certifi 2026.10.19 to drop compromised eMudhra roots (#2536) The Docker requirements and lockfiles still shipped CA bundles from before the October 2026 distrust. Co-authored-by: jiachengzhen Co-authored-by: Cursor --- docker/requirements-full.txt | 2 +- docker/requirements.txt | 2 +- poetry.lock | 6 +++--- uv.lock | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/requirements-full.txt b/docker/requirements-full.txt index b148d43d6..4e800d98c 100644 --- a/docker/requirements-full.txt +++ b/docker/requirements-full.txt @@ -8,7 +8,7 @@ attrs==25.3.0 authlib==1.6.0 beautifulsoup4==4.13.4 cachetools==6.2.1 -certifi==2025.7.14 +certifi==2026.10.19 cffi==1.17.1 charset-normalizer==3.4.2 chonkie==1.1.1 diff --git a/docker/requirements.txt b/docker/requirements.txt index 988e64b83..15b5685f7 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -4,7 +4,7 @@ attrs==25.4.0 Authlib==1.6.5 beartype==0.22.5 cachetools==6.2.2 -certifi==2025.11.12 +certifi==2026.10.19 cffi==2.0.0 charset-normalizer==3.4.4 chonkie==1.1.0 diff --git a/poetry.lock b/poetry.lock index 915204888..3aa5c9268 100644 --- a/poetry.lock +++ b/poetry.lock @@ -184,14 +184,14 @@ files = [ [[package]] name = "certifi" -version = "2025.7.14" +version = "2026.10.19" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main", "eval"] files = [ - {file = "certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2"}, - {file = "certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995"}, + {file = "certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b"}, + {file = "certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae"}, ] [[package]] diff --git a/uv.lock b/uv.lock index 385ef4382..d081fa9d5 100644 --- a/uv.lock +++ b/uv.lock @@ -132,11 +132,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.2.25" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -2069,9 +2069,10 @@ wheels = [ [[package]] name = "memoryos" -version = "2.0.14" +version = "2.0.33" source = { editable = "." } dependencies = [ + { name = "cachetools" }, { name = "concurrent-log-handler" }, { name = "fastapi", extra = ["all"] }, { name = "fastmcp" }, @@ -2090,7 +2091,6 @@ dependencies = [ [package.optional-dependencies] all = [ { name = "alibabacloud-oss-v2" }, - { name = "cachetools" }, { name = "chonkie" }, { name = "datasketch" }, { name = "jieba" }, @@ -2142,7 +2142,7 @@ tree-mem = [ requires-dist = [ { name = "alibabacloud-oss-v2", marker = "extra == 'all'", specifier = ">=1.2.2,<1.2.3" }, { name = "alibabacloud-oss-v2", marker = "extra == 'skill-mem'", specifier = ">=1.2.2,<1.2.3" }, - { name = "cachetools", marker = "extra == 'all'", specifier = ">=6.0.0" }, + { name = "cachetools", specifier = ">=6.0.0" }, { name = "chonkie", marker = "extra == 'all'", specifier = ">=1.0.7,<2.0.0" }, { name = "chonkie", marker = "extra == 'mem-reader'", specifier = ">=1.0.7,<2.0.0" }, { name = "concurrent-log-handler", specifier = ">=0.9.28,<1.0.0" }, From af0108d04c25ce197ddfe0ffa65f9e555f4df489 Mon Sep 17 00:00:00 2001 From: RerankerGuo Date: Tue, 13 Oct 2026 16:54:31 +0800 Subject: [PATCH 11/15] fix(api): strip Bearer scheme so keyed requests reach per-key rate-limit bucket (#2428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _get_client_key in src/memos/api/middleware/rate_limit.py tested whether request.headers['Authorization'] started with 'krlk_', but real clients send 'Authorization: Bearer krlk_' per RFC 6750. The prefix check never matched and every request — including ones carrying a valid API key — fell through to IP-based limiting. Behind a shared egress IP all keyed clients shared one bucket, so the per-key rate-limit was dead. The fix strips the 'Bearer ' scheme (case-insensitive, per RFC 7235) before the prefix check, so 'Bearer krlk_' correctly maps to 'ratelimit:key:'. Backwards compatibility is preserved: a bare 'krlk_' header still matches. - New tests/api/test_rate_limit_bearer.py with 7 cases covering Bearer prefix, lowercase bearer, non-key Bearer tokens, missing header, X-Forwarded-For preference, and the bare-krlk_ backwards-compat path. - No new imports, no new dependencies, no public API change. Fixes #2428. --- src/memos/api/middleware/rate_limit.py | 7 ++- tests/api/test_rate_limit_bearer.py | 78 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 tests/api/test_rate_limit_bearer.py diff --git a/src/memos/api/middleware/rate_limit.py b/src/memos/api/middleware/rate_limit.py index c547378ca..72808a13d 100644 --- a/src/memos/api/middleware/rate_limit.py +++ b/src/memos/api/middleware/rate_limit.py @@ -57,8 +57,13 @@ def _get_client_key(request: Request) -> str: Uses API key if available, otherwise falls back to IP. """ - # Try to get API key from header + # Try to get API key from header. Real clients send a Bearer scheme + # per RFC 6750 (``Authorization: Bearer krlk_``); we strip the + # scheme before the prefix check so keyed requests actually land on + # a per-key bucket (issue #2428). auth_header = request.headers.get("Authorization", "") + if auth_header.lower().startswith("bearer "): + auth_header = auth_header[len("bearer ") :].lstrip() if auth_header.startswith("krlk_"): # Use first 20 chars of key as identifier return f"ratelimit:key:{auth_header[:20]}" diff --git a/tests/api/test_rate_limit_bearer.py b/tests/api/test_rate_limit_bearer.py new file mode 100644 index 000000000..710346cf0 --- /dev/null +++ b/tests/api/test_rate_limit_bearer.py @@ -0,0 +1,78 @@ +"""Regression tests for issue #2428. + +``_get_client_key`` in ``src/memos/api/middleware/rate_limit.py`` checked +whether ``request.headers['Authorization']`` started with ``krlk_`` — but +real clients send ``Authorization: Bearer krlk_`` per RFC 6750, so +the prefix check never matched and every request fell through to IP-based +limiting. Behind a shared egress IP all keyed clients shared one bucket +and the per-key rate-limit was dead code. + +The fix strips the ``Bearer `` scheme before testing the key prefix, so +``Bearer krlk_`` correctly maps to ``ratelimit:key:``. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +from memos.api.middleware.rate_limit import _get_client_key + + +def _fake_request(authorization: str | None, *, client_host: str = "10.0.0.1"): + headers: dict[str, str] = {} + if authorization is not None: + headers["Authorization"] = authorization + return SimpleNamespace( + headers=headers, + client=SimpleNamespace(host=client_host), + ) + + +class TestGetClientKey: + def test_bearer_krlk_key_uses_per_key_bucket(self): + body = "krlk_" + "a" * 64 + request = _fake_request("Bearer " + body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] + assert "10.0.0.1" not in key + + def test_bearer_krlk_key_first_twenty_chars_of_actual_key(self): + body = "krlk_" + "f" * 64 + request = _fake_request("Bearer " + body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] + + def test_lowercase_bearer_scheme(self): + # RFC 7235: the scheme name is case-insensitive. We normalise to + # lowercase before the prefix check. + body = "krlk_" + "b" * 64 + request = _fake_request("bearer " + body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] + + def test_bearer_with_non_key_token_falls_back_to_ip(self): + request = _fake_request("Bearer some-jwt-token") + key = _get_client_key(request) + assert key.startswith("ratelimit:ip:") + assert "10.0.0.1" in key + + def test_missing_authorization_header_falls_back_to_ip(self): + request = _fake_request(None) + key = _get_client_key(request) + assert key.startswith("ratelimit:ip:") + assert "10.0.0.1" in key + + def test_x_forwarded_for_header_preferred_over_ip(self): + request = _fake_request(None, client_host="203.0.113.1") + request.headers["X-Forwarded-For"] = "198.51.100.1, 10.0.0.1" + key = _get_client_key(request) + assert key == "ratelimit:ip:198.51.100.1" + + def test_krlk_prefix_without_bearer_scheme_still_matches(self): + # Backwards compatibility: clients that sent a bare krlk_ header + # (before the Bearer scheme was added) must still match. + body = "krlk_" + "c" * 64 + request = _fake_request(body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] From 981130cc640b7543bc9b351fb4f55cf0cb3de627 Mon Sep 17 00:00:00 2001 From: RerankerGuo Date: Tue, 13 Oct 2026 16:59:01 +0800 Subject: [PATCH 12/15] fix(api): enforce canonical hex format on validate_key_format (#2427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate_key_format exists as a separate copy in two modules — src/memos/api/utils/api_keys.py (canonical home, used by the key-creation utilities) and src/memos/api/middleware/auth.py (the middleware that gates every authenticated request). Both used `int(hex_part, 16)` to verify the 64-char hex tail, which silently passes for: - a leading `+` or `-` sign (`int("+a..a", 16)` == ...), - a `0x` / `0X` prefix (`int("0xa..", 16)` == ...), - `_` separators (PEP 515: `int("a_a", 16)`), - surrounding whitespace. So the documented format `krlk_<64-lowercase-hex>` was not actually enforced — keys like `krlk_+a..a`, `krlk_0x..`, or `krlk_a_a` were accepted by the middleware, hashed verbatim, and looked up against the database, bypassing the canonical form that `generate_api_key()` produces. This commit: - Replaces the `int(hex_part, 16)` check in the canonical api_keys.py implementation with an anchored regex (`[0-9a-f]{64}`) so only the exact lowercase 64-hex form is valid. - Imports the fixed implementation into auth.py and keeps the public `validate_key_format` name as a thin re-export, so the middleware shares the same logic as the canonical utils copy. This prevents the two copies from drifting again — the original bug class. New tests/api/test_validate_key_format.py covers 21 cases: - canonical format and `generate_api_key()` output always pass - 15 non-canonical variants are rejected (signs, 0x prefix, underscore, uppercase, whitespace, wrong length, missing prefix, empty) - both copies (utils and middleware) enforce the same contract Fixes #2427. --- src/memos/api/middleware/auth.py | 28 +++++--- src/memos/api/utils/api_keys.py | 20 +++--- tests/api/test_validate_key_format.py | 99 +++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 20 deletions(-) create mode 100644 tests/api/test_validate_key_format.py diff --git a/src/memos/api/middleware/auth.py b/src/memos/api/middleware/auth.py index 15b217651..6d812337b 100644 --- a/src/memos/api/middleware/auth.py +++ b/src/memos/api/middleware/auth.py @@ -17,6 +17,13 @@ import memos.log +# Canonical format check; re-exported here so the middleware shares the +# same logic as ``src/memos/api/utils/api_keys.py``. Keeping one source of +# truth is what fixes issue #2427 (previously the middleware copy and the +# utils copy could drift independently). +from memos.api.utils.api_keys import validate_key_format as validate_key_format_canonical + + logger = memos.log.get_logger(__name__) # API key header configuration @@ -63,17 +70,16 @@ def hash_api_key(key: str) -> str: def validate_key_format(key: str) -> bool: - """Validate API key format: krlk_<64-hex>.""" - if not key or not key.startswith("krlk_"): - return False - hex_part = key[5:] # Remove 'krlk_' prefix - if len(hex_part) != 64: - return False - try: - int(hex_part, 16) - return True - except ValueError: - return False + """Validate API key format: krlk_<64-hex>. + + Re-exported from ``memos.api.utils.api_keys.validate_key_format`` for + backwards compatibility with callers that imported this symbol from + the middleware path. Issue #2427: the previous copy used ``int(...)`` + which accepted non-canonical hex (leading signs, ``0x`` prefix, ``_`` + separators, whitespace) — see the canonical implementation for the + fix. + """ + return validate_key_format_canonical(key) def get_key_prefix(key: str) -> str: diff --git a/src/memos/api/utils/api_keys.py b/src/memos/api/utils/api_keys.py index 29b493fd0..123420513 100644 --- a/src/memos/api/utils/api_keys.py +++ b/src/memos/api/utils/api_keys.py @@ -5,6 +5,7 @@ """ import hashlib +import re import secrets from dataclasses import dataclass @@ -45,11 +46,19 @@ def hash_key(key: str) -> str: return hashlib.sha256(key.encode()).hexdigest() +# Match the only form ``generate_api_key()`` produces: ``krlk_`` followed by +# exactly 64 lowercase hex characters. Using ``int(hex_part, 16)`` would also +# accept leading signs, ``0x`` prefixes, ``_`` separators, and surrounding +# whitespace, none of which a real key ever carries (issue #2427). +_KEY_FORMAT_HEX_RE = re.compile(r"[0-9a-f]{64}") + + def validate_key_format(key: str) -> bool: """ Validate API key format. - Valid format: krlk_<64-hex-chars> + Valid format: ``krlk_<64-hex-chars>`` where the hex part is exactly 64 + characters from ``[0-9a-f]`` — the only form ``generate_api_key()`` produces. """ if not key or not isinstance(key, str): return False @@ -58,14 +67,7 @@ def validate_key_format(key: str) -> bool: return False hex_part = key[5:] - if len(hex_part) != 64: - return False - - try: - int(hex_part, 16) - return True - except ValueError: - return False + return _KEY_FORMAT_HEX_RE.fullmatch(hex_part) is not None def generate_master_key() -> tuple[str, str]: diff --git a/tests/api/test_validate_key_format.py b/tests/api/test_validate_key_format.py new file mode 100644 index 000000000..757d2519a --- /dev/null +++ b/tests/api/test_validate_key_format.py @@ -0,0 +1,99 @@ +"""Regression tests for issue #2427. + +``validate_key_format`` exists in two copies — ``src/memos/api/utils/api_keys.py`` +(the canonical home) and ``src/memos/api/middleware/auth.py`` (the +middleware gating every authenticated request). Both check the hex part +with ``int(hex_part, 16)``, which accepts non-canonical forms a generated +key can never have: + +- leading ``+`` / ``-`` (sign) +- ``0x`` / ``0X`` prefix +- ``_`` separators (PEP 515) +- surrounding whitespace + +So the documented format ``krlk_<64-hex-chars>`` is not actually enforced: +keys with embedded underscores, signs, or ``0x`` prefixes pass the format +gate and are hashed/looked up as-is, bypassing the canonical form that +``generate_api_key()`` produces. + +These tests pin the new contract on both validator copies: + +- the hex part must match ``[0-9a-f]{64}`` exactly +- the only valid form is the canonical lowercase hex string +""" + +from __future__ import annotations + +import pytest + +from memos.api.middleware.auth import validate_key_format as validate_in_auth +from memos.api.utils.api_keys import generate_api_key, validate_key_format + + +CANONICAL = "krlk_" + "0123456789abcdef" * 4 # 64 hex chars, lowercase + + +class TestValidateKeyFormatUtils: + def test_canonical_key_passes(self): + assert validate_key_format(CANONICAL) is True + + def test_generated_key_always_passes(self): + # The validator must accept every key that generate_api_key() + # produces, since that is the documented format. + for _ in range(20): + assert validate_key_format(generate_api_key().key) is True + + @pytest.mark.parametrize( + "bad_key", + [ + # Leading sign + "krlk_" + "+" + "a" * 63, + "krlk_" + "-" + "a" * 63, + # 0x prefix + "krlk_" + "0x" + "a" * 62, + "krlk_" + "0X" + "a" * 62, + # Underscore separator (PEP 515) + "krlk_" + "a" * 32 + "_" + "b" * 31, + # Uppercase hex (the generator emits lowercase only) + "krlk_" + "A" * 64, + # Surrounding whitespace + " krlk_" + "a" * 64, + "krlk_" + "a" * 64 + " ", + # Wrong total length + "krlk_" + "a" * 63, + "krlk_" + "a" * 65, + # Empty hex part + "krlk_", + # Missing prefix + "0123456789abcdef" * 4, + "Bearer " + CANONICAL, + # Non-string + "", + ], + ) + def test_non_canonical_keys_rejected(self, bad_key): + assert validate_key_format(bad_key) is False, ( + f"validate_key_format({bad_key!r}) should return False" + ) + + +class TestAuthMiddlewareValidator: + """The middleware copy must enforce the same contract as the canonical + one — issue #2427 documents both copies accepting non-canonical hex.""" + + def test_canonical_key_passes(self): + assert validate_in_auth(CANONICAL) is True + + def test_signed_hex_rejected(self): + # The most surprising bit: ``int("+a..", 16)`` succeeds, so a signed + # hex passes the middleware's validator today. + assert validate_in_auth("krlk_" + "+" + "a" * 63) is False + + def test_underscore_rejected(self): + assert validate_in_auth("krlk_" + "a" * 32 + "_" + "b" * 31) is False + + def test_uppercase_rejected(self): + assert validate_in_auth("krlk_" + "A" * 64) is False + + def test_0x_prefix_rejected(self): + assert validate_in_auth("krlk_" + "0x" + "a" * 62) is False From faec48d7695ed5992cee81492ad72044fe1949d9 Mon Sep 17 00:00:00 2001 From: Wenqiang Wei <46308778+endxxxx@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:33:55 +0800 Subject: [PATCH 13/15] fix: preserve identifier types in PolarDB lightweight recall --- src/memos/graph_dbs/polardb.py | 25 +++++++- tests/graph_dbs/test_search_return_fields.py | 67 ++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/src/memos/graph_dbs/polardb.py b/src/memos/graph_dbs/polardb.py index bf74fbb8b..e7a8c4306 100644 --- a/src/memos/graph_dbs/polardb.py +++ b/src/memos/graph_dbs/polardb.py @@ -33,9 +33,30 @@ def _build_lightweight_return_columns(return_fields: list[str]) -> str: return "".join(f",\n {column}" for column in columns) -def _decode_lightweight_return_value(value: Any) -> Any: +def _decode_lightweight_return_value(value: Any, field: str | None = None) -> Any: if hasattr(value, "value"): value = value.value + # Drivers may return agtype identifiers as already-decoded scalar strings. + # Parsing a numeric ID again would change its type. + identifier_fields = { + "user_id", + "user_name", + "session_id", + "mem_cube_id", + "dialog_id", + "working_binding", + "project_id", + "manager_user_id", + } + if field in identifier_fields: + if isinstance(value, str): + try: + decoded = json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return decoded if decoded is None or isinstance(decoded, str) else value + if isinstance(value, int) and not isinstance(value, bool): + return str(value) if isinstance(value, str): try: return json.loads(value) @@ -1990,7 +2011,7 @@ def search_by_embedding( item = {"id": id_val, "score": score_val} if light_weight_mode: for offset, field in enumerate(validated_return_fields, start=5): - item[field] = _decode_lightweight_return_value(row[offset]) + item[field] = _decode_lightweight_return_value(row[offset], field=field) elif validated_return_fields: properties = row[1] # properties column item.update( diff --git a/tests/graph_dbs/test_search_return_fields.py b/tests/graph_dbs/test_search_return_fields.py index 1ed0f6173..a23374e5a 100644 --- a/tests/graph_dbs/test_search_return_fields.py +++ b/tests/graph_dbs/test_search_return_fields.py @@ -264,6 +264,73 @@ def fake_connection(): def test_polardb_declares_lightweight_vector_search_support(self, polardb_instance): assert polardb_instance.supports_lightweight_vector_search is True + @pytest.mark.parametrize( + "stored_user_id,expected_user_id", + [ + ("1234567890123456789", "1234567890123456789"), + (1234567890123456789, "1234567890123456789"), + ('"1234567890123456789"', "1234567890123456789"), + ("00123", "00123"), + (None, None), + ("null", None), + ], + ) + def test_lightweight_numeric_user_id_can_be_recalled( + self, polardb_instance, stored_user_id, expected_user_id + ): + from memos.memories.textual.tree_text_memory.retrieve.recall import GraphMemoryRetriever + + polardb_instance.db_name = "test_db" + polardb_instance.config = {"user_name": "default_user"} + polardb_instance._build_user_name_and_kb_ids_conditions_sql = MagicMock(return_value=[]) + polardb_instance._build_filter_conditions_sql = MagicMock(return_value=[]) + cursor = MagicMock() + cursor.fetchall.return_value = [ + ( + 1, + None, + None, + '"00000000-0000-4000-8000-000000000001"', + 0.2, + '"synthetic knowledge"', + '"LongTermMemory"', + stored_user_id, + '["synthetic"]', + '{"test":true}', + "0.9", + "true", + 0.8, + ) + ] + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + + @contextmanager + def fake_connection(): + yield connection + + polardb_instance._get_connection = fake_connection + hits = polardb_instance.search_by_embedding( + vector=[0.1] * 1024, + user_name="knowledge-cube", + light_weight_mode=True, + return_fields=[ + "memory", + "memory_type", + "user_id", + "tags", + "info", + "confidence", + "is_fast", + ], + ) + memory = GraphMemoryRetriever._memory_item_from_vector_hit(hits[0]) + assert memory.metadata.user_id == expected_user_id + assert memory.metadata.tags == ["synthetic"] + assert memory.metadata.info == {"test": True} + assert memory.metadata.confidence == 0.9 + assert memory.metadata.is_fast is True + class TestFieldNameValidation: """Tests for _validate_return_fields injection prevention.""" From b1a7d5fee60c06923933550c9c9c0755c54e383c Mon Sep 17 00:00:00 2001 From: Wenqiang Wei <46308778+endxxxx@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:40:53 +0800 Subject: [PATCH 14/15] refactor: reuse lightweight identifier field set --- src/memos/graph_dbs/polardb.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/memos/graph_dbs/polardb.py b/src/memos/graph_dbs/polardb.py index e7a8c4306..8bd9fe76d 100644 --- a/src/memos/graph_dbs/polardb.py +++ b/src/memos/graph_dbs/polardb.py @@ -20,6 +20,19 @@ logger = get_logger(__name__) +_LIGHTWEIGHT_IDENTIFIER_FIELDS = frozenset( + { + "user_id", + "user_name", + "session_id", + "mem_cube_id", + "dialog_id", + "working_binding", + "project_id", + "manager_user_id", + } +) + def _build_lightweight_return_columns(return_fields: list[str]) -> str: columns = [] @@ -38,17 +51,7 @@ def _decode_lightweight_return_value(value: Any, field: str | None = None) -> An value = value.value # Drivers may return agtype identifiers as already-decoded scalar strings. # Parsing a numeric ID again would change its type. - identifier_fields = { - "user_id", - "user_name", - "session_id", - "mem_cube_id", - "dialog_id", - "working_binding", - "project_id", - "manager_user_id", - } - if field in identifier_fields: + if field in _LIGHTWEIGHT_IDENTIFIER_FIELDS: if isinstance(value, str): try: decoded = json.loads(value) From 755e8eb2e08380dd2ba37835aeec8f0169139c80 Mon Sep 17 00:00:00 2001 From: Wenqiang Wei <46308778+endxxxx@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:45:34 +0800 Subject: [PATCH 15/15] chore: change version number to v2.0.36 --- pyproject.toml | 2 +- src/memos/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 314fce3d5..134d380ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ ############################################################################## name = "MemoryOS" -version = "2.0.33" +version = "2.0.36" description = "Intelligence Begins with Memory" license = {text = "Apache-2.0"} readme = "README.md" diff --git a/src/memos/__init__.py b/src/memos/__init__.py index 339107e75..d7df9f7d0 100644 --- a/src/memos/__init__.py +++ b/src/memos/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.33" +__version__ = "2.0.36" from memos.configs.mem_cube import GeneralMemCubeConfig from memos.configs.mem_os import MOSConfig