diff --git a/docker/requirements-full.txt b/docker/requirements-full.txt index b148d43d6..4e800d98c 100644 --- a/docker/requirements-full.txt +++ b/docker/requirements-full.txt @@ -8,7 +8,7 @@ attrs==25.3.0 authlib==1.6.0 beautifulsoup4==4.13.4 cachetools==6.2.1 -certifi==2025.7.14 +certifi==2026.10.19 cffi==1.17.1 charset-normalizer==3.4.2 chonkie==1.1.1 diff --git a/docker/requirements.txt b/docker/requirements.txt index 988e64b83..15b5685f7 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -4,7 +4,7 @@ attrs==25.4.0 Authlib==1.6.5 beartype==0.22.5 cachetools==6.2.2 -certifi==2025.11.12 +certifi==2026.10.19 cffi==2.0.0 charset-normalizer==3.4.4 chonkie==1.1.0 diff --git a/docs/cn/open_source/contribution/development_workflow.md b/docs/cn/open_source/contribution/development_workflow.md index 088784fb8..0f8353998 100644 --- a/docs/cn/open_source/contribution/development_workflow.md +++ b/docs/cn/open_source/contribution/development_workflow.md @@ -48,7 +48,7 @@ git fetch upstream git rebase upstream/dev # 把你的 feat 分支基于最新 dev 重放 ``` -提交更改时请遵循项目的提交规范(参见 [提交规范](commit_guidelines.md))。 +提交更改时请遵循项目的提交规范(参见 [提交规范](./commit_guidelines.md))。 #### 推送到你的 Fork 仓库 diff --git a/docs/cn/open_source/contribution/overview.md b/docs/cn/open_source/contribution/overview.md index 83f58c4bc..61678a8ac 100644 --- a/docs/cn/open_source/contribution/overview.md +++ b/docs/cn/open_source/contribution/overview.md @@ -3,11 +3,11 @@ title: 参与 MemOS 开发 desc: 欢迎阅读 MemOS 贡献指南!了解如何配置开发环境、遵循我们的开发流程、撰写规范的提交信息、完善文档以及添加测试用例。 --- -- **首次贡献者:** 请先阅读 [环境配置指南](setting_up.md),以准备你的开发环境。 -- **准备开始编码了吗?** 请查看 [开发流程](development_workflow.md) 指南,了解我们提交更改的流程。 -- **撰写规范的提交信息:** 请参考我们的 [提交规范](commit_guidelines.md)。 -- **参与文档编写:** 如果你正在帮助我们改进文档,请阅读 [文档编写指南](writing_docs.md)。 -- **添加或完善测试用例:** 请参考 [测试编写指南](writing_tests.md)。 +- **首次贡献者:** 请先阅读 [环境配置指南](./setting_up.md),以准备你的开发环境。 +- **准备开始编码了吗?** 请查看 [开发流程](./development_workflow.md) 指南,了解我们提交更改的流程。 +- **撰写规范的提交信息:** 请参考我们的 [提交规范](./commit_guidelines.md)。 +- **参与文档编写:** 如果你正在帮助我们改进文档,请阅读 [文档编写指南](./writing_docs.md)。 +- **添加或完善测试用例:** 请参考 [测试编写指南](./writing_tests.md)。 你的贡献让这个项目更加优秀!✨ 如果有任何问题,欢迎提交 issue 或参与讨论,也可以扫描下方二维码加入我们的 Discord 或微信社群与我们联系。 diff --git a/poetry.lock b/poetry.lock index 915204888..3aa5c9268 100644 --- a/poetry.lock +++ b/poetry.lock @@ -184,14 +184,14 @@ files = [ [[package]] name = "certifi" -version = "2025.7.14" +version = "2026.10.19" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" groups = ["main", "eval"] files = [ - {file = "certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2"}, - {file = "certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995"}, + {file = "certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b"}, + {file = "certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 314fce3d5..134d380ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ ############################################################################## name = "MemoryOS" -version = "2.0.33" +version = "2.0.36" description = "Intelligence Begins with Memory" license = {text = "Apache-2.0"} readme = "README.md" diff --git a/src/memos/__init__.py b/src/memos/__init__.py index 339107e75..d7df9f7d0 100644 --- a/src/memos/__init__.py +++ b/src/memos/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.33" +__version__ = "2.0.36" from memos.configs.mem_cube import GeneralMemCubeConfig from memos.configs.mem_os import MOSConfig diff --git a/src/memos/api/config.py b/src/memos/api/config.py index 0247dfa42..f09e51335 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -906,8 +906,8 @@ def get_polardb_config(user_id: str | None = None) -> dict[str, Any]: def get_postgres_config(user_id: str | None = None) -> dict[str, Any]: """Get PostgreSQL + pgvector configuration for MemOS graph storage. - Uses standard PostgreSQL with pgvector extension. - Schema: memos.memories, memos.edges + Uses standard PostgreSQL with pgvector. Shard count, index, and + full-text configuration keep the backend defaults unless overridden. """ user_name = os.getenv("MEMOS_USER_NAME", "default") if user_id: @@ -922,7 +922,7 @@ def get_postgres_config(user_id: str | None = None) -> dict[str, Any]: "schema_name": os.getenv("MEMOS_SCHEMA", "memos"), "user_name": user_name, "use_multi_db": False, - "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "384")), + "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "1024")), "maxconn": int(os.getenv("POSTGRES_MAX_CONN", "20")), } diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py index 58a96cd75..31acec6d0 100644 --- a/src/memos/api/handlers/chat_handler.py +++ b/src/memos/api/handlers/chat_handler.py @@ -482,12 +482,8 @@ def generate_chat_response() -> Generator[str, None, None]: yield f"data: {json.dumps({'type': 'status', 'data': '1'})}\n\n" - # Extract memories from search results (first search) - memories_list = [] - if search_response.data and search_response.data.get("text_mem"): - text_mem_results = search_response.data["text_mem"] - if text_mem_results and text_mem_results[0].get("memories"): - memories_list = text_mem_results[0]["memories"] + # Extract memories from every searched cube (first search) + memories_list = self._memories_from_cube_results(search_response.data) # Filter memories by threshold filtered_memories = self._filter_memories_by_threshold(memories_list)[:5] @@ -577,12 +573,8 @@ def generate_chat_response() -> Generator[str, None, None]: project_id=chat_req.project_id, ) - # Extract memories from search results (second search) - memories_list = [] - if search_response.data and search_response.data.get("text_mem"): - text_mem_results = search_response.data["text_mem"] - if text_mem_results and text_mem_results[0].get("memories"): - memories_list = text_mem_results[0]["memories"] + # Extract memories from every searched cube (second search) + memories_list = self._memories_from_cube_results(search_response.data) # Filter memories by threshold, min_num is the min number of memories for playground second_filtered_memories = self._filter_memories_by_threshold( @@ -1161,6 +1153,14 @@ def _format_mem_block( return "\n".join(lines_o), "\n".join(lines_p) + @staticmethod + def _memories_from_cube_results(search_data: dict[str, Any] | None) -> list[Any]: + """Collect textual memories from every cube in a search response.""" + memories_list: list[Any] = [] + for cube_result in (search_data or {}).get("text_mem") or []: + memories_list.extend(cube_result.get("memories") or []) + return memories_list + def _filter_memories_by_threshold( self, memories: list, diff --git a/src/memos/api/middleware/auth.py b/src/memos/api/middleware/auth.py index 15b217651..6d812337b 100644 --- a/src/memos/api/middleware/auth.py +++ b/src/memos/api/middleware/auth.py @@ -17,6 +17,13 @@ import memos.log +# Canonical format check; re-exported here so the middleware shares the +# same logic as ``src/memos/api/utils/api_keys.py``. Keeping one source of +# truth is what fixes issue #2427 (previously the middleware copy and the +# utils copy could drift independently). +from memos.api.utils.api_keys import validate_key_format as validate_key_format_canonical + + logger = memos.log.get_logger(__name__) # API key header configuration @@ -63,17 +70,16 @@ def hash_api_key(key: str) -> str: def validate_key_format(key: str) -> bool: - """Validate API key format: krlk_<64-hex>.""" - if not key or not key.startswith("krlk_"): - return False - hex_part = key[5:] # Remove 'krlk_' prefix - if len(hex_part) != 64: - return False - try: - int(hex_part, 16) - return True - except ValueError: - return False + """Validate API key format: krlk_<64-hex>. + + Re-exported from ``memos.api.utils.api_keys.validate_key_format`` for + backwards compatibility with callers that imported this symbol from + the middleware path. Issue #2427: the previous copy used ``int(...)`` + which accepted non-canonical hex (leading signs, ``0x`` prefix, ``_`` + separators, whitespace) — see the canonical implementation for the + fix. + """ + return validate_key_format_canonical(key) def get_key_prefix(key: str) -> str: diff --git a/src/memos/api/middleware/rate_limit.py b/src/memos/api/middleware/rate_limit.py index c547378ca..72808a13d 100644 --- a/src/memos/api/middleware/rate_limit.py +++ b/src/memos/api/middleware/rate_limit.py @@ -57,8 +57,13 @@ def _get_client_key(request: Request) -> str: Uses API key if available, otherwise falls back to IP. """ - # Try to get API key from header + # Try to get API key from header. Real clients send a Bearer scheme + # per RFC 6750 (``Authorization: Bearer krlk_``); we strip the + # scheme before the prefix check so keyed requests actually land on + # a per-key bucket (issue #2428). auth_header = request.headers.get("Authorization", "") + if auth_header.lower().startswith("bearer "): + auth_header = auth_header[len("bearer ") :].lstrip() if auth_header.startswith("krlk_"): # Use first 20 chars of key as identifier return f"ratelimit:key:{auth_header[:20]}" diff --git a/src/memos/api/utils/api_keys.py b/src/memos/api/utils/api_keys.py index 29b493fd0..123420513 100644 --- a/src/memos/api/utils/api_keys.py +++ b/src/memos/api/utils/api_keys.py @@ -5,6 +5,7 @@ """ import hashlib +import re import secrets from dataclasses import dataclass @@ -45,11 +46,19 @@ def hash_key(key: str) -> str: return hashlib.sha256(key.encode()).hexdigest() +# Match the only form ``generate_api_key()`` produces: ``krlk_`` followed by +# exactly 64 lowercase hex characters. Using ``int(hex_part, 16)`` would also +# accept leading signs, ``0x`` prefixes, ``_`` separators, and surrounding +# whitespace, none of which a real key ever carries (issue #2427). +_KEY_FORMAT_HEX_RE = re.compile(r"[0-9a-f]{64}") + + def validate_key_format(key: str) -> bool: """ Validate API key format. - Valid format: krlk_<64-hex-chars> + Valid format: ``krlk_<64-hex-chars>`` where the hex part is exactly 64 + characters from ``[0-9a-f]`` — the only form ``generate_api_key()`` produces. """ if not key or not isinstance(key, str): return False @@ -58,14 +67,7 @@ def validate_key_format(key: str) -> bool: return False hex_part = key[5:] - if len(hex_part) != 64: - return False - - try: - int(hex_part, 16) - return True - except ValueError: - return False + return _KEY_FORMAT_HEX_RE.fullmatch(hex_part) is not None def generate_master_key() -> tuple[str, str]: diff --git a/src/memos/configs/graph_db.py b/src/memos/configs/graph_db.py index 98de09812..984c95060 100644 --- a/src/memos/configs/graph_db.py +++ b/src/memos/configs/graph_db.py @@ -1,4 +1,4 @@ -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from pydantic import BaseModel, Field, field_validator, model_validator @@ -195,8 +195,7 @@ class PostgresGraphDBConfig(BaseConfig): Does NOT require Apache AGE or other graph extensions. Schema: - - memos_memories: Main table for memory nodes (id, memory, properties JSONB, embedding vector) - - memos_edges: Edge table for relationships (source_id, target_id, type) + - {schema}_graph_{n}."Memory" and "Edges", one physical shard per tenant Example: --- @@ -223,13 +222,33 @@ class PostgresGraphDBConfig(BaseConfig): default=False, description="If False: use single database with logical isolation by user_name", ) - embedding_dimension: int = Field( - default=768, description="Dimension of vector embedding (768 for all-mpnet-base-v2)" + embedding_dimension: int = Field(default=1024, description="Dimension of vector embedding") + shard_count: int = Field( + default=400, + ge=1, + le=4096, + description="Number of MD5 tenant shards", ) maxconn: int = Field( default=20, description="Maximum number of connections in the connection pool", ) + minconn: int = Field( + default=8, + ge=1, + description="Warm connections kept in the pool", + ) + vector_index_type: Literal["hnsw", "ivfflat"] = Field( + default="hnsw", + description="pgvector approximate index type", + ) + hnsw_m: int = Field(default=16, ge=2, le=100) + hnsw_ef_construction: int = Field(default=64, ge=4, le=1000) + ivfflat_lists: int = Field(default=100, ge=1) + fulltext_config: str = Field( + default="jiebacfg", + description="PostgreSQL text search configuration used to build search_vector", + ) @model_validator(mode="after") def validate_config(self): @@ -238,6 +257,10 @@ def validate_config(self): raise ValueError("`db_name` must be provided") if not self.use_multi_db and not self.user_name: raise ValueError("In single-database mode, `user_name` must be provided") + if self.minconn > self.maxconn: + raise ValueError(f"`minconn` ({self.minconn}) must be <= `maxconn` ({self.maxconn})") + if not self.fulltext_config.replace(".", "_").isidentifier(): + raise ValueError("`fulltext_config` must be a valid PostgreSQL configuration name") return self diff --git a/src/memos/graph_dbs/polardb.py b/src/memos/graph_dbs/polardb.py index bf74fbb8b..8bd9fe76d 100644 --- a/src/memos/graph_dbs/polardb.py +++ b/src/memos/graph_dbs/polardb.py @@ -20,6 +20,19 @@ logger = get_logger(__name__) +_LIGHTWEIGHT_IDENTIFIER_FIELDS = frozenset( + { + "user_id", + "user_name", + "session_id", + "mem_cube_id", + "dialog_id", + "working_binding", + "project_id", + "manager_user_id", + } +) + def _build_lightweight_return_columns(return_fields: list[str]) -> str: columns = [] @@ -33,9 +46,20 @@ def _build_lightweight_return_columns(return_fields: list[str]) -> str: return "".join(f",\n {column}" for column in columns) -def _decode_lightweight_return_value(value: Any) -> Any: +def _decode_lightweight_return_value(value: Any, field: str | None = None) -> Any: if hasattr(value, "value"): value = value.value + # Drivers may return agtype identifiers as already-decoded scalar strings. + # Parsing a numeric ID again would change its type. + if field in _LIGHTWEIGHT_IDENTIFIER_FIELDS: + if isinstance(value, str): + try: + decoded = json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return decoded if decoded is None or isinstance(decoded, str) else value + if isinstance(value, int) and not isinstance(value, bool): + return str(value) if isinstance(value, str): try: return json.loads(value) @@ -1990,7 +2014,7 @@ def search_by_embedding( item = {"id": id_val, "score": score_val} if light_weight_mode: for offset, field in enumerate(validated_return_fields, start=5): - item[field] = _decode_lightweight_return_value(row[offset]) + item[field] = _decode_lightweight_return_value(row[offset], field=field) elif validated_return_fields: properties = row[1] # properties column item.update( diff --git a/src/memos/graph_dbs/postgres.py b/src/memos/graph_dbs/postgres.py index 594f7e695..2820a8b54 100644 --- a/src/memos/graph_dbs/postgres.py +++ b/src/memos/graph_dbs/postgres.py @@ -5,16 +5,18 @@ No Apache AGE or other graph extensions required. Tables: -- {schema}.memories: Memory nodes with JSONB properties and vector embeddings -- {schema}.edges: Relationships between memory nodes +- {schema}_graph_{n}."Memory": one tenant shard of memory nodes +- {schema}_graph_{n}."Edges": relationships for that same shard """ +import hashlib import json +import random import re import time from contextlib import suppress -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Literal from memos.configs.graph_db import PostgresGraphDBConfig @@ -28,7 +30,7 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: """Ensure metadata has proper datetime fields and normalized types.""" - now = datetime.utcnow().isoformat() + now = datetime.now(timezone.utc).isoformat() metadata.setdefault("created_at", now) metadata.setdefault("updated_at", now) @@ -43,6 +45,9 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: class PostgresGraphDB(BaseGraphDB): """PostgreSQL + pgvector implementation of a graph memory store.""" + supports_lightweight_vector_search = True + prefer_lightweight_vector_search = True + @require_python_package( import_name="psycopg2", install_command="pip install psycopg2-binary", @@ -50,26 +55,29 @@ class PostgresGraphDB(BaseGraphDB): ) def __init__(self, config: PostgresGraphDBConfig): """Initialize PostgreSQL connection pool.""" - import psycopg2 import psycopg2.pool self.config = config + if not self._is_safe_field_name(config.schema_name): + raise ValueError(f"Invalid PostgreSQL schema name: {config.schema_name}") self.schema = config.schema_name + self.db_name = config.db_name self.user_name = config.user_name self._pool_closed = False + self._partition_remainders: dict[str, int] = {} logger.info(f"Connecting to PostgreSQL: {config.host}:{config.port}/{config.db_name}") # Create connection pool self.pool = psycopg2.pool.ThreadedConnectionPool( - minconn=2, + minconn=config.minconn, maxconn=config.maxconn, host=config.host, port=config.port, user=config.user, password=config.password, dbname=config.db_name, - connect_timeout=30, + connect_timeout=10, keepalives_idle=30, keepalives_interval=10, keepalives_count=5, @@ -78,8 +86,39 @@ def __init__(self, config: PostgresGraphDBConfig): # Initialize schema and tables self._init_schema() + def warm_up_search_connections_by_full(self, user_name: str | None = None) -> None: + """Warm one pooled PostgreSQL connection; tenant routing needs no fan-out.""" + conn = self._get_conn() + self._put_conn(conn) + + def get_memory_graph_table_name(self, user_name: str | None) -> str: + """Return the one partition table for this tenant, matching PolarDB shard routing.""" + target = user_name or self.user_name + return self._physical_table("memories", target) + + def create_extension(self) -> None: + self._init_schema() + + def create_graph(self) -> None: + self._init_schema() + + def create_edge(self) -> None: + self._init_schema() + + def create_index( + self, + label: str = "Memory", + vector_property: str = "embedding", + dimensions: int = 1024, + index_name: str = "memory_vector_index", + ) -> None: + self._init_schema() + + def format_param_value(self, value: str | None) -> str: + return "null" if value is None else json.dumps(value, ensure_ascii=False) + def _get_conn(self): - """Get connection from pool with health check.""" + """Get a pooled connection without an extra round trip.""" if self._pool_closed: raise RuntimeError("Connection pool is closed") @@ -91,9 +130,6 @@ def _get_conn(self): self.pool.putconn(conn, close=True) continue conn.autocommit = True - # Health check - with conn.cursor() as cur: - cur.execute("SELECT 1") return conn except Exception as e: if conn: @@ -114,66 +150,145 @@ def _put_conn(self, conn): conn.close() def _init_schema(self): - """Create schema and tables if they don't exist.""" + """Create PolarDB-style MD5 shards with a halfvec Memory table in each.""" + fulltext_config = getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_.]*$", fulltext_config): + raise ValueError(f"Invalid PostgreSQL text search config: {fulltext_config}") + vector_index_type = getattr(self.config, "vector_index_type", "hnsw") + shard_count = int(getattr(self.config, "shard_count", 400)) + if shard_count < 1 or shard_count > 4096: + raise ValueError("PostgreSQL shard_count must be between 1 and 4096") + conn = self._get_conn() try: with conn.cursor() as cur: - # Create schema cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}") - - # Enable pgvector cur.execute("CREATE EXTENSION IF NOT EXISTS vector") - # Create memories table - dim = self.config.embedding_dimension - cur.execute(f""" - CREATE TABLE IF NOT EXISTS {self.schema}.memories ( - id TEXT PRIMARY KEY, - memory TEXT NOT NULL DEFAULT '', - properties JSONB NOT NULL DEFAULT '{{}}', - embedding vector({dim}), - user_name TEXT, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - """) - - # Create edges table - cur.execute(f""" - CREATE TABLE IF NOT EXISTS {self.schema}.edges ( - id SERIAL PRIMARY KEY, - source_id TEXT NOT NULL, - target_id TEXT NOT NULL, - edge_type TEXT NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(source_id, target_id, edge_type) + cur.execute( + """ + SELECT c.relname, c.relkind + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = %s + AND c.relname = ANY(%s) + """, + ( + self.schema, + ["memories", "edges"], + ), + ) + legacy_rows = cur.fetchall() + if isinstance(legacy_rows, list) and legacy_rows: + raise RuntimeError( + f"{self.schema} still has the old memories layout; " + "configure a new MEMOS_SCHEMA for the MD5 shard layout " + "or migrate the existing data" ) - """) - - # Create indexes - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_user - ON {self.schema}.memories(user_name) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_props - ON {self.schema}.memories USING GIN(properties) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_embedding - ON {self.schema}.memories USING ivfflat(embedding vector_cosine_ops) - WITH (lists = 100) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_edges_source - ON {self.schema}.edges(source_id) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_edges_target - ON {self.schema}.edges(target_id) - """) - - logger.info(f"Schema {self.schema} initialized successfully") + + dim = int(self.config.embedding_dimension) + cur.execute( + """ + SELECT COUNT(*) + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE c.relname = 'Memory' + AND c.relkind = 'r' + AND n.nspname ~ %s + """, + (rf"^{re.escape(self.schema)}_graph_[0-9]+$",), + ) + count_row = cur.fetchone() + memory_tables = count_row[0] if count_row and isinstance(count_row[0], int) else -1 + if memory_tables != shard_count: + hnsw_m = int(getattr(self.config, "hnsw_m", 16)) + ef_construction = int(getattr(self.config, "hnsw_ef_construction", 64)) + lists = int(getattr(self.config, "ivfflat_lists", 100)) + statements: list[str] = [] + for remainder in range(shard_count): + shard_schema = f"{self.schema}_graph_{remainder}" + memory = f'"{shard_schema}"."Memory"' + edges = f'"{shard_schema}"."Edges"' + if vector_index_type == "ivfflat": + vector_index = ( + f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec " + f"ON {memory} USING ivfflat (embedding halfvec_cosine_ops) " + f"WITH (lists = {lists})" + ) + else: + vector_index = ( + f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec " + f"ON {memory} USING hnsw (embedding halfvec_cosine_ops) " + f"WITH (m = {hnsw_m}, ef_construction = {ef_construction})" + ) + statements.extend( + [ + f'CREATE SCHEMA IF NOT EXISTS "{shard_schema}"', + f""" + CREATE TABLE IF NOT EXISTS {memory} ( + id TEXT NOT NULL, + memory TEXT NOT NULL DEFAULT '', + properties JSONB NOT NULL DEFAULT '{{}}', + embedding halfvec({dim}), + user_name TEXT NOT NULL, + search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector( + '{fulltext_config}', + coalesce(memory, '') || ' ' || coalesce(properties::text, '') + ) + ) STORED, + deleted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (user_name, id) + ) + """, + f""" + CREATE TABLE IF NOT EXISTS {edges} ( + id BIGSERIAL NOT NULL, + user_name TEXT NOT NULL, + source_id TEXT NOT NULL, + target_id TEXT NOT NULL, + edge_type TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (user_name, id), + UNIQUE(user_name, source_id, target_id, edge_type) + ) + """, + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_user " + f"ON {memory}(user_name)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_props " + f"ON {memory} USING GIN(properties)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_search_vector " + f"ON {memory} USING GIN(search_vector)" + ), + vector_index, + ( + f"CREATE INDEX IF NOT EXISTS idx_edges_source " + f"ON {edges}(user_name, source_id)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_edges_target " + f"ON {edges}(user_name, target_id)" + ), + ( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_tenant_unique " + f"ON {edges}(user_name, source_id, target_id, edge_type)" + ), + ] + ) + cur.execute(";\n".join(statements)) + + logger.info( + "Schema %s initialized successfully with %d MD5 shards", + self.schema, + shard_count, + ) except Exception as e: logger.error(f"Failed to init schema: {e}") raise @@ -197,6 +312,8 @@ def remove_oldest_memory( """ user_name = user_name or self.user_name keep_latest = int(keep_latest) + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: @@ -206,7 +323,7 @@ def remove_oldest_memory( f""" WITH ranked AS ( SELECT id, ROW_NUMBER() OVER (ORDER BY updated_at DESC) as rn - FROM {self.schema}.memories + FROM {memories} WHERE user_name = %s AND properties->>'memory_type' = %s ) @@ -221,101 +338,140 @@ def remove_oldest_memory( # Delete edges first cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) + DELETE FROM {edges} + WHERE user_name = %s + AND (source_id = ANY(%s) OR target_id = ANY(%s)) """, - (ids_to_delete, ids_to_delete), + (user_name, ids_to_delete, ids_to_delete), ) # Delete nodes cur.execute( f""" - DELETE FROM {self.schema}.memories - WHERE id = ANY(%s) + DELETE FROM {memories} + WHERE id = ANY(%s) AND user_name = %s """, - (ids_to_delete,), + (ids_to_delete, user_name), ) logger.info( - f"Removed {len(ids_to_delete)} oldest {memory_type} memories for user {user_name}" + "Removed %d oldest %s memories for user %s", + len(ids_to_delete), + memory_type, + user_name, ) finally: self._put_conn(conn) - def add_node( - self, id: str, memory: str, metadata: dict[str, Any], user_name: str | None = None - ) -> None: - """Add a memory node.""" - user_name = user_name or self.user_name - metadata = _prepare_node_metadata(metadata.copy()) - - # Extract embedding - embedding = metadata.pop("embedding", None) - created_at = metadata.pop("created_at", datetime.utcnow().isoformat()) - updated_at = metadata.pop("updated_at", datetime.utcnow().isoformat()) + def _prepare_node_for_insert( + self, + node_id: str, + memory: str, + metadata: dict[str, Any] | None, + user_name: str, + ) -> tuple[Any, ...]: + """Build one write row with the same property shape PolarDB stores.""" + fields = _prepare_node_metadata(dict(metadata or {})) + fields["user_name"] = user_name + fields.setdefault("delete_time", "") + fields.setdefault("delete_record_id", "") + now = datetime.now(timezone.utc).isoformat() + created_at = fields.pop("created_at", now) + updated_at = fields.pop("updated_at", now) + for field_name in ("sources", "usage"): + value = fields.get(field_name) + if isinstance(value, list): + fields[field_name] = [ + item if isinstance(item, str) else json.dumps(item) for item in value + ] + embedding = fields.pop("embedding", None) + if not isinstance(embedding, list) or not embedding: + embedding = [round(random.uniform(-0.2, 0.2), 6) for _ in range(self._vector_dim())] + else: + embedding = [float(item) for item in embedding] + properties = { + "id": node_id, + "memory": memory, + "created_at": created_at, + "updated_at": updated_at, + "delete_time": fields.get("delete_time", ""), + "delete_record_id": fields.get("delete_record_id", ""), + **fields, + } + return ( + node_id, + memory, + json.dumps(properties, ensure_ascii=False), + embedding, + user_name, + created_at, + updated_at, + ) - # Serialize sources if present - if metadata.get("sources"): - metadata["sources"] = [ - json.dumps(s) if not isinstance(s, str) else s for s in metadata["sources"] - ] + def add_node(self, id: str, memory: str, metadata: dict[str, Any], user_name: str) -> None: + """Add a memory node.""" + if not user_name: + raise ValueError("add_node requires user_name") + self.add_nodes_batch( + [{"id": id, "memory": memory, "metadata": metadata}], + user_name, + ) + def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None: + """Insert every node for one tenant in a single statement.""" + if not nodes: + return + if not user_name: + raise ValueError("add_nodes_batch requires user_name") + rows: list[tuple[Any, ...]] = [] + for node in nodes: + try: + rows.append( + self._prepare_node_for_insert( + node["id"], + node["memory"], + node.get("metadata", {}), + user_name, + ) + ) + except Exception: + logger.error( + "add_nodes_batch failed to prepare node %s", + node.get("id", "unknown"), + exc_info=True, + ) + if not rows: + return + memories = self._physical_table("memories", user_name) + dim = self._vector_dim() + values_sql = ",".join([f"(%s, %s, %s, %s::halfvec({dim}), %s, %s, %s)"] * len(rows)) + params: list[Any] = [value for row in rows for value in row] conn = self._get_conn() try: with conn.cursor() as cur: - if embedding: - cur.execute( - f""" - INSERT INTO {self.schema}.memories - (id, memory, properties, embedding, user_name, created_at, updated_at) - VALUES (%s, %s, %s, %s::vector, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - memory = EXCLUDED.memory, - properties = EXCLUDED.properties, - embedding = EXCLUDED.embedding, - updated_at = EXCLUDED.updated_at - """, - ( - id, - memory, - json.dumps(metadata), - embedding, - user_name, - created_at, - updated_at, - ), - ) - else: - cur.execute( - f""" - INSERT INTO {self.schema}.memories - (id, memory, properties, user_name, created_at, updated_at) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - memory = EXCLUDED.memory, - properties = EXCLUDED.properties, - updated_at = EXCLUDED.updated_at + cur.execute( + f""" + INSERT INTO {memories} + (id, memory, properties, embedding, user_name, created_at, updated_at) + VALUES {values_sql} + ON CONFLICT (user_name, id) DO UPDATE SET + memory = EXCLUDED.memory, + properties = EXCLUDED.properties, + embedding = EXCLUDED.embedding, + updated_at = EXCLUDED.updated_at + WHERE "Memory".user_name = EXCLUDED.user_name """, - (id, memory, json.dumps(metadata), user_name, created_at, updated_at), - ) + params, + ) finally: self._put_conn(conn) - def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str | None = None) -> None: - """Batch add memory nodes.""" - for node in nodes: - self.add_node( - id=node["id"], - memory=node["memory"], - metadata=node.get("metadata", {}), - user_name=user_name, - ) - def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = None) -> None: """Update node fields.""" user_name = user_name or self.user_name if not fields: return + fields = fields.copy() # Get current node current = self.get_node(id, user_name=user_name) @@ -327,7 +483,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N embedding = fields.pop("embedding", None) memory = fields.pop("memory", current.get("memory", "")) props.update(fields) - props["updated_at"] = datetime.utcnow().isoformat() + props["updated_at"] = datetime.now(timezone.utc).isoformat() + memories = self._physical_table("memories", user_name) conn = self._get_conn() try: @@ -335,8 +492,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N if embedding: cur.execute( f""" - UPDATE {self.schema}.memories - SET memory = %s, properties = %s, embedding = %s::vector, updated_at = NOW() + UPDATE {memories} + SET memory = %s, properties = %s, embedding = %s::halfvec({self._vector_dim()}), updated_at = NOW() WHERE id = %s AND user_name = %s """, (memory, json.dumps(props), embedding, id, user_name), @@ -344,7 +501,7 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N else: cur.execute( f""" - UPDATE {self.schema}.memories + UPDATE {memories} SET memory = %s, properties = %s, updated_at = NOW() WHERE id = %s AND user_name = %s """, @@ -356,21 +513,23 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N def delete_node(self, id: str, user_name: str | None = None) -> None: """Delete a node and its edges.""" user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: # Delete edges cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = %s OR target_id = %s + DELETE FROM {edges} + WHERE user_name = %s AND (source_id = %s OR target_id = %s) """, - (id, id), + (user_name, id, id), ) # Delete node cur.execute( f""" - DELETE FROM {self.schema}.memories + DELETE FROM {memories} WHERE id = %s AND user_name = %s """, (id, user_name), @@ -378,22 +537,34 @@ def delete_node(self, id: str, user_name: str | None = None) -> None: finally: self._put_conn(conn) - def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[str, Any] | None: + def get_node( + self, id: str, include_embedding: bool = False, user_name: str | None = None + ) -> dict[str, Any] | None: """Get a single node by ID.""" - user_name = kwargs.get("user_name") or self.user_name + relation = self._tenant_relation("memories", [user_name] if user_name else []) conn = self._get_conn() try: with conn.cursor() as cur: cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" - cur.execute( - f""" - SELECT {cols} FROM {self.schema}.memories - WHERE id = %s AND user_name = %s - """, - (id, user_name), - ) + cols += ", user_name" + if user_name: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = %s AND user_name = %s + """, + (id, user_name), + ) + else: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = %s + """, + (id,), + ) row = cur.fetchone() if not row: return None @@ -402,32 +573,77 @@ def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[s self._put_conn(conn) def get_nodes( - self, ids: list, include_embedding: bool = False, **kwargs + self, ids: list[str], user_name: str | None = None, **kwargs ) -> list[dict[str, Any]]: """Get multiple nodes by IDs.""" if not ids: return [] - user_name = kwargs.get("user_name") or self.user_name + include_embedding = bool(kwargs.get("include_embedding", False)) + relation = self._tenant_relation("memories", [user_name] if user_name else []) conn = self._get_conn() try: with conn.cursor() as cur: cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" - cur.execute( - f""" - SELECT {cols} FROM {self.schema}.memories - WHERE id = ANY(%s) AND user_name = %s - """, - (ids, user_name), - ) + cols += ", user_name" + if user_name: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = ANY(%s) AND user_name = %s + AND COALESCE(properties->>'status', 'activated') <> 'archived' + """, + (ids, user_name), + ) + else: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = ANY(%s) + AND COALESCE(properties->>'status', 'activated') <> 'archived' + """, + (ids,), + ) return [self._parse_row(row, include_embedding) for row in cur.fetchall()] finally: self._put_conn(conn) + @staticmethod + def _normalize_node_properties( + raw_properties: dict[str, Any], user_name: str | None = None + ) -> dict[str, Any]: + """Restore the logical PolarDB metadata shape from relational storage.""" + properties = dict(raw_properties) + if user_name: + properties["user_name"] = user_name + + sources = properties.get("sources") + if isinstance(sources, list): + normalized_sources: list[Any] = [] + for source in sources: + if isinstance(source, str): + try: + parsed_source = json.loads(source) + except (json.JSONDecodeError, TypeError): + parsed_source = {"type": "doc", "content": source} + normalized_sources.append(parsed_source) + elif isinstance(source, dict): + normalized_sources.append(source) + else: + normalized_sources.append({"type": "doc", "content": str(source)}) + properties["sources"] = normalized_sources + + return properties + def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]: """Parse database row to node dict.""" - props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + raw_props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + user_name_index = 6 if include_embedding else 5 + user_name = row[user_name_index] if len(row) > user_name_index else None + props = self._normalize_node_properties(raw_props, user_name=user_name) + props["id"] = row[0] + props["memory"] = row[1] or "" props["created_at"] = row[3].isoformat() if row[3] else None props["updated_at"] = row[4].isoformat() if row[4] else None result = { @@ -436,9 +652,21 @@ def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]: "metadata": props, } if include_embedding and len(row) > 5: - result["metadata"]["embedding"] = row[5] + result["metadata"]["embedding"] = self._coerce_embedding(row[5]) return result + @staticmethod + def _coerce_embedding(value: Any) -> list[float]: + """Normalize pgvector values returned as lists, tuples, or text.""" + if value is None: + return [] + if isinstance(value, str): + value = value.strip().strip("[]") + if not value: + return [] + return [float(item) for item in value.split(",")] + return [float(item) for item in value] + @staticmethod def _is_safe_field_name(field: str) -> bool: """Validate field names used in dynamic SQL fragments.""" @@ -486,31 +714,40 @@ def _build_single_filter_condition( op_map = {"gt": ">", "lt": "<", "gte": ">=", "lte": "<="} sql_op = op_map[op] if raw_key in timestamp_fields or raw_key.endswith("_at"): - parts.append(f"({text_expr})::timestamptz {sql_op} %s::timestamptz") + parts.append(f"({text_expr})::timestamp {sql_op} %s::timestamp") params.append(op_value) else: parts.append(f"NULLIF({text_expr}, '')::numeric {sql_op} %s") params.append(op_value) elif op == "contains": - if raw_key in array_fields: - parts.append(f"{json_expr} @> %s::jsonb") - params.append(json.dumps([op_value])) - else: - parts.append(f"{text_expr} ILIKE %s") - params.append(f"%{op_value}%") + values = op_value if isinstance(op_value, list) else [op_value] + or_parts = [] + for item in values: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([item], ensure_ascii=False)) + parts.append("(" + " OR ".join(or_parts) + ")") elif op == "in": if not isinstance(op_value, list): raise ValueError( - f"in operator expects list for '{key}', got {type(op_value).__name__}" + f"in operator expects list for '{key}', " + f"got {type(op_value).__name__}" ) - if raw_key in array_fields: - parts.append(f"{json_expr} ?| %s") - params.append([str(v) for v in op_value]) + if len(op_value) == 0: + parts.append("FALSE") + elif raw_key in array_fields or raw_key == "related_id": + or_parts = [] + for item in op_value: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([str(item)], ensure_ascii=False)) + parts.append("(" + " OR ".join(or_parts) + ")") else: parts.append(f"{text_expr} = ANY(%s)") params.append([str(v) for v in op_value]) elif op == "like": - parts.append(f"{text_expr} ILIKE %s") + parts.append(f"{text_expr} LIKE %s") + params.append(f"%{op_value}%") + elif op == "nolike": + parts.append(f"{text_expr} NOT LIKE %s") params.append(f"%{op_value}%") else: raise ValueError(f"Unsupported filter operator: {op}") @@ -532,6 +769,7 @@ def _build_single_filter_condition( def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[Any]) -> str: """Build SQL WHERE fragment from filter dict.""" + filter_dict = self.parse_filter(filter_dict) if not filter_dict: return "" @@ -542,7 +780,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A parts: list[str] = [] for cond in and_conditions: if isinstance(cond, dict): - cond_sql = self._build_single_filter_condition(cond, params) + cond_sql = self._build_filter_where_clause(cond, params) if cond_sql: parts.append(f"({cond_sql})") return " AND ".join(parts) @@ -554,7 +792,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A parts: list[str] = [] for cond in or_conditions: if isinstance(cond, dict): - cond_sql = self._build_single_filter_condition(cond, params) + cond_sql = self._build_filter_where_clause(cond, params) if cond_sql: parts.append(f"({cond_sql})") return f"({' OR '.join(parts)})" if parts else "" @@ -562,6 +800,55 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A cond_sql = self._build_single_filter_condition(filter_dict, params) return cond_sql or "" + def parse_filter(self, filter_dict: dict | None = None): + """Apply the same implicit info.* field routing as the PolarDB backend.""" + if filter_dict is None: + return None + full_fields = { + "id", + "key", + "tags", + "type", + "usage", + "memory", + "status", + "sources", + "user_id", + "graph_id", + "user_name", + "background", + "confidence", + "created_at", + "session_id", + "updated_at", + "memory_type", + "node_type", + "info", + "source", + "file_ids", + "project_id", + "manager_user_id", + "delete_time", + "related_id", + "message_ids", + } + + def normalize(condition: Any) -> Any: + if not isinstance(condition, dict): + return condition + normalized: dict[str, Any] = {} + for key, value in condition.items(): + if key.lower() in {"and", "or"} and isinstance(value, list): + normalized[key] = [normalize(item) for item in value] + continue + normalized_key = ( + key if key in full_fields or key.startswith("info.") else f"info.{key}" + ) + normalized[normalized_key] = value + return normalized + + return normalize(filter_dict) + def delete_node_by_prams( self, writable_cube_ids: list[str] | None = None, @@ -571,7 +858,8 @@ def delete_node_by_prams( ) -> int: """Delete nodes by memory_ids, file_ids, or filter.""" logger.info( - "[delete_node_by_prams] memory_ids: %s, file_ids: %s, filter: %s, writable_cube_ids: %s", + "[delete_node_by_prams] memory_ids: %s, file_ids: %s, " + "filter: %s, writable_cube_ids: %s", memory_ids, file_ids, filter, @@ -600,7 +888,8 @@ def delete_node_by_prams( if not where_conditions: logger.warning( - "[delete_node_by_prams] No nodes to delete (no memory_ids, file_ids, or filter provided)" + "[delete_node_by_prams] No nodes to delete " + "(no memory_ids, file_ids, or filter provided)" ) return 0 @@ -609,32 +898,148 @@ def delete_node_by_prams( params.append(writable_cube_ids) where_clause = " AND ".join(where_conditions) + memory_tables = self._physical_tables("memories", writable_cube_ids) + edge_tables = self._physical_tables("edges", writable_cube_ids) conn = self._get_conn() try: with conn.cursor() as cur: - query = f""" - WITH to_delete AS ( - SELECT id - FROM {self.schema}.memories - WHERE {where_clause} - ), - deleted_edges AS ( - DELETE FROM {self.schema}.edges e + deleted_count = 0 + for memories, edges in zip(memory_tables, edge_tables, strict=True): + query = f""" + WITH to_delete AS ( + SELECT id, user_name + FROM {memories} + WHERE {where_clause} + ), + deleted_edges AS ( + DELETE FROM {edges} e + USING to_delete d + WHERE e.user_name = d.user_name + AND (e.source_id = d.id OR e.target_id = d.id) + ) + DELETE FROM {memories} m USING to_delete d - WHERE e.source_id = d.id OR e.target_id = d.id - ) - DELETE FROM {self.schema}.memories m - USING to_delete d - WHERE m.id = d.id - """ - cur.execute(query, params) - deleted_count = cur.rowcount if cur.rowcount is not None else 0 + WHERE m.id = d.id + """ + cur.execute(query, params) + deleted_count += cur.rowcount if cur.rowcount is not None else 0 logger.info("[delete_node_by_prams] Deleted %s nodes", deleted_count) return deleted_count finally: self._put_conn(conn) + @staticmethod + def _normalize_cube_memory_groups( + cube_memory_groups: list[dict[str, Any]], + ) -> list[tuple[str, str]]: + """Normalize and de-duplicate (tenant, memory-id) requests.""" + pairs: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for group in cube_memory_groups: + cube_id = str(group.get("cube_id") or "").strip() + if not cube_id: + continue + for raw_memory_id in group.get("memory_ids") or []: + memory_id = str(raw_memory_id).strip() + pair = (cube_id, memory_id) + if memory_id and pair not in seen: + seen.add(pair) + pairs.append(pair) + return pairs + + def delete_node_by_cube_memory_groups(self, cube_memory_groups: list[dict[str, Any]]) -> int: + """Delete exactly the requested (cube, memory-id) pairs. + + PolarDB locates these pairs in one of 400 schemas. PostgreSQL keeps the + same isolation rule in a shared table by joining both user_name and id. + """ + pairs = self._normalize_cube_memory_groups(cube_memory_groups) + if not pairs: + return 0 + user_names = [pair[0] for pair in pairs] + memory_ids = [pair[1] for pair in pairs] + tenant_names = list(dict.fromkeys(user_names)) + memory_tables = self._physical_tables("memories", tenant_names) + edge_tables = self._physical_tables("edges", tenant_names) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + deleted_count = 0 + for memories, edges in zip(memory_tables, edge_tables, strict=True): + cur.execute( + f""" + WITH requested(user_name, memory_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ), to_delete AS ( + SELECT m.id, m.user_name + FROM {memories} m + JOIN requested + ON requested.user_name = m.user_name + AND requested.memory_id = m.id + ), deleted_edges AS ( + DELETE FROM {edges} e + USING to_delete d + WHERE e.user_name = d.user_name + AND (e.source_id = d.id OR e.target_id = d.id) + ), deleted AS ( + DELETE FROM {memories} m + USING to_delete d + WHERE m.id = d.id AND m.user_name = d.user_name + RETURNING 1 + ) + SELECT COUNT(*) FROM deleted + """, + (user_names, memory_ids), + ) + row = cur.fetchone() + deleted_count += int(row[0]) if row and row[0] is not None else 0 + return deleted_count + finally: + self._put_conn(conn) + + def get_user_names_by_cube_memory_groups( + self, cube_memory_groups: list[dict[str, Any]] + ) -> dict[str, str | None]: + """Resolve memory ownership without scanning unrelated tenants.""" + raw_ids = [ + str(memory_id).strip() + for group in cube_memory_groups + for memory_id in (group.get("memory_ids") or []) + if str(memory_id).strip() + ] + result: dict[str, str | None] = dict.fromkeys(raw_ids) + pairs = self._normalize_cube_memory_groups(cube_memory_groups) + if not pairs: + return result + + user_names = [pair[0] for pair in pairs] + memory_ids = [pair[1] for pair in pairs] + memories = self._tenant_relation("memories", list(dict.fromkeys(user_names)), alias="m") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH requested(user_name, memory_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ) + SELECT requested.memory_id, m.user_name + FROM requested + LEFT JOIN {memories} + ON m.id = requested.memory_id + AND m.user_name = requested.user_name + """, + (user_names, memory_ids), + ) + for memory_id, owner in cur.fetchall(): + if owner: + result[str(memory_id)] = str(owner) + return result + finally: + self._put_conn(conn) + # ========================================================================= # Edge Management # ========================================================================= @@ -643,16 +1048,36 @@ def add_edge( self, source_id: str, target_id: str, type: str, user_name: str | None = None ) -> None: """Create an edge between nodes.""" + user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - INSERT INTO {self.schema}.edges (source_id, target_id, edge_type) - VALUES (%s, %s, %s) - ON CONFLICT (source_id, target_id, edge_type) DO NOTHING + INSERT INTO {edges} + (user_name, source_id, target_id, edge_type) + SELECT %s, %s, %s, %s + WHERE EXISTS ( + SELECT 1 FROM {memories} + WHERE id = %s AND user_name = %s + ) AND EXISTS ( + SELECT 1 FROM {memories} + WHERE id = %s AND user_name = %s + ) + ON CONFLICT (user_name, source_id, target_id, edge_type) DO NOTHING """, - (source_id, target_id, type), + ( + user_name, + source_id, + target_id, + type, + source_id, + user_name, + target_id, + user_name, + ), ) finally: self._put_conn(conn) @@ -661,31 +1086,62 @@ def delete_edge( self, source_id: str, target_id: str, type: str, user_name: str | None = None ) -> None: """Delete an edge.""" + user_name = user_name or self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = %s AND target_id = %s AND edge_type = %s + DELETE FROM {edges} + WHERE user_name = %s + AND source_id = %s AND target_id = %s AND edge_type = %s """, - (source_id, target_id, type), + (user_name, source_id, target_id, type), ) finally: self._put_conn(conn) - def edge_exists(self, source_id: str, target_id: str, type: str) -> bool: + def edge_exists( + self, + source_id: str, + target_id: str, + type: str = "ANY", + direction: str = "OUTGOING", + user_name: str | None = None, + ) -> bool: """Check if edge exists.""" + user_name = user_name or self.user_name + direction = direction.upper() + if direction not in {"OUTGOING", "INCOMING", "ANY"}: + raise ValueError("direction must be OUTGOING, INCOMING, or ANY") + if direction == "INCOMING": + source_id, target_id = target_id, source_id + + direction_sql = "source_id = %s AND target_id = %s" + params: list[Any] = [user_name, source_id, target_id] + if direction == "ANY": + direction_sql = ( + "((source_id = %s AND target_id = %s) OR (source_id = %s AND target_id = %s))" + ) + params = [user_name, source_id, target_id, target_id, source_id] + + type_sql = "" + if type != "ANY": + type_sql = " AND edge_type = %s" + params.append(type) + + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - SELECT 1 FROM {self.schema}.edges - WHERE source_id = %s AND target_id = %s AND edge_type = %s + SELECT 1 FROM {edges} + WHERE user_name = %s AND {direction_sql}{type_sql} LIMIT 1 """, - (source_id, target_id, type), + params, ) return cur.fetchone() is not None finally: @@ -696,43 +1152,132 @@ def edge_exists(self, source_id: str, target_id: str, type: str) -> bool: # ========================================================================= def get_neighbors( - self, id: str, type: str, direction: Literal["in", "out", "both"] = "out" + self, + id: str, + type: str, + direction: Literal["in", "out", "both"] = "out", ) -> list[str]: """Get neighboring node IDs.""" + user_name = self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: if direction == "out": cur.execute( f""" - SELECT target_id FROM {self.schema}.edges - WHERE source_id = %s AND edge_type = %s + SELECT target_id FROM {edges} + WHERE user_name = %s AND source_id = %s AND edge_type = %s """, - (id, type), + (user_name, id, type), ) elif direction == "in": cur.execute( f""" - SELECT source_id FROM {self.schema}.edges - WHERE target_id = %s AND edge_type = %s + SELECT source_id FROM {edges} + WHERE user_name = %s AND target_id = %s AND edge_type = %s """, - (id, type), + (user_name, id, type), ) else: # both cur.execute( f""" - SELECT target_id FROM {self.schema}.edges WHERE source_id = %s AND edge_type = %s + SELECT target_id FROM {edges} + WHERE user_name = %s AND source_id = %s AND edge_type = %s UNION - SELECT source_id FROM {self.schema}.edges WHERE target_id = %s AND edge_type = %s + SELECT source_id FROM {edges} + WHERE user_name = %s AND target_id = %s AND edge_type = %s """, - (id, type, id, type), + (user_name, id, type, user_name, id, type), ) return [row[0] for row in cur.fetchall()] finally: self._put_conn(conn) + def get_children_with_embeddings( + self, id: str, user_name: str | None = None + ) -> list[dict[str, Any]]: + """Return PARENT-edge children with vectors for structure optimization.""" + user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT child.id, child.embedding, child.memory + FROM {edges} e + JOIN {memories} child + ON child.id = e.target_id AND child.user_name = e.user_name + WHERE e.user_name = %s + AND e.source_id = %s + AND e.edge_type = 'PARENT' + AND child.deleted_at IS NULL + ORDER BY e.id + """, + (user_name, id), + ) + return [ + { + "id": row[0], + "embedding": self._coerce_embedding(row[1]), + "memory": row[2] or "", + } + for row in cur.fetchall() + ] + finally: + self._put_conn(conn) + + def get_edges( + self, + id: str, + type: str = "ANY", + direction: str = "ANY", + user_name: str | None = None, + ) -> list[dict[str, str]]: + """Return tenant-scoped edges connected to a node.""" + user_name = user_name or self.user_name + direction = direction.upper() + if direction not in {"OUTGOING", "INCOMING", "ANY"}: + raise ValueError("direction must be OUTGOING, INCOMING, or ANY") + + conditions = ["user_name = %s"] + params: list[Any] = [user_name] + if direction == "OUTGOING": + conditions.append("source_id = %s") + params.append(id) + elif direction == "INCOMING": + conditions.append("target_id = %s") + params.append(id) + else: + conditions.append("(source_id = %s OR target_id = %s)") + params.extend([id, id]) + if type != "ANY": + conditions.append("edge_type = %s") + params.append(type) + + edges = self._physical_table("edges", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT source_id, target_id, edge_type + FROM {edges} + WHERE {" AND ".join(conditions)} + ORDER BY id + """, + params, + ) + return [{"from": row[0], "to": row[1], "type": row[2]} for row in cur.fetchall()] + finally: + self._put_conn(conn) + def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[str]: """Get path between nodes using recursive CTE.""" + user_name = self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: @@ -740,13 +1285,14 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s f""" WITH RECURSIVE path AS ( SELECT source_id, target_id, ARRAY[source_id] as nodes, 1 as depth - FROM {self.schema}.edges - WHERE source_id = %s + FROM {edges} + WHERE user_name = %s AND source_id = %s UNION ALL SELECT e.source_id, e.target_id, p.nodes || e.source_id, p.depth + 1 - FROM {self.schema}.edges e + FROM {edges} e JOIN path p ON e.source_id = p.target_id - WHERE p.depth < %s AND NOT e.source_id = ANY(p.nodes) + WHERE e.user_name = %s + AND p.depth < %s AND NOT e.source_id = ANY(p.nodes) ) SELECT nodes || target_id as full_path FROM path @@ -754,34 +1300,95 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s ORDER BY depth LIMIT 1 """, - (source_id, max_depth, target_id), + (user_name, source_id, user_name, max_depth, target_id), ) row = cur.fetchone() return row[0] if row else [] finally: self._put_conn(conn) - def get_subgraph(self, center_id: str, depth: int = 2) -> list[str]: - """Get subgraph around center node.""" + def get_subgraph( + self, + center_id: str, + depth: int = 2, + center_status: str = "activated", + user_name: str | None = None, + ) -> dict[str, Any]: + """Return the PolarDB-compatible local subgraph structure.""" + if not 1 <= depth <= 5: + raise ValueError("depth must be 1-5") + center_id = center_id.strip('"') + user_name = user_name or self.user_name + core_node = self.get_node(center_id, user_name=user_name) + if core_node is None: + return {"core_node": None, "neighbors": [], "edges": []} + actual_status = core_node.get("metadata", {}).get("status", "activated") + if center_status and actual_status != center_status: + return {"core_node": None, "neighbors": [], "edges": []} + + edges_table = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - WITH RECURSIVE subgraph AS ( - SELECT %s::text as node_id, 0 as level - UNION - SELECT CASE WHEN e.source_id = s.node_id THEN e.target_id ELSE e.source_id END, - s.level + 1 - FROM {self.schema}.edges e + WITH RECURSIVE subgraph(node_id, level, path) AS ( + SELECT %s::text, 0, ARRAY[%s::text] + UNION ALL + SELECT + CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END, + s.level + 1, + s.path || CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END + FROM {edges_table} e JOIN subgraph s ON (e.source_id = s.node_id OR e.target_id = s.node_id) - WHERE s.level < %s + WHERE e.user_name = %s + AND s.level < %s + AND NOT ( + CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END + ) = ANY(s.path) ) - SELECT DISTINCT node_id FROM subgraph - """, - (center_id, depth), + SELECT DISTINCT node_id FROM subgraph ORDER BY node_id + """, + (center_id, center_id, user_name, depth), ) - return [row[0] for row in cur.fetchall()] + node_ids = [str(row[0]) for row in cur.fetchall()] + neighbor_ids = [node_id for node_id in node_ids if node_id != center_id] + neighbors = ( + self.get_nodes(neighbor_ids, user_name=user_name) if neighbor_ids else [] + ) + + if node_ids: + cur.execute( + f""" + SELECT source_id, target_id, edge_type + FROM {edges_table} + WHERE user_name = %s + AND source_id = ANY(%s) + AND target_id = ANY(%s) + ORDER BY id + """, + (user_name, node_ids, node_ids), + ) + edges = [ + {"source": row[0], "target": row[1], "type": row[2]} + for row in cur.fetchall() + ] + else: + edges = [] + return { + "core_node": core_node, + "neighbors": neighbors, + "edges": edges, + } finally: self._put_conn(conn) @@ -789,95 +1396,638 @@ def get_context_chain(self, id: str, type: str = "FOLLOWS") -> list[str]: """Get ordered chain following relationship type.""" return self.get_neighbors(id, type, "out") + def get_neighbors_by_tag( + self, + tags: list[str], + exclude_ids: list[str], + top_k: int = 5, + min_overlap: int = 1, + include_embedding: bool = False, + user_name: str | None = None, + ) -> list[dict[str, Any]]: + """Find active tenant memories ranked by tag overlap.""" + if not tags or top_k <= 0: + return [] + user_name = user_name or self.user_name + params: list[Any] = [user_name] + exclude_sql = "" + if exclude_ids: + exclude_sql = "AND id <> ALL(%s)" + params.append(exclude_ids) + + cols = "id, memory, properties, created_at, updated_at" + if include_embedding: + cols += ", embedding" + cols += ", user_name" + memories = self._physical_table("memories", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT {cols} + FROM {memories} + WHERE user_name = %s + AND COALESCE(properties->>'status', 'activated') = 'activated' + AND COALESCE(properties->>'node_type', '') <> 'reasoning' + AND COALESCE(properties->>'memory_type', '') <> 'WorkingMemory' + {exclude_sql} + """, + params, + ) + ranked: list[tuple[int, dict[str, Any]]] = [] + requested_tags = set(tags) + for row in cur.fetchall(): + node = self._parse_row(row, include_embedding) + node_tags = node.get("metadata", {}).get("tags", []) + if isinstance(node_tags, str): + try: + node_tags = json.loads(node_tags) + except (TypeError, json.JSONDecodeError): + node_tags = [] + overlap = len(requested_tags.intersection(node_tags or [])) + if overlap >= min_overlap: + ranked.append((overlap, node)) + ranked.sort(key=lambda item: item[0], reverse=True) + return [node for _, node in ranked[:top_k]] + finally: + self._put_conn(conn) + # ========================================================================= # Search Operations # ========================================================================= - def search_by_embedding( + def _effective_user_names( self, - vector: list[float], - top_k: int = 5, - scope: str | None = None, - status: str | None = None, - threshold: float | None = None, - search_filter: dict | None = None, user_name: str | None = None, - filter: dict | None = None, knowledgebase_ids: list[str] | None = None, - **kwargs, - ) -> list[dict]: - """Search nodes by vector similarity using pgvector.""" - user_name = user_name or self.user_name + ) -> list[str]: + """Resolve the memory cubes that a query is allowed to read.""" + names: list[str] = [] + primary = user_name or self.user_name + if primary: + names.append(primary) + for knowledgebase_id in knowledgebase_ids or []: + if knowledgebase_id and knowledgebase_id not in names: + names.append(knowledgebase_id) + return names + + def _shard_count(self) -> int: + return int(getattr(self.config, "shard_count", 400)) + + def _remainder_cache(self) -> dict[str, int]: + cache = getattr(self, "_partition_remainders", None) + if cache is None: + cache = {} + self._partition_remainders = cache + return cache + + def _partition_remainder(self, user_name: str, conn=None) -> int: + """Return the PolarDB MD5 shard number for one tenant.""" + del conn + cache = self._remainder_cache() + cached = cache.get(user_name) + if cached is not None: + return cached + remainder = ( + int(hashlib.md5(user_name.encode("utf-8")).hexdigest(), 16) % self._shard_count() + ) + cache[user_name] = remainder + return remainder - # Build WHERE clause - conditions = ["embedding IS NOT NULL"] - params = [] + def _qualified_partition(self, table: str, remainder: int) -> str: + physical_names = { + "memories": "Memory", + "edges": "Edges", + } + physical = physical_names.get(table) + if physical is None: + raise ValueError(f"Invalid partitioned table: {table}") + schema = f"{self.schema}_graph_{remainder}" + if not self._is_safe_field_name(schema): + raise ValueError(f"Invalid partition schema: {schema}") + return f'"{schema}"."{physical}"' + + def _all_partitions_relation(self, table: str, alias: str | None = None) -> str: + """Read every hash partition explicitly, as PolarDB unions every shard.""" + branches = " UNION ALL ".join( + f"SELECT * FROM {self._qualified_partition(table, remainder)}" + for remainder in range(self._shard_count()) + ) + return f"({branches}) {alias or table}" - if user_name: + def _tenant_relation( + self, + table: str, + user_names: list[str] | None, + alias: str | None = None, + conn=None, + ) -> str: + """Return one child table, or a union of only the touched partitions.""" + names = [name for name in (user_names or []) if name] + if not names: + return self._all_partitions_relation(table, alias) + remainders = sorted({self._partition_remainder(name, conn) for name in names}) + if len(remainders) == 1: + relation = self._qualified_partition(table, remainders[0]) + else: + branches = " UNION ALL ".join( + f"SELECT * FROM {self._qualified_partition(table, remainder)}" + for remainder in remainders + ) + relation = f"({branches})" + if alias is None: + alias = table + return f"{relation} {alias}" if alias else relation + + def _physical_tables(self, table: str, user_names: list[str] | None, conn=None) -> list[str]: + """Real tables a write can target. A union subquery cannot be updated.""" + names = [name for name in (user_names or []) if name] + if not names: + return [ + self._qualified_partition(table, remainder) + for remainder in range(self._shard_count()) + ] + remainders = sorted({self._partition_remainder(name, conn) for name in names}) + return [self._qualified_partition(table, remainder) for remainder in remainders] + + def _physical_table(self, table: str, user_name: str | None, conn=None) -> str: + """Return the one partition table PolarDB would address for this tenant.""" + if not user_name: + return f"{self.schema}.{table}" + return self._qualified_partition(table, self._partition_remainder(user_name, conn)) + + def _vector_dim(self) -> int: + dim = int(getattr(self.config, "embedding_dimension", 1024)) + if dim < 1 or dim > 4096: + raise ValueError(f"Invalid embedding dimension: {dim}") + return dim + + def _ensure_hnsw_search_settings(self, conn) -> None: + """Match PolarDB's HNSW session settings once for each pooled connection.""" + ready = getattr(self, "_hnsw_ready_ids", None) + if ready is None: + ready = set() + self._hnsw_ready_ids = ready + marker = id(conn) + if marker in ready: + return + with conn.cursor() as cur: + cur.execute( + """ + SELECT set_config('hnsw.ef_search', '100', false), + set_config('hnsw.iterative_scan', 'relaxed_order', false) + """ + ) + ready.add(marker) + + def _projected_field_exprs(self, return_fields: list[str] | None) -> list[tuple[str, str]]: + projected: list[tuple[str, str]] = [] + for field in self._validate_return_fields(return_fields): + if field in {"id", "memory", "score", "user_name", "created_at", "updated_at"}: + continue + if field == "embedding": + projected.append((field, "embedding")) + else: + projected.append((field, f"properties->'{field}'")) + return projected + + @staticmethod + def _decode_projected_value(value: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return value + + def _build_projected_search_result( + self, row: tuple[Any, ...], extra_fields: list[str] + ) -> dict[str, Any]: + created_at, updated_at = row[3], row[4] + result: dict[str, Any] = { + "id": row[0], + "score": float(row[1]), + "memory": row[2] or "", + "user_name": row[5], + } + if created_at is not None: + result["created_at"] = ( + created_at.isoformat() if hasattr(created_at, "isoformat") else created_at + ) + if updated_at is not None: + result["updated_at"] = ( + updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at + ) + for offset, field in enumerate(extra_fields): + result[field] = self._decode_projected_value(row[6 + offset]) + return result + + def _append_user_name_condition( + self, conditions: list[str], params: list[Any], user_names: list[str] + ) -> None: + if len(user_names) == 1: conditions.append("user_name = %s") - params.append(user_name) + params.append(user_names[0]) + elif user_names: + conditions.append("user_name = ANY(%s)") + params.append(user_names) + def _build_search_conditions( + self, + params: list[Any], + *, + user_name: str | None = None, + knowledgebase_ids: list[str] | None = None, + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + filter: dict | None = None, + default_activated: bool = True, + ) -> list[str]: + conditions: list[str] = [] + user_names = self._effective_user_names(user_name, knowledgebase_ids) + self._append_user_name_condition(conditions, params, user_names) if scope: conditions.append("properties->>'memory_type' = %s") params.append(scope) - if status: conditions.append("properties->>'status' = %s") params.append(status) - else: - conditions.append( - "(properties->>'status' = 'activated' OR properties->>'status' IS NULL)" - ) + elif default_activated: + conditions.append("properties->>'status' = 'activated'") + + for filter_dict in (search_filter, filter): + if filter_dict: + filter_sql = self._build_filter_where_clause(filter_dict, params) + if filter_sql: + conditions.append(f"({filter_sql})") + return conditions + + def _build_search_result( + self, row: tuple[Any, ...], return_fields: list[str] | None + ) -> dict[str, Any]: + result: dict[str, Any] = {"id": row[0], "score": float(row[1])} + fields = self._validate_return_fields(return_fields) + if not fields: + return result + + memory = row[2] or "" + created_at, updated_at, user_name = row[4], row[5], row[6] + raw_properties = row[3] if isinstance(row[3], dict) else json.loads(row[3] or "{}") + properties = dict(raw_properties) + properties["user_name"] = user_name + properties.update( + { + "id": row[0], + "memory": memory, + "created_at": created_at.isoformat() if created_at else None, + "updated_at": updated_at.isoformat() if updated_at else None, + "user_name": user_name, + } + ) + for field in fields: + if field != "id" and field in properties: + result[field] = properties[field] + return result - if search_filter: - for k, v in search_filter.items(): - conditions.append(f"properties->>'{k}' = %s") - params.append(str(v)) + def _build_keyword_result( + self, row: tuple[Any, ...], return_fields: list[str] | None + ) -> dict[str, Any]: + result: dict[str, Any] = {"id": row[0]} + fields = self._validate_return_fields(return_fields) + if not fields: + return result + + raw_properties = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + properties = dict(raw_properties) + properties["user_name"] = row[5] + properties.update( + { + "id": row[0], + "memory": row[1] or "", + "created_at": row[3].isoformat() if row[3] else None, + "updated_at": row[4].isoformat() if row[4] else None, + "user_name": row[5], + } + ) + for field in fields: + if field != "id" and field in properties: + result[field] = properties[field] + return result + def search_by_embedding( + self, + vector: list[float], + user_name: str, + top_k: int = 5, + scope: str | None = None, + status: str | None = None, + threshold: float | None = None, + search_filter: dict | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + return_fields: list[str] | None = None, + light_weight_mode: bool = False, + **kwargs, + ) -> list[dict]: + """Search nodes by vector similarity using pgvector.""" + if light_weight_mode and not return_fields: + return_fields = [ + "memory", + "memory_type", + "user_name", + "status", + "key", + "tags", + "confidence", + "source", + ] + params: list[Any] = [] + conditions = ["embedding IS NOT NULL"] + conditions.extend( + self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + ) where_clause = " AND ".join(conditions) + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) - # pgvector cosine distance: 1 - (a <=> b) gives similarity score + # Match PolarDB: one distance, index-ordered LIMIT, then drop rows at or past 0.9. + dim = self._vector_dim() + distance_sql = "embedding <=> (SELECT query_vector FROM q)" conn = self._get_conn() try: + self._ensure_hnsw_search_settings(conn) with conn.cursor() as cur: + projected = self._projected_field_exprs(return_fields) if light_weight_mode else [] + if light_weight_mode: + inner_columns = "memory, created_at, updated_at, user_name" + extra_inner = "".join( + f', {expression} AS "{field}"' for field, expression in projected + ) + extra_outer = "".join(f', "{field}"' for field, _ in projected) + else: + inner_columns = "memory, properties, created_at, updated_at, user_name" + extra_inner = "" + extra_outer = "" cur.execute( f""" - SELECT id, 1 - (embedding <=> %s::vector) as score - FROM {self.schema}.memories - WHERE {where_clause} - ORDER BY embedding <=> %s::vector - LIMIT %s - """, - (vector, *params, vector, top_k), + WITH q AS ( + SELECT %s::halfvec({dim}) AS query_vector + ), + t AS MATERIALIZED ( + SELECT COALESCE(properties->>'id', id) AS id, + ({distance_sql}) AS scope_distance, + {inner_columns} + {extra_inner} + FROM {relation} + WHERE {where_clause} + ORDER BY {distance_sql} + LIMIT %s + ) + SELECT id, + ((1 - scope_distance) + 1) / 2.0 AS score, + {inner_columns} + {extra_outer} + FROM t + WHERE scope_distance < 0.9 + """, + (vector, *params, top_k), ) results = [] for row in cur.fetchall(): score = float(row[1]) if threshold is None or score >= threshold: - results.append({"id": row[0], "score": score}) + if light_weight_mode: + results.append( + self._build_projected_search_result( + row, [field for field, _ in projected] + ) + ) + else: + results.append(self._build_search_result(row, return_fields)) return results finally: self._put_conn(conn) - def get_by_metadata( + def search_by_fulltext( self, - filters: list[dict[str, Any]], + query_words: list[str], + top_k: int = 10, + scope: str | None = None, + status: str | None = None, + threshold: float | None = None, + search_filter: dict | None = None, + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + tsvector_field: str = "properties_tsvector_zh", + tsquery_config: str = "jiebacfg", + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Search with PolarDB's AND tsquery on one tenant partition.""" + terms = [str(word).strip() for word in query_words if str(word).strip()] + if not terms: + return [] + + query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config): + raise ValueError(f"Invalid PostgreSQL text search config: {query_config}") + quoted_terms = [] + for term in terms: + if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}: + term = term[1:-1] + quoted_terms.append("'" + term.replace("'", "''") + "'") + params: list[Any] = [" & ".join(quoted_terms)] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH q AS ( + SELECT to_tsquery('{query_config}', %s) AS query + ) + SELECT COALESCE(properties->>'id', id) AS id, + ts_rank(search_vector, q.query) AS score, + memory, properties, created_at, updated_at, user_name + FROM {relation} + CROSS JOIN q + WHERE {where_clause} + AND search_vector @@ q.query + ORDER BY score DESC, updated_at DESC + LIMIT %s + """, + (*params, top_k), + ) + output = [] + for row in cur.fetchall(): + result = self._build_search_result(row, return_fields) + if threshold is None or result["score"] >= threshold: + output.append(result) + return output + finally: + self._put_conn(conn) + + def search_by_keywords_tfidf( + self, + query_words: list[str], + scope: str | None = None, status: str | None = None, + search_filter: dict | None = None, user_name: str | None = None, filter: dict | None = None, knowledgebase_ids: list[str] | None = None, + tsvector_field: str = "properties_tsvector_zh", + tsquery_config: str = "jiebaqry", + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Return keyword matches with PolarDB's OR tsquery and no score.""" + if not user_name: + return [] + terms = [str(word).strip() for word in query_words if str(word).strip()] + if not terms: + return [] + + query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config): + raise ValueError(f"Invalid PostgreSQL text search config: {query_config}") + quoted_terms = [] + for term in terms: + if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}: + term = term[1:-1] + quoted_terms.append("'" + term.replace("'", "''") + "'") + params: list[Any] = [" | ".join(quoted_terms)] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH q AS ( + SELECT to_tsquery('{query_config}', %s) AS query + ) + SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name + FROM {relation} + CROSS JOIN q + WHERE {where_clause} + AND search_vector @@ q.query + """, + (*params,), + ) + return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()] + finally: + self._put_conn(conn) + + def search_by_keywords_like( + self, + query_word: str, + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Fallback substring search compatible with the PolarDB adapter.""" + if not user_name: + return [] + pattern = f"%{query_word}%" + params: list[Any] = [] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + search_filter=search_filter, + filter=filter, + status=status, + scope=scope, + ) + conditions.append("COALESCE(properties->>'memory', '') LIKE %s") + params.append(pattern) + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name + FROM {relation} + WHERE {" AND ".join(conditions)} + """, + params, + ) + return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()] + finally: + self._put_conn(conn) + + def get_by_metadata( + self, + filters: list[dict[str, Any]], + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list | None = None, user_name_flag: bool = True, + status: str | None = None, + **kwargs, ) -> list[str]: """Get node IDs matching metadata filters.""" - user_name = user_name or self.user_name + if not user_name: + raise ValueError("get_by_metadata requires user_name") + conditions: list[str] = [] + params: list[Any] = [] + user_names: list[str] = [] - conditions = [] - params = [] - - if user_name_flag and user_name: - conditions.append("user_name = %s") - params.append(user_name) + if user_name_flag: + user_names = self._effective_user_names(user_name, knowledgebase_ids) + self._append_user_name_condition(conditions, params, user_names) if status: conditions.append("properties->>'status' = %s") @@ -887,29 +2037,52 @@ def get_by_metadata( field = f["field"] op = f.get("op", "=") value = f["value"] + text_expr, json_expr = self._field_expr(field) if op == "=": - conditions.append(f"properties->>'{field}' = %s") + conditions.append(f"{text_expr} = %s") params.append(str(value)) elif op == "in": - placeholders = ",".join(["%s"] * len(value)) - conditions.append(f"properties->>'{field}' IN ({placeholders})") - params.extend([str(v) for v in value]) + if not isinstance(value, list): + raise ValueError("in operator expects a list") + conditions.append(f"{text_expr} = ANY(%s)") + params.append([str(item) for item in value]) elif op in (">", ">=", "<", "<="): - conditions.append(f"(properties->>'{field}')::numeric {op} %s") + conditions.append(f"NULLIF({text_expr}, '')::numeric {op} %s") params.append(value) elif op == "contains": - conditions.append(f"properties->'{field}' @> %s::jsonb") - params.append(json.dumps([value])) + values = value if isinstance(value, list) else [value] + or_parts = [] + for item in values: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([item], ensure_ascii=False)) + conditions.append("(" + " OR ".join(or_parts) + ")") + elif op == "like": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"%{value}%") + elif op == "starts_with": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"{value}%") + elif op == "ends_with": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"%{value}") + else: + raise ValueError(f"Unsupported operator: {op}") + + if filter: + filter_sql = self._build_filter_where_clause(filter, params) + if filter_sql: + conditions.append(f"({filter_sql})") where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation("memories", user_names) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - SELECT id FROM {self.schema}.memories + SELECT id FROM {relation} WHERE {where_clause} """, params, @@ -921,23 +2094,34 @@ def get_by_metadata( def get_all_memory_items( self, scope: str, + user_name: str | None = None, include_embedding: bool = False, - status: str | None = None, filter: dict | None = None, - knowledgebase_ids: list[str] | None = None, - **kwargs, + knowledgebase_ids: list | None = None, + status: str | None = None, ) -> list[dict]: """Get all memory items of a specific type.""" - user_name = kwargs.get("user_name") or self.user_name - - conditions = ["properties->>'memory_type' = %s", "user_name = %s"] - params = [scope, user_name] + if not user_name: + return [] + if scope not in {"WorkingMemory", "LongTermMemory", "UserMemory", "OuterMemory"}: + raise ValueError(f"Unsupported memory type scope: {scope}") - if status: - conditions.append("properties->>'status' = %s") - params.append(status) + params: list[Any] = [] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + filter=filter, + default_activated=False, + ) - where_clause = " AND ".join(conditions) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) conn = self._get_conn() try: @@ -945,10 +2129,12 @@ def get_all_memory_items( cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" + cols += ", user_name" cur.execute( f""" - SELECT {cols} FROM {self.schema}.memories + SELECT {cols} FROM {relation} WHERE {where_clause} + LIMIT 100 """, params, ) @@ -957,29 +2143,44 @@ def get_all_memory_items( self._put_conn(conn) def get_structure_optimization_candidates( - self, scope: str, include_embedding: bool = False + self, + scope: str, + include_embedding: bool = False, + user_name: str | None = None, ) -> list[dict]: """Find isolated nodes (no edges).""" - user_name = self.user_name + if not user_name: + raise ValueError("get_structure_optimization_candidates requires user_name") + memories = self._tenant_relation("memories", [user_name]) + edges = self._tenant_relation("edges", [user_name]) conn = self._get_conn() try: with conn.cursor() as cur: cols = "m.id, m.memory, m.properties, m.created_at, m.updated_at" + if include_embedding: + cols += ", m.embedding" + cols += ", m.user_name" cur.execute( f""" SELECT {cols} - FROM {self.schema}.memories m - LEFT JOIN {self.schema}.edges e1 ON m.id = e1.source_id - LEFT JOIN {self.schema}.edges e2 ON m.id = e2.target_id + FROM {memories} m + LEFT JOIN {edges} e1 + ON m.id = e1.source_id + AND e1.user_name = m.user_name + AND e1.edge_type = 'PARENT' + LEFT JOIN {edges} e2 + ON m.id = e2.target_id + AND e2.user_name = m.user_name + AND e2.edge_type = 'PARENT' WHERE m.properties->>'memory_type' = %s AND m.user_name = %s AND m.properties->>'status' = 'activated' AND e1.id IS NULL AND e2.id IS NULL - """, + """, (scope, user_name), ) - return [self._parse_row(row, False) for row in cur.fetchall()] + return [self._parse_row(row, include_embedding) for row in cur.fetchall()] finally: self._put_conn(conn) @@ -987,6 +2188,190 @@ def get_structure_optimization_candidates( # Maintenance # ========================================================================= + def get_memory_count(self, memory_type: str, user_name: str | None = None) -> int: + """Count active memories of one type for a tenant.""" + user_name = user_name or self.user_name + relation = self._tenant_relation("memories", [user_name] if user_name else []) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COUNT(*) + FROM {relation} + WHERE user_name = %s + AND properties->>'memory_type' = %s + AND COALESCE(properties->>'status', 'activated') = 'activated' + """, + (user_name, memory_type), + ) + row = cur.fetchone() + return int(row[0]) if row else 0 + finally: + self._put_conn(conn) + + def node_not_exist(self, scope: str, user_name: str | None = None) -> int: + """Return 1 when no active node exists for the requested scope.""" + return int(self.get_memory_count(scope, user_name=user_name) == 0) + + def count_nodes(self, scope: str, user_name: str | None = None) -> int: + """Compatibility alias for memory counts.""" + return self.get_memory_count(scope, user_name=user_name) + + def get_user_names_by_memory_ids(self, memory_ids: list[str]) -> dict[str, str | None]: + normalized_ids = list( + dict.fromkeys( + str(memory_id).strip() for memory_id in memory_ids if str(memory_id).strip() + ) + ) + if not normalized_ids: + return {} + result: dict[str, str | None] = dict.fromkeys(normalized_ids) + memories = self._all_partitions_relation("memories") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, user_name + FROM {memories} + WHERE id = ANY(%s) + """, + (normalized_ids,), + ) + result.update({str(row[0]): row[1] for row in cur.fetchall()}) + return result + finally: + self._put_conn(conn) + + def exist_user_name(self, user_name: str) -> dict[str, bool]: + memories = self._physical_table("memories", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f"SELECT EXISTS(SELECT 1 FROM {memories} WHERE user_name = %s)", + (user_name,), + ) + row = cur.fetchone() + return {user_name: bool(row and row[0])} + finally: + self._put_conn(conn) + + def delete_node_by_mem_cube_id( + self, + mem_cube_id: str, + delete_record_id: str, + hard_delete: bool = False, + ) -> int: + """Soft-delete a tenant's memories, or purge a previously deleted batch.""" + if not mem_cube_id: + raise ValueError("delete_node_by_mem_cube_id mem_cube_id is required") + if not delete_record_id: + raise ValueError("delete_node_by_mem_cube_id delete_record_id is required") + + memories = self._physical_table("memories", mem_cube_id) + edges = self._physical_table("edges", mem_cube_id) + conn = self._get_conn() + try: + with conn.cursor() as cur: + if hard_delete: + cur.execute( + f""" + WITH deleted AS ( + DELETE FROM {memories} + WHERE user_name = %s + AND properties->>'delete_record_id' = %s + RETURNING id + ), deleted_edges AS ( + DELETE FROM {edges} e + USING deleted d + WHERE e.user_name = %s + AND (e.source_id = d.id OR e.target_id = d.id) + RETURNING e.id + ) + SELECT COUNT(*) FROM deleted + """, + (mem_cube_id, delete_record_id, mem_cube_id), + ) + row = cur.fetchone() + return int(row[0]) if row else 0 + else: + deleted_at = datetime.now(timezone.utc).isoformat() + cur.execute( + f""" + UPDATE {memories} + SET properties = properties || %s::jsonb, + deleted_at = %s::timestamptz, + updated_at = NOW() + WHERE user_name = %s + AND COALESCE(properties->>'status', 'activated') <> 'deleted' + """, + ( + json.dumps( + { + "status": "deleted", + "delete_time": deleted_at, + "delete_record_id": delete_record_id, + } + ), + deleted_at, + mem_cube_id, + ), + ) + return max(int(cur.rowcount or 0), 0) + finally: + self._put_conn(conn) + + def recover_memory_by_mem_cube_id( + self, + mem_cube_id: str, + delete_record_id: str, + ) -> int: + """Recover one tenant soft-delete batch.""" + if not mem_cube_id: + raise ValueError("recover_memory_by_mem_cube_id mem_cube_id is required") + if not delete_record_id: + raise ValueError("recover_memory_by_mem_cube_id delete_record_id is required") + + memories = self._physical_table("memories", mem_cube_id) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + UPDATE {memories} + SET properties = properties || %s::jsonb, + deleted_at = NULL, + updated_at = NOW() + WHERE user_name = %s + AND properties->>'delete_record_id' = %s + AND properties->>'status' = 'deleted' + """, + ( + json.dumps( + { + "status": "activated", + "delete_time": "", + "delete_record_id": "", + } + ), + mem_cube_id, + delete_record_id, + ), + ) + return max(int(cur.rowcount or 0), 0) + finally: + self._put_conn(conn) + + def drop_database(self) -> None: + """Compatibility operation: clear only the configured tenant. + + The standard PostgreSQL backend is shared, so dropping the physical + database or schema would delete other tenants. + """ + self.clear(user_name=self.user_name) + def deduplicate_nodes(self) -> None: """Not implemented - handled at application level.""" @@ -1012,6 +2397,8 @@ def get_grouped_counts( user_name = user_name or self.user_name if not group_fields: raise ValueError("group_fields cannot be empty") + if any(not self._is_safe_field_name(field) for field in group_fields): + raise ValueError("group_fields contains an invalid field name") # Build SELECT and GROUP BY clauses # Fields come from JSONB properties column @@ -1033,10 +2420,11 @@ def get_grouped_counts( query_params.extend(params.values()) where_sql = " AND ".join(conditions) + memories = self._physical_table("memories", user_name) query = f""" SELECT {select_fields}, COUNT(*) AS count - FROM {self.schema}.memories + FROM {memories} WHERE {where_sql} GROUP BY {group_by} """ @@ -1067,13 +2455,15 @@ def merge_nodes(self, id1: str, id2: str) -> str: def clear(self, user_name: str | None = None) -> None: """Clear all data for user.""" user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: # Get all node IDs for user cur.execute( f""" - SELECT id FROM {self.schema}.memories WHERE user_name = %s + SELECT id FROM {memories} WHERE user_name = %s """, (user_name,), ) @@ -1083,16 +2473,17 @@ def clear(self, user_name: str | None = None) -> None: # Delete edges cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) + DELETE FROM {edges} + WHERE user_name = %s + AND (source_id = ANY(%s) OR target_id = ANY(%s)) """, - (ids, ids), + (user_name, ids, ids), ) # Delete nodes cur.execute( f""" - DELETE FROM {self.schema}.memories WHERE user_name = %s + DELETE FROM {memories} WHERE user_name = %s """, (user_name,), ) @@ -1100,36 +2491,148 @@ def clear(self, user_name: str | None = None) -> None: finally: self._put_conn(conn) - def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, Any]: - """Export all data.""" - user_name = kwargs.get("user_name") or self.user_name + def export_graph( + self, + user_name: str | None = None, + include_embedding: bool = False, + user_id: str | None = None, + page: int | None = None, + page_size: int | None = None, + filter: dict | None = None, + memory_type: list[str] | None = None, + status: list[str] | None = None, + **kwargs, + ) -> dict[str, Any]: + """Export memories with the same filters and pagination as PolarDB.""" + + extracted_object_type: str | None = None + extracted_mem_cube_id: str | None = None + + def extract_special_values(value: Any) -> Any: + nonlocal extracted_object_type, extracted_mem_cube_id + if not isinstance(value, dict): + return value + cleaned: dict[str, Any] = {} + for key, item in value.items(): + if key in {"and", "or"} and isinstance(item, list): + children = [extract_special_values(child) for child in item] + children = [child for child in children if child not in (None, {}, [])] + if children: + cleaned[key] = children + elif key == "object_type" and isinstance(item, str): + extracted_object_type = extracted_object_type or item + elif key == "mem_cube_id" and isinstance(item, str): + extracted_mem_cube_id = extracted_mem_cube_id or item + else: + cleaned[key] = item + return cleaned or None + + filter_for_sql = extract_special_values(filter) + conditions: list[str] = [] + params: list[Any] = [] + has_object_filter = bool(extracted_object_type and extracted_mem_cube_id) + object_type = (extracted_object_type or "").strip().lower() + + if has_object_filter: + if object_type == "user": + conditions.append("user_name <> %s") + params.append(extracted_mem_cube_id) + elif object_type == "public": + conditions.append("user_name = %s") + params.append(extracted_mem_cube_id) + elif user_name: + conditions.append("user_name = %s") + params.append(user_name) + + effective_user_id = user_id or getattr(self.config, "user_id", None) + if effective_user_id: + conditions.append("properties->>'user_id' = %s") + params.append(effective_user_id) + if memory_type: + conditions.append("properties->>'memory_type' = ANY(%s)") + params.append([str(item) for item in memory_type]) + if status: + conditions.append("properties->>'status' = ANY(%s)") + params.append([str(item) for item in status]) + else: + conditions.append("COALESCE(properties->>'status', 'activated') <> 'deleted'") + conditions.append("deleted_at IS NULL") + if filter_for_sql: + filter_sql = self._build_filter_where_clause(filter_for_sql, params) + if filter_sql: + conditions.append(f"({filter_sql})") + + where_clause = " AND ".join(conditions) if conditions else "TRUE" + target_user = user_name + if has_object_filter and object_type == "public": + target_user = extracted_mem_cube_id + elif has_object_filter and object_type == "user": + target_user = None + memories = self._physical_table("memories", target_user) + use_pagination = page is not None and page_size is not None + if use_pagination: + page = max(int(page), 1) + page_size = max(int(page_size), 1) + offset = (page - 1) * page_size + conn = self._get_conn() try: with conn.cursor() as cur: - # Get nodes + cur.execute( + f"SELECT COUNT(*) FROM {memories} WHERE {where_clause}", + params, + ) + count_row = cur.fetchone() + total_nodes = int(count_row[0]) if count_row else 0 + cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" + cols += ", user_name" + pagination_sql = "" + data_params = list(params) + if use_pagination: + pagination_sql = "LIMIT %s OFFSET %s" + data_params.extend([page_size, offset]) cur.execute( f""" - SELECT {cols} FROM {self.schema}.memories - WHERE user_name = %s - ORDER BY created_at DESC - """, - (user_name,), + SELECT {cols} + FROM {memories} + WHERE {where_clause} + ORDER BY created_at DESC NULLS LAST, id DESC + {pagination_sql} + """, + data_params, ) nodes = [self._parse_row(row, include_embedding) for row in cur.fetchall()] - # Get edges node_ids = [n["id"] for n in nodes] - if node_ids: + node_user_names = [ + n.get("metadata", {}).get("user_name") or user_name for n in nodes + ] + if node_ids and all(node_user_names): + edges_table = self._tenant_relation( + "edges", + list(dict.fromkeys(str(name) for name in node_user_names if name)), + alias="e", + conn=conn, + ) cur.execute( f""" - SELECT source_id, target_id, edge_type - FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) - """, - (node_ids, node_ids), + WITH selected_nodes(user_name, node_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ) + SELECT e.source_id, e.target_id, e.edge_type + FROM {edges_table} + JOIN selected_nodes AS source_node + ON source_node.user_name = e.user_name + AND source_node.node_id = e.source_id + JOIN selected_nodes AS target_node + ON target_node.user_name = e.user_name + AND target_node.node_id = e.target_id + ORDER BY e.id + """, + (node_user_names, node_ids), ) edges = [ {"source": row[0], "target": row[1], "type": row[2]} @@ -1141,7 +2644,7 @@ def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, A return { "nodes": nodes, "edges": edges, - "total_nodes": len(nodes), + "total_nodes": total_nodes, "total_edges": len(edges), } finally: @@ -1164,6 +2667,7 @@ def import_graph(self, data: dict[str, Any], user_name: str | None = None) -> No source_id=edge["source"], target_id=edge["target"], type=edge["type"], + user_name=user_name, ) def close(self): diff --git a/src/memos/llms/openai_new.py b/src/memos/llms/openai_new.py index 766a17fda..50c87b78f 100644 --- a/src/memos/llms/openai_new.py +++ b/src/memos/llms/openai_new.py @@ -50,7 +50,7 @@ def generate(self, messages: MessageList, **kwargs) -> str: output_reasoning = [ item for item in response.output if isinstance(item, ResponseReasoningItem) ] - summary = output_reasoning[0].summary + summary = output_reasoning[0].summary if output_reasoning else None if self.config.remove_think_prefix: return remove_thinking_tags(output_text) @@ -138,7 +138,7 @@ def generate(self, messages: MessageList, **kwargs) -> str: output_reasoning = [ item for item in response.output if isinstance(item, ResponseReasoningItem) ] - summary = output_reasoning[0].summary + summary = output_reasoning[0].summary if output_reasoning else None if self.config.remove_think_prefix: return remove_thinking_tags(output_text) diff --git a/src/memos/mem_feedback/feedback.py b/src/memos/mem_feedback/feedback.py index 28f495afb..18ebb0b49 100644 --- a/src/memos/mem_feedback/feedback.py +++ b/src/memos/mem_feedback/feedback.py @@ -657,11 +657,6 @@ def _info_comparison(self, memory: TextualMemoryItem, _info: dict, include_keys: def _retrieve(self, query: str, info=None, top_k=20, user_name=None): """Retrieve memory items""" - def check_has_edges(mem_item: TextualMemoryItem) -> tuple[TextualMemoryItem, bool]: - """Check if a memory item has edges.""" - edges = self.searcher.graph_store.get_edges(mem_item.id, user_name=user_name) - return (mem_item, len(edges) == 0) - logger.info(f"[feedback _retrieve] query: {query}, user_name: {user_name}") text_mems = self.searcher.search( query=query, @@ -686,22 +681,16 @@ def check_has_edges(mem_item: TextualMemoryItem) -> tuple[TextualMemoryItem, boo pref_mems = [item[0] for item in pref_mems if float(item[1]) > 0.01] text_mems.extend(pref_mems) - # Memory with edges is not modified by feedback - retrieved_mems = [] - with ContextThreadPoolExecutor(max_workers=10) as executor: - futures = {executor.submit(check_has_edges, item): item for item in text_mems} - for future in concurrent.futures.as_completed(futures): - try: - mem_item, has_no_edges = future.result() - if has_no_edges: - retrieved_mems.append(mem_item) - except Exception as e: - logger.error(f"[0107 Feedback Core: _retrieve] Error checking edges: {e}") + retrieved_mems = [ + item + for item in text_mems + if not (getattr(item.metadata, "knowledge_material_ids", None) or []) + ] if len(retrieved_mems) < len(text_mems): logger.info( f"[0107 Feedback Core: _retrieve] {len(text_mems) - len(retrieved_mems)} " - f"text memories are not modified by feedback due to edges." + f"knowledge memories are not modified by feedback." ) return retrieved_mems diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index c46331f1d..eb226c27d 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -769,30 +769,52 @@ def _process_one_item( except Exception as e: logger.error(f"[MultiModalFine] parse error: {e}") - # save rawfile node - if self.save_rawfile and prompt_type == "doc" and len(fine_items) > 0: - rawfile_chunk = mem_str - file_info = fine_items[0].metadata.sources[0].file_info - source = self.multi_modal_parser.file_content_parser.create_source( - message={"file": file_info}, - info=info_per_item, - chunk_index=chunk_idx, - chunk_total=total_chunks, - chunk_content="", - ) - rawfile_node = self._make_memory_item( - value=rawfile_chunk, - info=info_per_item, - memory_type="RawFileMemory", - tags=[ - "mode:fine", - "multimodal:file", - f"chunk:{chunk_idx + 1}/{total_chunks}", - ], - sources=[source], - ) - rawfile_node.metadata.summary_ids = [mem_node.id for mem_node in fine_items] - fine_items.append(rawfile_node) + # Save the original file chunk independently from prompt selection. + file_source = next( + ( + source + for source in sources + if ( + source.get("type") + if isinstance(source, dict) + else getattr(source, "type", None) + ) + == "file" + ), + None, + ) + if self.save_rawfile and file_source is not None and fine_items: + if isinstance(file_source, dict): + file_info = file_source.get("file_info") or {} + file_content = file_source.get("content") + else: + file_info = file_source.file_info or {} + file_content = file_source.content + + rawfile_chunk = file_content or file_info.get("file_data", "") + if rawfile_chunk: + source = self.multi_modal_parser.file_content_parser.create_source( + message={"file": file_info}, + info=info_per_item, + chunk_index=chunk_idx, + chunk_total=total_chunks, + chunk_content=rawfile_chunk, + ) + rawfile_node = self._make_memory_item( + value=rawfile_chunk, + info=info_per_item, + memory_type="RawFileMemory", + tags=[ + "mode:fine", + "multimodal:file", + f"chunk:{chunk_idx + 1}/{total_chunks}", + ], + sources=[source], + ) + rawfile_node.metadata.summary_ids = [mem_node.id for mem_node in fine_items] + fine_items.append(rawfile_node) + else: + logger.warning("[RawFile] File source has no content; skipping raw-file node") enriched_items = trigger_hook( H.MEMORY_ITEMS_AFTER_FINE_EXTRACT, items=fine_items, diff --git a/src/memos/mem_reader/read_multi_modal/image_parser.py b/src/memos/mem_reader/read_multi_modal/image_parser.py index ec19ba304..40561c430 100644 --- a/src/memos/mem_reader/read_multi_modal/image_parser.py +++ b/src/memos/mem_reader/read_multi_modal/image_parser.py @@ -27,6 +27,18 @@ logger = get_logger(__name__) +def is_sendable_image_url(url: str) -> bool: + """Return whether a vision API can accept this image address. + + Accepted forms are http(s) URLs and ``data:image/...;base64,...`` payloads. + Relative paths such as ``images/.jpg`` are rejected. + """ + candidate = url.strip().lower() + if candidate.startswith(("http://", "https://")): + return True + return candidate.startswith("data:image/") and "base64," in candidate + + class ImageParser(BaseMessageParser): """Parser for image_url content parts.""" @@ -177,6 +189,13 @@ def parse_fine( logger.warning("[ImageParser] No image URL found in message") return [] + if not is_sendable_image_url(url): + logger.warning( + "[ImageParser] Skip image URL that cannot be sent to the vision model: %s", + url[:200], + ) + return [] + # Create source for this image source = self.create_source(message, info) diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index a954794b7..5e663c99e 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -209,7 +209,7 @@ def _process_memories_with_reader( for memory in flattened_memories if memory.metadata.memory_type == "RawFileMemory" ] - text_mem.add_rawfile_nodes_n_edges( + text_mem.add_rawfile_nodes( raw_file_mem_group, enhanced_mem_ids, user_id=user_id, @@ -328,9 +328,7 @@ def _process_memories_with_reader( add_meta_legacy: list[dict] = [] update_content_legacy: list[dict] = [] update_meta_legacy: list[dict] = [] - for item_id, item in zip( - enhanced_mem_ids, flattened_memories, strict=False - ): + for item_id, item in zip(enhanced_mem_ids, mem_group, strict=False): key = getattr(item.metadata, "key", None) or transform_name_to_key( name=item.memory ) diff --git a/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py b/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py index 3123bfdee..e39f4bbd9 100644 --- a/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py +++ b/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py @@ -61,7 +61,7 @@ def __init__(self): self.rabbitmq_conn_sleep_seconds = 1 # Thread management - self._rabbitmq_io_loop_thread = None # For IOLoop execution + self._io_loop_thread = None # For IOLoop execution self._rabbitmq_stop_flag = False # Graceful shutdown flag # Use RLock because publishing may trigger initialization, which also grabs the lock. self._rabbitmq_lock = threading.RLock() diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index 23474cc20..a874ac77b 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -159,6 +159,16 @@ class TextualMemoryMetadata(BaseModel): description="Internal algorithm metadata reserved for system use.", ) + @field_validator("internal_info", mode="before") + @classmethod + def _decode_internal_info(cls, value: Any) -> Any: + # Neo4j stores dictionaries as JSON strings; decode them before model validation. + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, dict): + raise ValueError("internal_info JSON must decode to an object") + return value + model_config = ConfigDict(extra="allow") covered_history: Any | None = Field( @@ -212,6 +222,14 @@ class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata): default_factory=list, description="The ids of the files associated with the memory.", ) + knowledge_material_ids: list[str] = Field( + default_factory=list, + description="RawFileMemory ids that support this knowledge summary.", + ) + knowledge_summary_ids: list[str] = Field( + default_factory=list, + description="Knowledge summary ids extracted from this raw file memory.", + ) @field_validator("sources", mode="before") @classmethod diff --git a/src/memos/memories/textual/tree.py b/src/memos/memories/textual/tree.py index 426a2359b..c9d74d43f 100644 --- a/src/memos/memories/textual/tree.py +++ b/src/memos/memories/textual/tree.py @@ -399,10 +399,13 @@ def get_all( ) return graph_output - def delete(self, memory_ids: list[str], user_name: str | None = None) -> None: + def delete(self, memory_ids: str | list[str], user_name: str | None = None) -> None: """Hard delete: permanently remove nodes and their edges from the graph.""" if not memory_ids: return + if isinstance(memory_ids, str): + memory_ids = [memory_ids] + self._unlink_knowledge_relations(memory_ids, user_name=user_name) for mid in memory_ids: try: self.graph_store.delete_node(mid, user_name=user_name) @@ -412,6 +415,7 @@ def delete(self, memory_ids: list[str], user_name: str | None = None) -> None: def delete_by_memory_ids(self, memory_ids: list[str]) -> None: """Delete memories by memory_ids.""" try: + self._unlink_knowledge_relations(memory_ids) self.graph_store.delete_node_by_prams(memory_ids=memory_ids) except Exception as e: logger.error(f"An error occurred while deleting memories by memory_ids: {e}") @@ -432,10 +436,71 @@ def delete_by_filter( filter: dict | None = None, ) -> None: """Delete memories by filter.""" + if file_ids: + user_names = writable_cube_ids or [None] + for user_name in user_names: + memory_ids: set[str] = set() + for file_id in file_ids: + memory_ids.update( + self.graph_store.get_by_metadata( + [{"field": "file_ids", "op": "contains", "value": file_id}], + user_name=user_name, + ) + ) + self._unlink_knowledge_relations(list(memory_ids), user_name=user_name) self.graph_store.delete_node_by_prams( writable_cube_ids=writable_cube_ids, file_ids=file_ids, filter=filter ) + @staticmethod + def _node_metadata(node: dict[str, Any] | TextualMemoryItem) -> dict[str, Any]: + if isinstance(node, TextualMemoryItem): + return node.metadata.model_dump() + return node.get("metadata") or {} + + def _get_node_map( + self, memory_ids: list[str] | set[str], user_name: str | None = None + ) -> dict[str, dict[str, Any] | TextualMemoryItem]: + if not memory_ids: + return {} + nodes = self.graph_store.get_nodes(ids=list(memory_ids), user_name=user_name) or [] + return { + node.id if isinstance(node, TextualMemoryItem) else node["id"]: node for node in nodes + } + + def _unlink_knowledge_relations( + self, memory_ids: list[str], user_name: str | None = None + ) -> None: + """Remove reverse knowledge ids before deleting or deactivating nodes.""" + deleting_ids = set(memory_ids) + deleting_nodes = self._get_node_map(deleting_ids, user_name=user_name) + removals: dict[str, dict[str, set[str]]] = {} + + for memory_id, node in deleting_nodes.items(): + metadata = self._node_metadata(node) + for material_id in metadata.get("knowledge_material_ids") or []: + if material_id not in deleting_ids: + removals.setdefault(material_id, {}).setdefault( + "knowledge_summary_ids", set() + ).add(memory_id) + for summary_id in metadata.get("knowledge_summary_ids") or []: + if summary_id not in deleting_ids: + removals.setdefault(summary_id, {}).setdefault( + "knowledge_material_ids", set() + ).add(memory_id) + + related_nodes = self._get_node_map(set(removals), user_name=user_name) + for related_id, fields in removals.items(): + node = related_nodes.get(related_id) + if node is None: + continue + metadata = self._node_metadata(node) + updates = { + field: [value for value in metadata.get(field) or [] if value not in removed_ids] + for field, removed_ids in fields.items() + } + self.graph_store.update_node(related_id, updates, user_name=user_name) + def load(self, dir: str, user_name: str | None = None) -> None: try: memory_file = os.path.join(dir, self.config.memory_filename) @@ -593,6 +658,109 @@ def add_rawfile_nodes_n_edges( f"[RawFile] Time taken to add edges: {end_time - start_time} seconds for {len(from_ids)} edges" ) + def add_rawfile_nodes( + self, + raw_file_mem_group: list[TextualMemoryItem], + mem_ids: list[str], + user_id: str | None = None, + user_name: str | None = None, + ) -> None: + """Persist raw-file nodes and their knowledge relationships without graph edges.""" + valid_summary_ids = set(mem_ids) + summary_to_material_ids: dict[str, list[str]] = {} + + for raw_file_mem in raw_file_mem_group: + summary_ids = list( + dict.fromkeys( + summary_id + for summary_id in getattr(raw_file_mem.metadata, "summary_ids", []) or [] + if summary_id in valid_summary_ids + ) + ) + raw_file_mem.metadata.knowledge_summary_ids = summary_ids + for summary_id in summary_ids: + summary_to_material_ids.setdefault(summary_id, []).append(raw_file_mem.id) + + self.add(raw_file_mem_group, user_name=user_name) + summary_nodes = self._get_node_map(set(summary_to_material_ids), user_name=user_name) + + merged_from_by_summary: dict[str, list[str]] = {} + old_summary_ids: set[str] = set() + for summary_id, node in summary_nodes.items(): + info = self._node_metadata(node).get("info") or {} + merged_from = info.get("merged_from") or [] + if isinstance(merged_from, str): + merged_from = [merged_from] + merged_from_by_summary[summary_id] = list(dict.fromkeys(merged_from)) + old_summary_ids.update(merged_from) + + old_summary_nodes = self._get_node_map(old_summary_ids, user_name=user_name) + material_relinks: dict[str, dict[str, Any]] = {} + + for summary_id, direct_material_ids in summary_to_material_ids.items(): + summary_node = summary_nodes.get(summary_id) + if summary_node is None: + logger.warning( + "Knowledge summary %s was not found after raw-file write", summary_id + ) + continue + + metadata = self._node_metadata(summary_node) + merged_from = merged_from_by_summary.get(summary_id, []) + inherited_material_ids: list[str] = [] + for old_summary_id in merged_from: + old_summary = old_summary_nodes.get(old_summary_id) + if old_summary is not None: + inherited_material_ids.extend( + self._node_metadata(old_summary).get("knowledge_material_ids") or [] + ) + + material_ids = list( + dict.fromkeys( + [ + *(metadata.get("knowledge_material_ids") or []), + *direct_material_ids, + *inherited_material_ids, + ] + ) + ) + self.graph_store.update_node( + summary_id, + {"knowledge_material_ids": material_ids}, + user_name=user_name, + ) + + for material_id in inherited_material_ids: + relink = material_relinks.setdefault(material_id, {"remove": set(), "add": []}) + relink["remove"].update(merged_from) + relink["add"].append(summary_id) + + material_nodes = self._get_node_map(set(material_relinks), user_name=user_name) + for material_id, relink in material_relinks.items(): + material_node = material_nodes.get(material_id) + if material_node is None: + continue + current_summary_ids = ( + self._node_metadata(material_node).get("knowledge_summary_ids") or [] + ) + updated_summary_ids = [ + summary_id + for summary_id in current_summary_ids + if summary_id not in relink["remove"] + ] + updated_summary_ids.extend(relink["add"]) + self.graph_store.update_node( + material_id, + {"knowledge_summary_ids": list(dict.fromkeys(updated_summary_ids))}, + user_name=user_name, + ) + + logger.info( + "[RawFile] Added %s chunks with ID mappings for user %s", + len(raw_file_mem_group), + user_id, + ) + def add_graph_edges( self, from_ids: list[str], to_ids: list[str], types: list[str], user_name: str | None = None ) -> None: @@ -624,6 +792,7 @@ def soft_delete( user_name: str, evolve_to_ids: list[str] | None = None, ) -> None: + self._unlink_knowledge_relations(memory_ids, user_name=user_name) # for ruff check... if not evolve_to_ids: update_fields = {"status": "deleted"} diff --git a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py index 5c9cce78e..b44d28218 100644 --- a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py +++ b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py @@ -3,8 +3,6 @@ import re import traceback -from concurrent.futures import as_completed - from memos.context.context import ContextThreadPoolExecutor from memos.embedders.factory import OllamaEmbedder from memos.graph_dbs.factory import Neo4jGraphDB @@ -1282,7 +1280,7 @@ def _sort_and_trim( @timed def _deduplicate_rawfile_results(self, results, user_name: str | None = None): """ - Deduplicate rawfile related memories by edge + Deduplicate raw-file results using their persisted summary ids. """ if not results: return results @@ -1292,29 +1290,10 @@ def _deduplicate_rawfile_results(self, results, user_name: str | None = None): if not rawfile_items: return results - with ContextThreadPoolExecutor(max_workers=min(len(rawfile_items), 10)) as executor: - futures = [ - executor.submit( - self.graph_store.get_edges, - rawfile_item.id, - type="SUMMARY", - direction="OUTGOING", - user_name=user_name, - ) - for rawfile_item in rawfile_items - ] - for future in as_completed(futures): - try: - edges = future.result() - for edge in edges: - summary_target_id = edge.get("to") - if summary_target_id: - summary_ids_to_remove.add(summary_target_id) - logger.debug( - f"[DEDUP] Marking summary node {summary_target_id} for removal (pointed by RawFileMemory)" - ) - except Exception as e: - logger.warning(f"[DEDUP] Failed to get summary target ids: {e}") + for rawfile_item in rawfile_items: + summary_ids_to_remove.update( + getattr(rawfile_item.metadata, "knowledge_summary_ids", None) or [] + ) filtered_results = [] for item in results: diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py index 27399259a..0b9829b7f 100644 --- a/src/memos/multi_mem_cube/single_cube.py +++ b/src/memos/multi_mem_cube/single_cube.py @@ -441,22 +441,35 @@ def extract_edge_info(edges_info: list[dict], neighbor_relativity: float): ) return edge_mems + def get_neighbor(chunk_target_id: str | None, neighbor_relativity: float): + if not chunk_target_id: + return None + item_neighbor = self.searcher.graph_store.get_node(chunk_target_id, user_name=user_name) + if not item_neighbor: + return None + item_neighbor_mem = TextualMemoryItem(**item_neighbor) + item_neighbor_mem.metadata.relativity = neighbor_relativity + self.logger.info("Add neighbor chunk by id: %s", chunk_target_id) + return item_neighbor_mem + final_items = [] if neighbor_discovery: for item in search_results: if item.metadata.memory_type == "RawFileMemory": neighbor_relativity = item.metadata.relativity * 0.8 - preceding_info = self.searcher.graph_store.get_edges( - item.id, type="PRECEDING", direction="OUTGOING", user_name=user_name + preceding = get_neighbor( + getattr(item.metadata, "preceding_id", None), neighbor_relativity ) - final_items.extend(extract_edge_info(preceding_info, neighbor_relativity)) + if preceding is not None: + final_items.append(preceding) final_items.append(item) - following_info = self.searcher.graph_store.get_edges( - item.id, type="FOLLOWING", direction="OUTGOING", user_name=user_name + following = get_neighbor( + getattr(item.metadata, "following_id", None), neighbor_relativity ) - final_items.extend(extract_edge_info(following_info, neighbor_relativity)) + if following is not None: + final_items.append(following) else: final_items.append(item) @@ -748,7 +761,7 @@ def _process_text_mem( for memory in flattened_local if memory.metadata.memory_type == "RawFileMemory" ] - self.naive_mem_cube.text_mem.add_rawfile_nodes_n_edges( + self.naive_mem_cube.text_mem.add_rawfile_nodes( raw_file_mem_group, mem_ids_local, user_id=add_req.user_id, diff --git a/tests/api/test_playground_cube_recall.py b/tests/api/test_playground_cube_recall.py new file mode 100644 index 000000000..4b1d8fd40 --- /dev/null +++ b/tests/api/test_playground_cube_recall.py @@ -0,0 +1,24 @@ +"""Playground recall must keep memories from every searched cube.""" + +from memos.api.handlers.chat_handler import ChatHandler + + +def test_playground_recall_keeps_memories_from_later_cubes(): + search_data = { + "text_mem": [ + {"cube_id": "knowledge-base", "memories": []}, + { + "cube_id": "personal", + "memories": [{"id": "mem-1", "memory": "use event_id as the idempotency key"}], + }, + ] + } + + memories = ChatHandler._memories_from_cube_results(search_data) + + assert [item["id"] for item in memories] == ["mem-1"] + + +def test_playground_recall_returns_empty_when_search_data_missing(): + assert ChatHandler._memories_from_cube_results(None) == [] + assert ChatHandler._memories_from_cube_results({}) == [] diff --git a/tests/api/test_rate_limit_bearer.py b/tests/api/test_rate_limit_bearer.py new file mode 100644 index 000000000..710346cf0 --- /dev/null +++ b/tests/api/test_rate_limit_bearer.py @@ -0,0 +1,78 @@ +"""Regression tests for issue #2428. + +``_get_client_key`` in ``src/memos/api/middleware/rate_limit.py`` checked +whether ``request.headers['Authorization']`` started with ``krlk_`` — but +real clients send ``Authorization: Bearer krlk_`` per RFC 6750, so +the prefix check never matched and every request fell through to IP-based +limiting. Behind a shared egress IP all keyed clients shared one bucket +and the per-key rate-limit was dead code. + +The fix strips the ``Bearer `` scheme before testing the key prefix, so +``Bearer krlk_`` correctly maps to ``ratelimit:key:``. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +from memos.api.middleware.rate_limit import _get_client_key + + +def _fake_request(authorization: str | None, *, client_host: str = "10.0.0.1"): + headers: dict[str, str] = {} + if authorization is not None: + headers["Authorization"] = authorization + return SimpleNamespace( + headers=headers, + client=SimpleNamespace(host=client_host), + ) + + +class TestGetClientKey: + def test_bearer_krlk_key_uses_per_key_bucket(self): + body = "krlk_" + "a" * 64 + request = _fake_request("Bearer " + body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] + assert "10.0.0.1" not in key + + def test_bearer_krlk_key_first_twenty_chars_of_actual_key(self): + body = "krlk_" + "f" * 64 + request = _fake_request("Bearer " + body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] + + def test_lowercase_bearer_scheme(self): + # RFC 7235: the scheme name is case-insensitive. We normalise to + # lowercase before the prefix check. + body = "krlk_" + "b" * 64 + request = _fake_request("bearer " + body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] + + def test_bearer_with_non_key_token_falls_back_to_ip(self): + request = _fake_request("Bearer some-jwt-token") + key = _get_client_key(request) + assert key.startswith("ratelimit:ip:") + assert "10.0.0.1" in key + + def test_missing_authorization_header_falls_back_to_ip(self): + request = _fake_request(None) + key = _get_client_key(request) + assert key.startswith("ratelimit:ip:") + assert "10.0.0.1" in key + + def test_x_forwarded_for_header_preferred_over_ip(self): + request = _fake_request(None, client_host="203.0.113.1") + request.headers["X-Forwarded-For"] = "198.51.100.1, 10.0.0.1" + key = _get_client_key(request) + assert key == "ratelimit:ip:198.51.100.1" + + def test_krlk_prefix_without_bearer_scheme_still_matches(self): + # Backwards compatibility: clients that sent a bare krlk_ header + # (before the Bearer scheme was added) must still match. + body = "krlk_" + "c" * 64 + request = _fake_request(body) + key = _get_client_key(request) + assert key == "ratelimit:key:" + body[:20] diff --git a/tests/api/test_validate_key_format.py b/tests/api/test_validate_key_format.py new file mode 100644 index 000000000..757d2519a --- /dev/null +++ b/tests/api/test_validate_key_format.py @@ -0,0 +1,99 @@ +"""Regression tests for issue #2427. + +``validate_key_format`` exists in two copies — ``src/memos/api/utils/api_keys.py`` +(the canonical home) and ``src/memos/api/middleware/auth.py`` (the +middleware gating every authenticated request). Both check the hex part +with ``int(hex_part, 16)``, which accepts non-canonical forms a generated +key can never have: + +- leading ``+`` / ``-`` (sign) +- ``0x`` / ``0X`` prefix +- ``_`` separators (PEP 515) +- surrounding whitespace + +So the documented format ``krlk_<64-hex-chars>`` is not actually enforced: +keys with embedded underscores, signs, or ``0x`` prefixes pass the format +gate and are hashed/looked up as-is, bypassing the canonical form that +``generate_api_key()`` produces. + +These tests pin the new contract on both validator copies: + +- the hex part must match ``[0-9a-f]{64}`` exactly +- the only valid form is the canonical lowercase hex string +""" + +from __future__ import annotations + +import pytest + +from memos.api.middleware.auth import validate_key_format as validate_in_auth +from memos.api.utils.api_keys import generate_api_key, validate_key_format + + +CANONICAL = "krlk_" + "0123456789abcdef" * 4 # 64 hex chars, lowercase + + +class TestValidateKeyFormatUtils: + def test_canonical_key_passes(self): + assert validate_key_format(CANONICAL) is True + + def test_generated_key_always_passes(self): + # The validator must accept every key that generate_api_key() + # produces, since that is the documented format. + for _ in range(20): + assert validate_key_format(generate_api_key().key) is True + + @pytest.mark.parametrize( + "bad_key", + [ + # Leading sign + "krlk_" + "+" + "a" * 63, + "krlk_" + "-" + "a" * 63, + # 0x prefix + "krlk_" + "0x" + "a" * 62, + "krlk_" + "0X" + "a" * 62, + # Underscore separator (PEP 515) + "krlk_" + "a" * 32 + "_" + "b" * 31, + # Uppercase hex (the generator emits lowercase only) + "krlk_" + "A" * 64, + # Surrounding whitespace + " krlk_" + "a" * 64, + "krlk_" + "a" * 64 + " ", + # Wrong total length + "krlk_" + "a" * 63, + "krlk_" + "a" * 65, + # Empty hex part + "krlk_", + # Missing prefix + "0123456789abcdef" * 4, + "Bearer " + CANONICAL, + # Non-string + "", + ], + ) + def test_non_canonical_keys_rejected(self, bad_key): + assert validate_key_format(bad_key) is False, ( + f"validate_key_format({bad_key!r}) should return False" + ) + + +class TestAuthMiddlewareValidator: + """The middleware copy must enforce the same contract as the canonical + one — issue #2427 documents both copies accepting non-canonical hex.""" + + def test_canonical_key_passes(self): + assert validate_in_auth(CANONICAL) is True + + def test_signed_hex_rejected(self): + # The most surprising bit: ``int("+a..", 16)`` succeeds, so a signed + # hex passes the middleware's validator today. + assert validate_in_auth("krlk_" + "+" + "a" * 63) is False + + def test_underscore_rejected(self): + assert validate_in_auth("krlk_" + "a" * 32 + "_" + "b" * 31) is False + + def test_uppercase_rejected(self): + assert validate_in_auth("krlk_" + "A" * 64) is False + + def test_0x_prefix_rejected(self): + assert validate_in_auth("krlk_" + "0x" + "a" * 62) is False diff --git a/tests/graph_dbs/test_postgres_400_compat.py b/tests/graph_dbs/test_postgres_400_compat.py new file mode 100644 index 000000000..bd23fd8ff --- /dev/null +++ b/tests/graph_dbs/test_postgres_400_compat.py @@ -0,0 +1,391 @@ +import json + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def postgres_db(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db.schema = "memos" + db.db_name = "memos" + db.user_name = "tenant-a" + db._pool_closed = False + db.config = SimpleNamespace( + embedding_dimension=3, + fulltext_config="simple", + vector_index_type="hnsw", + hnsw_m=16, + hnsw_ef_construction=64, + ivfflat_lists=100, + shard_count=400, + ) + + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + db._get_conn = MagicMock(return_value=connection) + db._put_conn = MagicMock() + db._partition_remainders = { + "tenant-a": 17, + "tenant-b": 17, + "kb-1": 17, + } + return db, cursor + + +def _executed_sql(cursor: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_schema_does_not_create_profile_storage(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "profile_templates" not in sql + assert "profile_instances" not in sql + + +def test_schema_creates_400_md5_memory_shards(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "PARTITION BY HASH" not in sql + assert '"Memory"' in sql + assert "embedding halfvec(" in sql + assert "memos_graph_0" in sql + assert "memos_graph_399" in sql + assert "MODULUS" not in sql + assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 401 + assert "PRIMARY KEY (user_name, id)" in sql + + +def test_schema_rejects_legacy_nonpartitioned_parent(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("memories", "r")] + + with pytest.raises(RuntimeError, match="configure a new MEMOS_SCHEMA"): + db._init_schema() + + +def test_schema_skips_partition_ddl_when_layout_is_complete(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (400,) + + db._init_schema() + + sql = _executed_sql(cursor) + assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 1 + assert "memos_graph_0" not in sql + + +def test_node_upsert_targets_composite_partition_key(postgres_db): + db, cursor = postgres_db + + db.add_node( + "memory-1", + "memory", + {"embedding": [1.0, 0.0, 0.0]}, + user_name="tenant-a", + ) + + assert "ON CONFLICT (user_name, id)" in _executed_sql(cursor) + + +def test_batch_insert_uses_one_statement_and_polardb_properties(postgres_db): + db, cursor = postgres_db + + db.add_nodes_batch( + [ + {"id": "m1", "memory": "one", "metadata": {"embedding": [1.0, 0.0, 0.0]}}, + {"id": "m2", "memory": "two", "metadata": {"embedding": [0.0, 1.0, 0.0]}}, + ], + user_name="tenant-a", + ) + + assert cursor.execute.call_count == 1 + query, params = cursor.execute.call_args.args + assert query.count("::halfvec(") == 2 + assert "ON CONFLICT (user_name, id)" in query + first_properties = json.loads(params[2]) + second_properties = json.loads(params[9]) + assert first_properties["id"] == "m1" + assert first_properties["memory"] == "one" + assert second_properties["id"] == "m2" + assert second_properties["memory"] == "two" + assert "embedding" not in first_properties + + +def test_node_upsert_persists_tenant_in_metadata(postgres_db): + db, cursor = postgres_db + + db.add_node( + "memory-1", + "memory", + {"embedding": [1.0, 0.0, 0.0]}, + user_name="tenant-a", + ) + + _, params = cursor.execute.call_args.args + properties = json.loads(params[2]) + assert properties["user_name"] == "tenant-a" + assert properties["delete_time"] == "" + assert properties["delete_record_id"] == "" + + +def test_batch_node_metadata_matches_polardb_shape(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + + node = db._parse_row( + ( + "memory-1", + "service HTTP comparison memory", + {"status": "activated"}, + now, + now, + ) + ) + + assert node["metadata"]["id"] == "memory-1" + assert node["metadata"]["memory"] == "service HTTP comparison memory" + + +def test_parse_row_restores_tenant_and_deserializes_sources(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + source = {"type": "chat", "role": "user", "content": "hello"} + + node = db._parse_row( + ( + "memory-1", + "memory", + {"sources": [json.dumps(source)]}, + now, + now, + "tenant-a", + ) + ) + + assert node["metadata"]["user_name"] == "tenant-a" + assert node["metadata"]["sources"] == [source] + + +def test_new_polardb_business_methods_are_available(): + required_methods = { + "delete_node_by_cube_memory_groups", + "get_user_names_by_cube_memory_groups", + "get_children_with_embeddings", + "get_neighbors_by_tag", + } + + missing = sorted(name for name in required_methods if not hasattr(PostgresGraphDB, name)) + assert missing == [] + + +def test_get_subgraph_matches_polardb_return_shape(postgres_db): + db, cursor = postgres_db + core = { + "id": "core", + "memory": "core memory", + "metadata": {"status": "activated"}, + } + neighbor = { + "id": "neighbor", + "memory": "neighbor memory", + "metadata": {"status": "activated"}, + } + db.get_node = MagicMock(return_value=core) + db.get_nodes = MagicMock(return_value=[neighbor]) + cursor.fetchall.side_effect = [ + [("core",), ("neighbor",)], + [("core", "neighbor", "RELATE_TO")], + ] + + result = db.get_subgraph( + "core", + depth=2, + center_status="activated", + user_name="tenant-a", + ) + + assert result == { + "core_node": core, + "neighbors": [neighbor], + "edges": [{"source": "core", "target": "neighbor", "type": "RELATE_TO"}], + } + assert "user_name = %s" in _executed_sql(cursor) + db.get_node.assert_called_once_with("core", user_name="tenant-a") + db.get_nodes.assert_called_once_with(["neighbor"], user_name="tenant-a") + + +def test_cube_group_delete_pairs_ids_with_tenants(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (2,) + + deleted = db.delete_node_by_cube_memory_groups( + [ + {"cube_id": "tenant-a", "memory_ids": ["m1"]}, + {"cube_id": "tenant-b", "memory_ids": ["m2"]}, + ] + ) + + query, params = cursor.execute.call_args.args + assert deleted == 2 + assert "unnest" in query.lower() + assert "requested.user_name" in query + assert params == (["tenant-a", "tenant-b"], ["m1", "m2"]) + + +def test_memory_owner_lookup_preserves_missing_ids(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("m1", "tenant-a")] + + owners = db.get_user_names_by_memory_ids(["m1", "missing"]) + + assert owners == {"m1": "tenant-a", "missing": None} + + +def test_export_graph_supports_latest_filters_and_pagination(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchone.return_value = (3,) + cursor.fetchall.side_effect = [ + [ + ( + "m1", + "memory", + {"memory_type": "LongTermMemory", "status": "activated"}, + now, + now, + ) + ], + [("m1", "m2", "RELATE_TO")], + ] + + result = db.export_graph( + user_name="tenant-a", + page=2, + page_size=1, + filter={"agent_id": "agent-1"}, + memory_type=["LongTermMemory"], + status=["activated"], + ) + + sql = _executed_sql(cursor) + assert result["total_nodes"] == 3 + assert result["nodes"][0]["id"] == "m1" + assert "properties->'info'->>'agent_id'" in sql + assert "LIMIT %s OFFSET %s" in sql + assert "properties->>'memory_type' = ANY(%s)" in sql + assert "JOIN selected_nodes AS source_node" in sql + assert "source_node.user_name = e.user_name" in sql + + +def test_keyword_like_matches_polardb_filters_and_parameter_name(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "memory-1", + "needle memory", + {"source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + result = db.search_by_keywords_like( + query_word="needle", + scope="LongTermMemory", + status="activated", + search_filter={"source": "conversation"}, + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + return_fields=["memory", "source"], + ) + + query, params = cursor.execute.call_args.args + assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}] + assert "user_name = ANY(%s)" in query + assert ["tenant-a", "kb-1"] in params + assert "properties->>'source'" in query + assert "properties->>'memory', '') LIKE %s" in query + + +def test_keyword_tfidf_matches_polardb_result_shape(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "memory-1", + "needle memory", + {"source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + result = db.search_by_keywords_tfidf( + ["needle", "memory"], + scope="LongTermMemory", + user_name="tenant-a", + return_fields=["memory", "source"], + ) + + assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}] + query, params = cursor.execute.call_args.args + assert params[0] == "'needle' | 'memory'" + assert "to_tsquery('jiebaqry'" in query + assert "ILIKE" not in query + + +def test_keyword_searches_require_explicit_user_name(postgres_db): + db, cursor = postgres_db + + assert db.search_by_keywords_like("needle") == [] + assert db.search_by_keywords_tfidf(["needle"]) == [] + cursor.execute.assert_not_called() + + +def test_get_all_memory_items_matches_polardb_defaults(postgres_db): + db, cursor = postgres_db + + assert db.get_all_memory_items("LongTermMemory") == [] + cursor.execute.assert_not_called() + + with pytest.raises(ValueError, match="Unsupported memory type scope"): + db.get_all_memory_items("Context", user_name="tenant-a") + + cursor.fetchall.return_value = [] + db.get_all_memory_items("LongTermMemory", user_name="tenant-a") + query, _ = cursor.execute.call_args.args + assert "properties->>'status'" not in query + + +def test_metadata_contains_list_uses_json_array_overlap(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("m1",)] + + result = db.get_by_metadata( + [{"field": "tags", "op": "contains", "value": ["blue", "green"]}], + user_name="tenant-a", + ) + + query, params = cursor.execute.call_args.args + assert result == ["m1"] + assert "properties->'tags' @> %s::jsonb" in query + assert " OR " in query + assert json.dumps(["blue"], ensure_ascii=False) in params + assert json.dumps(["green"], ensure_ascii=False) in params diff --git a/tests/graph_dbs/test_postgres_pgvector.py b/tests/graph_dbs/test_postgres_pgvector.py new file mode 100644 index 000000000..d404cf230 --- /dev/null +++ b/tests/graph_dbs/test_postgres_pgvector.py @@ -0,0 +1,461 @@ +import hashlib +import json + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.configs.graph_db import PostgresGraphDBConfig +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def postgres_db(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db.schema = "memos" + db.db_name = "memos" + db.user_name = "tenant-a" + db._pool_closed = False + db.config = SimpleNamespace( + embedding_dimension=3, + fulltext_config="simple", + vector_index_type="hnsw", + hnsw_m=16, + hnsw_ef_construction=64, + ivfflat_lists=100, + shard_count=400, + ) + db._partition_remainders = { + "tenant-a": 17, + "tenant-b": 17, + "kb-1": 17, + } + + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + db._get_conn = MagicMock(return_value=connection) + db._put_conn = MagicMock() + return db, cursor + + +def _executed_sql(cursor: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_postgres_pool_keeps_eight_warm_connections(): + config = PostgresGraphDBConfig( + host="postgres", + user="postgres", + password="secret", + db_name="memos", + user_name="tenant-a", + ) + assert config.minconn == 8 + assert config.maxconn == 20 + + +def test_get_conn_does_not_ping_before_the_query(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db._pool_closed = False + cursor = MagicMock() + connection = MagicMock() + connection.closed = 0 + connection.cursor.return_value.__enter__.return_value = cursor + db.pool = MagicMock() + db.pool.getconn.return_value = connection + + assert db._get_conn() is connection + cursor.execute.assert_not_called() + + +def test_postgres_config_defaults_to_current_embedding_dimension(): + config = PostgresGraphDBConfig( + host="postgres", + user="memos", + password="secret", + db_name="memos", + user_name="tenant-a", + ) + + assert config.embedding_dimension == 1024 + assert config.shard_count == 400 + + +def test_init_schema_creates_tenant_edges_fulltext_and_hnsw(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "user_name TEXT NOT NULL" in sql + assert "UNIQUE(user_name, source_id, target_id, edge_type)" in sql + assert "search_vector tsvector" in sql + assert "properties::text" in sql + assert "USING GIN(search_vector)" in sql + assert "halfvec_cosine_ops" in sql + assert "idx_memory_embedding_halfvec" in sql + assert "embedding halfvec(" in sql + assert '"Memory"' in sql + assert "USING ivfflat" not in sql + + +def test_edge_operations_are_tenant_scoped_and_support_any_direction(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (1,) + cursor.fetchall.return_value = [("source", "target", "RELATE_TO")] + + db.add_edge("source", "target", "RELATE_TO", user_name="tenant-a") + assert db.edge_exists( + "source", + "target", + "RELATE_TO", + direction="ANY", + user_name="tenant-a", + ) + edges = db.get_edges("source", type="ANY", direction="ANY", user_name="tenant-a") + + sql = _executed_sql(cursor) + assert "user_name" in sql + assert edges == [{"from": "source", "to": "target", "type": "RELATE_TO"}] + assert any( + "tenant-a" in call.args[1] for call in cursor.execute.call_args_list if len(call.args) > 1 + ) + + db.edge_exists("source", "target", type="ANY", user_name="tenant-a") + query, params = cursor.execute.call_args.args + assert "edge_type = %s" not in query + assert params == ["tenant-a", "source", "target"] + + +def test_vector_search_applies_filters_kb_ids_and_return_fields(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.95, + "remember this", + {"status": "activated", "source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + results = db.search_by_embedding( + vector=[0.1, 0.2, 0.3], + top_k=5, + scope="LongTermMemory", + status="activated", + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + filter={"and": [{"info.agent_id": "agent-1"}, {"tags": {"contains": "blue"}}]}, + return_fields=["memory", "source", "created_at"], + ) + + query, params = cursor.execute.call_args.args + assert "user_name = ANY(%s)" in query + assert "COALESCE(properties->>'id', id)" in query + assert "properties->'info'->>'agent_id'" in query + assert "properties->'tags' @> %s::jsonb" in query + assert ["tenant-a", "kb-1"] in params + assert results[0]["memory"] == "remember this" + assert results[0]["source"] == "conversation" + assert results[0]["created_at"] == now.isoformat() + + +def test_lightweight_vector_search_returns_memory_without_extra_lookup(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.95, + "remember this", + now, + now, + "tenant-a", + '"LongTermMemory"', + '"activated"', + None, + None, + None, + None, + ) + ] + + results = db.search_by_embedding( + vector=[0.1, 0.2, 0.3], + top_k=5, + scope="LongTermMemory", + user_name="tenant-a", + light_weight_mode=True, + ) + + assert results[0]["memory"] == "remember this" + assert results[0]["memory_type"] == "LongTermMemory" + assert PostgresGraphDB.supports_lightweight_vector_search is True + assert PostgresGraphDB.prefer_lightweight_vector_search is True + + +def test_search_result_does_not_add_postgres_only_property_aliases(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + row = ( + "node-1", + 0.95, + "remember this", + {"status": "activated"}, + now, + now, + "tenant-a", + ) + + result = db._build_search_result(row, ["metadata", "properties"]) + + assert result == {"id": "node-1", "score": 0.95} + + +def test_search_result_keeps_polardb_serialized_source_shape(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + source = json.dumps({"type": "chat", "role": "user", "content": "hello"}) + row = ( + "node-1", + 0.95, + "remember this", + {"sources": [source]}, + now, + now, + "tenant-a", + ) + + result = db._build_search_result(row, ["sources"]) + + assert result["sources"] == [source] + + +def test_fulltext_search_uses_polardb_and_tsquery(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.8, + "中文记忆内容", + {"status": "activated"}, + now, + now, + "tenant-a", + ) + ] + + results = db.search_by_fulltext( + query_words=["中文", "记忆"], + top_k=10, + scope="LongTermMemory", + user_name="tenant-a", + filter={"info.agent_id": "agent-1"}, + return_fields=["memory"], + ) + + query, params = cursor.execute.call_args.args + assert "to_tsquery('jiebacfg'" in query + assert "search_vector @@" in query + assert "ILIKE" not in query + assert "properties->'info'->>'agent_id'" in query + assert params[0] == "'中文' & '记忆'" + assert results == [{"id": "node-1", "score": 0.8, "memory": "中文记忆内容"}] + + +def test_get_node_without_tenant_searches_all_partitions(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchone.return_value = ( + "node-1", + "remember this", + {"status": "activated", "user_name": "tenant-b"}, + now, + now, + ) + + result = db.get_node("node-1") + + query, params = cursor.execute.call_args.args + assert "WHERE id = %s" in query + assert "user_name = %s" not in query + assert '"memos_graph_0"."Memory"' in query + assert '"memos_graph_399"."Memory"' in query + assert "FROM memos.memories" not in query + assert params == ("node-1",) + assert result["id"] == "node-1" + + +def test_get_node_with_tenant_remains_tenant_scoped(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = None + + assert db.get_node("node-1", user_name="tenant-b") is None + + query, params = cursor.execute.call_args.args + assert "WHERE id = %s AND user_name = %s" in query + assert params == ("node-1", "tenant-b") + + +def test_get_nodes_without_tenant_searches_all_partitions(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + assert db.get_nodes(["node-1", "node-2"]) == [] + + query, params = cursor.execute.call_args.args + assert "WHERE id = ANY(%s)" in query + assert "user_name = %s" not in query + assert '"memos_graph_0"."Memory"' in query + assert '"memos_graph_399"."Memory"' in query + assert "FROM memos.memories" not in query + assert params == (["node-1", "node-2"],) + + +def test_cube_soft_delete_recover_and_counts_are_available(postgres_db): + db, cursor = postgres_db + cursor.rowcount = 3 + cursor.fetchone.side_effect = [(3,), (0,)] + + assert db.delete_node_by_mem_cube_id("tenant-a", "delete-1") == 3 + assert db.recover_memory_by_mem_cube_id("tenant-a", "delete-1") == 3 + assert db.get_memory_count("LongTermMemory", user_name="tenant-a") == 3 + assert db.node_not_exist("UserMemory", user_name="tenant-a") == 1 + + sql = _executed_sql(cursor) + assert "delete_record_id" in sql + assert "status" in sql + assert "user_name = %s" in sql + + +def test_drop_database_is_scoped_to_current_tenant(postgres_db): + db, _cursor = postgres_db + db.clear = MagicMock() + + db.drop_database() + + db.clear.assert_called_once_with(user_name="tenant-a") + + +def test_structure_candidates_accept_tenant_and_optional_embedding(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + "remember this", + {"status": "activated", "memory_type": "LongTermMemory"}, + now, + now, + [0.1, 0.2, 0.3], + ) + ] + + results = db.get_structure_optimization_candidates( + "LongTermMemory", + include_embedding=True, + user_name="tenant-b", + ) + + query, params = cursor.execute.call_args.args + assert "m.embedding" in query + assert params == ("LongTermMemory", "tenant-b") + assert results[0]["metadata"]["embedding"] == [0.1, 0.2, 0.3] + + +def test_structure_candidates_only_exclude_parent_edges(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + db.get_structure_optimization_candidates( + "LongTermMemory", + user_name="tenant-a", + ) + + query, _ = cursor.execute.call_args.args + assert "e1.edge_type = 'PARENT'" in query + assert "e2.edge_type = 'PARENT'" in query + + +def test_time_filter_uses_timestamp_comparison(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + db.search_by_embedding( + [0.1, 0.2, 0.3], + user_name="tenant-a", + top_k=5, + filter={"created_at": {"gt": "2026-01-01T00:00:00"}}, + ) + + query, _params = cursor.execute.call_args.args + assert "::timestamp >" in query + assert "timestamptz" not in query + + +def test_single_tenant_vector_search_reads_one_hash_partition(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {"tenant-a": 17} + cursor.fetchall.return_value = [] + + db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5) + + query, params = cursor.execute.call_args.args + assert 'FROM "memos_graph_17"."Memory"' in query + assert "FROM memos.memories" not in query + assert query.count("%s::halfvec(3)") == 1 + assert "AS MATERIALIZED" in query + assert "ORDER BY embedding <=> (SELECT query_vector FROM q)" in query + materialized_and_score, outer_filter = query.split("FROM t", 1) + materialized_body = materialized_and_score.split("AS MATERIALIZED", 1)[1] + assert "< 0.9" not in materialized_body + assert "scope_distance < 0.9" in outer_filter + assert "((1 - scope_distance) + 1) / 2.0" in materialized_and_score + assert "embedding::halfvec" not in query + assert "user_name = %s" in query + assert "user_name = ANY(%s)" not in query + assert params[0] == [0.1, 0.2, 0.3] + assert params[1] == "tenant-a" + assert params[-1] == 5 + assert any("hnsw.iterative_scan" in str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_cross_shard_vector_search_reads_only_those_partitions(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {"tenant-a": 17, "kb-1": 19} + cursor.fetchall.return_value = [] + + db.search_by_embedding( + [0.1, 0.2, 0.3], + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + top_k=5, + ) + + query, _params = cursor.execute.call_args.args + assert '"memos_graph_17"."Memory"' in query + assert '"memos_graph_19"."Memory"' in query + assert "UNION ALL" in query + assert "FROM memos.memories" not in query + + +def test_partition_remainder_uses_polardb_md5(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {} + cursor.fetchall.return_value = [] + shard = int(hashlib.md5(b"tenant-a").hexdigest(), 16) % 400 + + db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5) + + query = cursor.execute.call_args.args[0] + assert f'"memos_graph_{shard}"."Memory"' in query + assert "satisfies_hash_partition" not in _executed_sql(cursor) + assert db._partition_remainder("tenant-a") == shard diff --git a/tests/graph_dbs/test_search_return_fields.py b/tests/graph_dbs/test_search_return_fields.py index 1ed0f6173..a23374e5a 100644 --- a/tests/graph_dbs/test_search_return_fields.py +++ b/tests/graph_dbs/test_search_return_fields.py @@ -264,6 +264,73 @@ def fake_connection(): def test_polardb_declares_lightweight_vector_search_support(self, polardb_instance): assert polardb_instance.supports_lightweight_vector_search is True + @pytest.mark.parametrize( + "stored_user_id,expected_user_id", + [ + ("1234567890123456789", "1234567890123456789"), + (1234567890123456789, "1234567890123456789"), + ('"1234567890123456789"', "1234567890123456789"), + ("00123", "00123"), + (None, None), + ("null", None), + ], + ) + def test_lightweight_numeric_user_id_can_be_recalled( + self, polardb_instance, stored_user_id, expected_user_id + ): + from memos.memories.textual.tree_text_memory.retrieve.recall import GraphMemoryRetriever + + polardb_instance.db_name = "test_db" + polardb_instance.config = {"user_name": "default_user"} + polardb_instance._build_user_name_and_kb_ids_conditions_sql = MagicMock(return_value=[]) + polardb_instance._build_filter_conditions_sql = MagicMock(return_value=[]) + cursor = MagicMock() + cursor.fetchall.return_value = [ + ( + 1, + None, + None, + '"00000000-0000-4000-8000-000000000001"', + 0.2, + '"synthetic knowledge"', + '"LongTermMemory"', + stored_user_id, + '["synthetic"]', + '{"test":true}', + "0.9", + "true", + 0.8, + ) + ] + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + + @contextmanager + def fake_connection(): + yield connection + + polardb_instance._get_connection = fake_connection + hits = polardb_instance.search_by_embedding( + vector=[0.1] * 1024, + user_name="knowledge-cube", + light_weight_mode=True, + return_fields=[ + "memory", + "memory_type", + "user_id", + "tags", + "info", + "confidence", + "is_fast", + ], + ) + memory = GraphMemoryRetriever._memory_item_from_vector_hit(hits[0]) + assert memory.metadata.user_id == expected_user_id + assert memory.metadata.tags == ["synthetic"] + assert memory.metadata.info == {"test": True} + assert memory.metadata.confidence == 0.9 + assert memory.metadata.is_fast is True + class TestFieldNameValidation: """Tests for _validate_return_fields injection prevention.""" diff --git a/tests/llms/test_openai_responses_llm.py b/tests/llms/test_openai_responses_llm.py new file mode 100644 index 000000000..d6915b122 --- /dev/null +++ b/tests/llms/test_openai_responses_llm.py @@ -0,0 +1,104 @@ +"""Tests for OpenAIResponsesLLM and AzureResponsesLLM generate reasoning handling.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary + +from memos.configs.llm import AzureLLMConfig, OpenAIResponsesLLMConfig +from memos.llms.openai_new import AzureResponsesLLM, OpenAIResponsesLLM + + +MESSAGES = [{"role": "user", "content": "Hello"}] + + +def _make_openai_llm( + *, enable_thinking: bool = False, remove_think_prefix: bool = False +) -> OpenAIResponsesLLM: + llm = OpenAIResponsesLLM.__new__(OpenAIResponsesLLM) + llm.config = OpenAIResponsesLLMConfig( + api_key="test-key", + model_name_or_path="gpt-4o", + enable_thinking=enable_thinking, + remove_think_prefix=remove_think_prefix, + ) + llm.client = MagicMock() + return llm + + +def _make_azure_llm( + *, enable_thinking: bool = False, remove_think_prefix: bool = False +) -> AzureResponsesLLM: + llm = AzureResponsesLLM.__new__(AzureResponsesLLM) + llm.config = AzureLLMConfig( + api_key="test-key", + model_name_or_path="gpt-4o", + enable_thinking=enable_thinking, + remove_think_prefix=remove_think_prefix, + ) + llm.client = MagicMock() + return llm + + +@pytest.mark.parametrize( + "make_llm", + [_make_openai_llm, _make_azure_llm], + ids=["openai", "azure"], +) +class TestResponsesLLMGenerate: + def test_generate_without_reasoning_item_returns_output_text(self, make_llm): + llm = make_llm(enable_thinking=False) + llm.client.responses.create.return_value = SimpleNamespace( + output=[SimpleNamespace(type="message")], + output_text="plain answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "plain answer" + + def test_generate_with_empty_reasoning_summary_returns_output_text(self, make_llm): + llm = make_llm(enable_thinking=True) + reasoning_item = ResponseReasoningItem(id="rs_1", summary=[], type="reasoning") + llm.client.responses.create.return_value = SimpleNamespace( + output=[reasoning_item], + output_text="plain answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "plain answer" + + def test_generate_with_reasoning_summary_prepends_think_tag(self, make_llm): + llm = make_llm(enable_thinking=True, remove_think_prefix=False) + reasoning_item = ResponseReasoningItem( + id="rs_1", + summary=[Summary(text="step by step", type="summary_text")], + type="reasoning", + ) + llm.client.responses.create.return_value = SimpleNamespace( + output=[reasoning_item], + output_text="final answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "step by stepfinal answer" + + def test_generate_with_remove_think_prefix_strips_thinking(self, make_llm): + llm = make_llm(enable_thinking=True, remove_think_prefix=True) + reasoning_item = ResponseReasoningItem( + id="rs_1", + summary=[Summary(text="step by step", type="summary_text")], + type="reasoning", + ) + llm.client.responses.create.return_value = SimpleNamespace( + output=[reasoning_item], + output_text="inline thoughtfinal answer", + ) + + result = llm.generate(MESSAGES) + + assert result == "final answer" diff --git a/tests/mem_cube/test_single_cube_knowledge_neighbors.py b/tests/mem_cube/test_single_cube_knowledge_neighbors.py new file mode 100644 index 000000000..5cb433d77 --- /dev/null +++ b/tests/mem_cube/test_single_cube_knowledge_neighbors.py @@ -0,0 +1,50 @@ +import uuid + +from unittest.mock import Mock + +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata + + +def make_raw(item_id: str, relativity: float, **extra) -> TextualMemoryItem: + return TextualMemoryItem( + id=item_id, + memory=item_id, + metadata=TreeNodeTextualMemoryMetadata( + memory_type="RawFileMemory", + relativity=relativity, + **extra, + ), + ) + + +def test_postformat_discovers_rawfile_neighbors_by_id_without_edges(): + import memos.api.handlers # noqa: F401 - initialize package before known circular import + + from memos.multi_mem_cube.single_cube import SingleCubeView + + previous = make_raw(str(uuid.uuid4()), 0.1) + following = make_raw(str(uuid.uuid4()), 0.1) + current = make_raw(str(uuid.uuid4()), 1.0, preceding_id=previous.id, following_id=following.id) + searcher = Mock() + neighbors = { + previous.id: previous.model_dump(), + following.id: following.model_dump(), + } + searcher.graph_store.get_node.side_effect = lambda node_id, **_: neighbors.get(node_id) + view = SingleCubeView( + cube_id="cube-1", + naive_mem_cube=Mock(), + mem_reader=Mock(), + mem_scheduler=Mock(), + logger=Mock(), + searcher=searcher, + ) + + result = view._postformat_memories( + [current], "cube-1", include_embedding=False, neighbor_discovery=True + ) + + assert [item["id"] for item in result] == [previous.id, current.id, following.id] + assert result[0]["metadata"]["relativity"] == 0.8 + assert result[2]["metadata"]["relativity"] == 0.8 + searcher.graph_store.get_edges.assert_not_called() diff --git a/tests/mem_feedback/test_feedback.py b/tests/mem_feedback/test_feedback.py index bbf30ae94..3f86ac482 100644 --- a/tests/mem_feedback/test_feedback.py +++ b/tests/mem_feedback/test_feedback.py @@ -1,3 +1,5 @@ +import uuid + from unittest.mock import Mock from memos.mem_feedback.feedback import MemFeedback @@ -139,3 +141,21 @@ def test_update_judgement_prompt_omits_memory_text_from_response_schema(): assert '"reason"' in output_section assert '"text"' not in output_section assert '"old_memory"' not in output_section + + +def test_retrieve_locks_only_knowledge_summaries_without_edge_queries(): + feedback = MemFeedback.__new__(MemFeedback) + feedback.pref_feedback = False + feedback.searcher = Mock() + normal = TextualMemoryItem(id=str(uuid.uuid4()), memory="normal") + knowledge = TextualMemoryItem( + id=str(uuid.uuid4()), + memory="knowledge", + metadata={"knowledge_material_ids": [str(uuid.uuid4())]}, + ) + feedback.searcher.search.return_value = [(normal, 0.9), (knowledge, 0.8)] + + result = feedback._retrieve("query", user_name="cube-1") + + assert result == [normal] + feedback.searcher.graph_store.get_edges.assert_not_called() diff --git a/tests/mem_reader/test_image_parser.py b/tests/mem_reader/test_image_parser.py new file mode 100644 index 000000000..374ac17e9 --- /dev/null +++ b/tests/mem_reader/test_image_parser.py @@ -0,0 +1,76 @@ +"""Tests for vision image URL validation in ImageParser.parse_fine.""" + +import json + +from unittest.mock import MagicMock + +import pytest + +from memos.mem_reader.read_multi_modal.image_parser import ImageParser + + +def _parser() -> tuple[ImageParser, MagicMock]: + llm = MagicMock() + llm.generate.return_value = json.dumps( + { + "memory list": [ + { + "key": "visible label", + "memory_type": "LongTermMemory", + "value": "A product label is visible.", + "tags": ["image"], + } + ], + "summary": "A labeled product.", + } + ) + embedder = MagicMock() + embedder.embed.return_value = [[0.1, 0.2]] + return ImageParser(embedder, llm), llm + + +def _image_message(url: str) -> dict: + return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}} + + +INFO = {"user_id": "user-1", "session_id": "session-1"} + + +@pytest.mark.parametrize( + "url", + [ + "images/c502e59b213998cd695d7f57a6e33de73f5493f2d3fbab6c4e9635341aa7e7b7.jpg", + "image/photo.png", + "/tmp/photo.jpg", + "file:///tmp/photo.jpg", + "data:images/jpeg;base64,aaaa", + "data:image/jpeg,not-base64", + ], +) +def test_parse_fine_skips_non_sendable_image_url(url: str) -> None: + parser, llm = _parser() + + result = parser.parse_fine(_image_message(url), INFO) + + assert result == [] + llm.generate.assert_not_called() + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/device.jpg", + "http://example.com/images/device.jpg", + "data:image/jpeg;base64,aaaa", + "DATA:IMAGE/PNG;base64,bbbb", + ], +) +def test_parse_fine_sends_http_and_base64_image_urls(url: str) -> None: + parser, llm = _parser() + + result = parser.parse_fine(_image_message(url), INFO) + + assert len(result) == 1 + llm.generate.assert_called_once() + sent_url = llm.generate.call_args.args[0][0]["content"][1]["image_url"]["url"] + assert sent_url == url diff --git a/tests/mem_reader/test_multi_modal_rawfile.py b/tests/mem_reader/test_multi_modal_rawfile.py new file mode 100644 index 000000000..f583b7173 --- /dev/null +++ b/tests/mem_reader/test_multi_modal_rawfile.py @@ -0,0 +1,89 @@ +from unittest.mock import MagicMock, patch + +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) + + +def test_mixed_file_source_creates_rawfile_without_changing_prompt_type(): + reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + reader.save_rawfile = True + reader.memory_version_switch = "off" + reader.embedder = MagicMock() + reader.embedder.embed.return_value = [[0.1]] + reader.multi_modal_parser = MagicMock() + + file_info = { + "file_id": "file-1", + "filename": "knowledge.txt", + "file_data": "Complete file content", + } + file_source = SourceMessage( + type="file", + role=None, + content="Current file chunk", + file_info=file_info, + ) + fast_item = TextualMemoryItem( + memory="Please parse this file\nCurrent file chunk\nRecord it completely", + metadata=TreeNodeTextualMemoryMetadata( + user_id="user-1", + session_id="session-1", + memory_type="LongTermMemory", + sources=[ + SourceMessage(type="chat", role="user", content="Please parse this file"), + file_source, + SourceMessage(type="chat", role="user", content="Record it completely"), + ], + file_ids=[file_info["file_id"]], + ), + ) + reader.multi_modal_parser.file_content_parser.create_source.return_value = SourceMessage( + type="file", + content=file_source.content, + doc_path=file_info["filename"], + ) + + llm_response = { + "memory list": [ + { + "key": "Knowledge summary", + "memory_type": "LongTermMemory", + "value": "Summarized knowledge", + "tags": ["knowledge"], + } + ], + "summary": "Summary background", + } + + with ( + patch.object(reader, "_get_llm_response", return_value=llm_response) as get_llm_response, + patch.object( + reader, + "_get_maybe_merged_memory", + side_effect=lambda extracted_memory_dict, **kwargs: extracted_memory_dict, + ), + patch( + "memos.mem_reader.multi_modal_struct.trigger_hook", + side_effect=lambda _hook, **kwargs: kwargs["items"], + ), + ): + result = reader._process_string_fine( + [fast_item], + {"user_id": "user-1", "session_id": "session-1"}, + ) + + summaries = [item for item in result if item.metadata.memory_type != "RawFileMemory"] + rawfiles = [item for item in result if item.metadata.memory_type == "RawFileMemory"] + + assert get_llm_response.call_args.args[3] == "chat" + assert len(summaries) == 1 + assert len(rawfiles) == 1 + assert rawfiles[0].memory == file_source.content + assert rawfiles[0].metadata.summary_ids == [summaries[0].id] + assert reader.multi_modal_parser.file_content_parser.create_source.call_args.kwargs[ + "message" + ] == {"file": file_info} diff --git a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py index c54b49de6..8cdb873ce 100644 --- a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py +++ b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py @@ -4,11 +4,18 @@ from unittest.mock import MagicMock -def _make_text_item(item_id: str, memory: str): +def _make_text_item( + item_id: str, + memory: str, + *, + memory_type: str = "LongTermMemory", + key: str | None = None, + info: dict | None = None, +): metadata = SimpleNamespace( - memory_type="LongTermMemory", - info={}, - key=None, + memory_type=memory_type, + info=info if info is not None else {}, + key=key, status="activated", confidence=0.9, tags=[], @@ -17,8 +24,10 @@ def _make_text_item(item_id: str, memory: str): class _FakeTextMem: - def __init__(self, items): + def __init__(self, items, added_ids: list[str] | None = None): self._items = {item.id: item for item in items} + self._added_ids = added_ids + self.add_calls: list = [] self.delete_calls: list = [] self.soft_delete_calls: list = [] self.memory_manager = SimpleNamespace( @@ -29,6 +38,13 @@ def __init__(self, items): def get(self, mem_id, user_name=None): return self._items[mem_id] + def add(self, memories, user_name=None): + mem_list = list(memories) + self.add_calls.append((mem_list, user_name)) + if self._added_ids is not None: + return list(self._added_ids) + return [item.id for item in mem_list] + def delete(self, ids, user_name=None): self.delete_calls.append((list(ids), user_name)) @@ -98,3 +114,124 @@ def test_fine_transfer_error_does_not_soft_delete_fast_memory_with_versions(): assert text_mem.delete_calls == [] assert text_mem.soft_delete_calls == [] + + +def test_playground_legacy_events_align_enhanced_ids_with_filtered_mem_group(monkeypatch): + import memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler as handler_module + + from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import ( + MemReadMessageHandler, + ) + + monkeypatch.setattr(handler_module, "is_playground_api", lambda: True) + + raw_chunk = _make_text_item("raw_1", "raw chunk") + raw_file_mem = _make_text_item( + "raw_file_1", + "raw file content", + memory_type="RawFileMemory", + key="raw-file-key", + ) + added_mem = _make_text_item( + "mem_add_1", + "user prefers dark mode", + memory_type="LongTermMemory", + key="theme-pref", + ) + updated_mem = _make_text_item( + "mem_upd_1", + "user lives in Shanghai", + memory_type="UserMemory", + key="city-pref", + info={"merged_from": ["old_mem_1"]}, + ) + + text_mem = _FakeTextMem( + [raw_chunk], + added_ids=["enhanced_id_add_1", "enhanced_id_upd_1"], + ) + mem_cube = SimpleNamespace(text_mem=text_mem) + + mem_reader = MagicMock() + mem_reader.fine_transfer_simple_mem.return_value = [[raw_file_mem, added_mem, updated_mem]] + mem_reader.memory_version_switch = "off" + mem_reader.graph_db = None + mem_reader.save_rawfile = False + + submitted_events: list = [] + services = SimpleNamespace( + create_event_log=lambda **kwargs: SimpleNamespace(**kwargs), + submit_web_logs=lambda events, **kwargs: submitted_events.extend(events), + map_memcube_name=lambda mem_cube_id: "UserMemCube", + submit_messages=lambda messages: None, + ) + scheduler_context = SimpleNamespace( + get_mem_cube=lambda: mem_cube, + get_mem_reader=lambda: mem_reader, + services=services, + ) + + handler = MemReadMessageHandler.__new__(MemReadMessageHandler) + handler.scheduler_context = scheduler_context + + handler._process_memories_with_reader( + mem_ids=["raw_1"], + user_id="user_1", + mem_cube_id="cube_1", + text_mem=text_mem, + user_name="cube_1", + task_id="task_1", + info={"trigger_source": "Messages"}, + ) + + assert len(text_mem.add_calls) == 1 + assert text_mem.add_calls[0] == ([added_mem, updated_mem], "cube_1") + + assert len(submitted_events) == 2 + add_event, update_event = submitted_events + + assert add_event.label == "addMemory" + assert add_event.task_id == "task_1" + assert add_event.memory_len == 1 + assert add_event.memcube_log_content == [ + { + "content": "theme-pref: user prefers dark mode", + "ref_id": "enhanced_id_add_1", + } + ] + assert add_event.metadata == [ + { + "ref_id": "enhanced_id_add_1", + "id": "enhanced_id_add_1", + "key": "theme-pref", + "memory": "user prefers dark mode", + "memory_type": "LongTermMemory", + "status": "activated", + "confidence": 0.9, + "tags": [], + "updated_at": None, + } + ] + + assert update_event.label == "updateMemory" + assert update_event.task_id == "task_1" + assert update_event.memory_len == 1 + assert update_event.memcube_log_content == [ + { + "content": "city-pref: user lives in Shanghai", + "ref_id": "enhanced_id_upd_1", + } + ] + assert update_event.metadata == [ + { + "ref_id": "enhanced_id_upd_1", + "id": "enhanced_id_upd_1", + "key": "city-pref", + "memory": "user lives in Shanghai", + "memory_type": "UserMemory", + "status": "activated", + "confidence": 0.9, + "tags": [], + "updated_at": None, + } + ] diff --git a/tests/mem_scheduler/test_rabbitmq_shutdown.py b/tests/mem_scheduler/test_rabbitmq_shutdown.py new file mode 100644 index 000000000..5cb594c43 --- /dev/null +++ b/tests/mem_scheduler/test_rabbitmq_shutdown.py @@ -0,0 +1,110 @@ +"""Native scheduler cleanup only; no models, broker or business data.""" + +import threading +import time + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from memos.api import lifecycle +from memos.configs.mem_scheduler import GeneralSchedulerConfig +from memos.mem_scheduler.optimized_scheduler import OptimizedScheduler +from memos.mem_scheduler.webservice_modules.rabbitmq_service import RabbitMQSchedulerModule + + +def test_close_without_rabbitmq_initialization_is_safe_and_repeatable(): + scheduler = RabbitMQSchedulerModule() + scheduler.rabbitmq_close() + scheduler.rabbitmq_close() + assert scheduler._io_loop_thread is None + assert scheduler._rabbitmq_stop_flag is True + + +def test_early_initialization_skip_still_closes(monkeypatch): + monkeypatch.setenv("CI", "true") + monkeypatch.delenv("MEMOS_ENABLE_RABBITMQ", raising=False) + scheduler = RabbitMQSchedulerModule() + scheduler.initialize_rabbitmq() + assert scheduler._rabbitmq_initializing is False + scheduler.rabbitmq_close() + assert scheduler._io_loop_thread is None + + +def test_real_io_thread_and_connection_cleanup_still_execute(): + scheduler = RabbitMQSchedulerModule() + stopped = threading.Event() + calls = [] + scheduler.rabbitmq_channel = SimpleNamespace( + is_open=True, close=lambda: calls.append("channel") + ) + scheduler.rabbitmq_connection = SimpleNamespace( + is_open=True, + close=lambda: calls.append("connection"), + ioloop=SimpleNamespace(stop=lambda: (calls.append("ioloop"), stopped.set())), + ) + worker = threading.Thread(target=stopped.wait, daemon=True) + scheduler._io_loop_thread = worker + worker.start() + try: + scheduler.rabbitmq_close() + assert not worker.is_alive() + assert calls == ["channel", "connection", "ioloop"] + assert scheduler._rabbitmq_stop_flag is True + finally: + stopped.set() + worker.join(timeout=1) + + +def test_actual_optimized_scheduler_lifecycle_joins_threads_and_shuts_executor(monkeypatch): + scheduler = OptimizedScheduler(GeneralSchedulerConfig(use_redis_queue=False)) + scheduler._running = True + + def native_running_loop(): + while scheduler._running: + time.sleep(0.005) + + consumer = threading.Thread(target=native_running_loop, daemon=True) + monitor = threading.Thread(target=native_running_loop, daemon=True) + scheduler._consumer_thread = consumer + scheduler._monitor_thread = monitor + consumer.start() + monitor.start() + task = scheduler.dispatcher.dispatcher_executor.submit(lambda: 42) + assert task.result(timeout=2) == 42 + shutdown = Mock(wraps=scheduler.dispatcher.shutdown) + monkeypatch.setattr(scheduler.dispatcher, "shutdown", shutdown) + error = Mock() + monkeypatch.setattr(lifecycle.logger, "exception", error) + try: + lifecycle.shutdown_components({"mem_scheduler": scheduler}) + assert not consumer.is_alive() and not monitor.is_alive() + assert scheduler._running is False + assert task.done() and scheduler.dispatcher.dispatcher_executor._shutdown is True + deadline = time.monotonic() + 2 + while ( + any(t.is_alive() for t in scheduler.dispatcher.dispatcher_executor._threads) + and time.monotonic() < deadline + ): + time.sleep(0.005) + assert not any(t.is_alive() for t in scheduler.dispatcher.dispatcher_executor._threads) + shutdown.assert_called_once() + assert scheduler._rabbitmq_stop_flag is True + error.assert_not_called() + finally: + scheduler._running = False + consumer.join(timeout=1) + monitor.join(timeout=1) + scheduler.dispatcher.shutdown() + + +def test_unexpected_join_error_is_not_hidden(): + scheduler = RabbitMQSchedulerModule() + + def fail_join(**kwargs): + raise RuntimeError("fixture join failed") + + scheduler._io_loop_thread = SimpleNamespace(is_alive=lambda: True, join=fail_join) + with pytest.raises(RuntimeError, match="fixture join failed"): + scheduler.rabbitmq_close() diff --git a/tests/memories/textual/test_item.py b/tests/memories/textual/test_item.py new file mode 100644 index 000000000..71c206742 --- /dev/null +++ b/tests/memories/textual/test_item.py @@ -0,0 +1,91 @@ +"""Regression tests for JSON-encoded internal metadata read from graph storage.""" + +import copy +import json +import uuid + +import pytest + +from pydantic import ValidationError + +from memos.memories.textual.item import ( + PreferenceTextualMemoryMetadata, + SearchedTreeNodeTextualMemoryMetadata, + TextualMemoryItem, + TextualMemoryMetadata, + TreeNodeTextualMemoryMetadata, +) + + +@pytest.mark.parametrize( + "metadata_cls", + [ + TextualMemoryMetadata, + TreeNodeTextualMemoryMetadata, + SearchedTreeNodeTextualMemoryMetadata, + PreferenceTextualMemoryMetadata, + ], +) +def test_internal_info_decodes_json_object(metadata_cls): + value = {"ingest_batch_id": "test-batch", "nested": {"chunk_index": 0, "values": [True, None]}} + + metadata = metadata_cls(internal_info=json.dumps(value)) + + assert metadata.internal_info == value + assert metadata.model_dump()["internal_info"] == value + + +@pytest.mark.parametrize("value", [None, {}, {"ingest_batch_id": "test-batch", "nested": [1, 2]}]) +def test_internal_info_preserves_existing_values(value): + original = copy.deepcopy(value) + + metadata = TextualMemoryMetadata(internal_info=value) + + assert metadata.internal_info == original + assert value == original + + +@pytest.mark.parametrize( + "value", + ["{", "not-json", "[]", "null", "1", "true", '"text"', json.dumps('{"batch": "test"}')], +) +def test_internal_info_rejects_malformed_json_and_encoded_non_objects(value): + with pytest.raises(ValidationError): + TextualMemoryMetadata(internal_info=value) + + +@pytest.mark.parametrize("value", [[], [["batch", "test"]], 1, False]) +def test_internal_info_preserves_rejection_of_invalid_types(value): + with pytest.raises(ValidationError): + TextualMemoryMetadata(internal_info=value) + + +def test_search_item_from_dict_decodes_internal_info_without_changing_other_fields(): + value = {"ingest_batch_id": "test-batch", "chunk_index": 0, "chunk_total": 2} + record = { + "id": str(uuid.uuid4()), + "memory": "Synthetic memory for a graph storage round trip.", + "metadata": { + "memory_type": "LongTermMemory", + "internal_info": json.dumps(value), + "relativity": 0.75, + "sources": [{"type": "chat", "role": "user", "content": "Synthetic source."}], + "session_id": "test-session", + }, + } + original = copy.deepcopy(record) + + item = TextualMemoryItem.from_dict(record) + + assert item.metadata.internal_info == value + assert item.id == record["id"] + assert item.memory == record["memory"] + assert item.metadata.sources[0].content == "Synthetic source." + assert item.metadata.session_id == "test-session" + assert item.metadata.relativity == 0.75 + assert record == original + + # Graph storage serializes dictionaries as JSON strings again on the next write. + stored = item.to_dict() + stored["metadata"]["internal_info"] = json.dumps(stored["metadata"]["internal_info"]) + assert TextualMemoryItem.from_dict(stored).to_dict() == item.to_dict() diff --git a/tests/memories/textual/test_tree_knowledge_relations.py b/tests/memories/textual/test_tree_knowledge_relations.py new file mode 100644 index 000000000..79bd80743 --- /dev/null +++ b/tests/memories/textual/test_tree_knowledge_relations.py @@ -0,0 +1,182 @@ +import uuid + +from unittest.mock import Mock + +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.memories.textual.tree import TreeTextMemory + + +class FakeGraphStore: + def __init__(self, items: list[TextualMemoryItem]) -> None: + self.nodes = {item.id: item.model_dump() for item in items} + self.edge_calls: list[tuple] = [] + self.deleted_by_params: list[dict] = [] + + def get_nodes(self, ids, user_name=None): + return [self.nodes[node_id] for node_id in ids if node_id in self.nodes] + + def update_node(self, id, fields, user_name=None): + self.nodes[id]["metadata"].update(fields) + + def delete_node(self, id, user_name=None): + self.nodes.pop(id, None) + + def delete_node_by_prams(self, **kwargs): + self.deleted_by_params.append(kwargs) + for node_id in kwargs.get("memory_ids") or []: + self.nodes.pop(node_id, None) + + def get_by_metadata(self, filters, user_name=None, status=None): + matched = [] + for node_id, node in self.nodes.items(): + metadata = node["metadata"] + if all( + ( + item["value"] in (metadata.get(item["field"]) or []) + if item.get("op") == "contains" + else metadata.get(item["field"]) == item["value"] + ) + for item in filters + ): + matched.append(node_id) + return matched + + def add_edge(self, *args, **kwargs): + self.edge_calls.append((args, kwargs)) + + def get_edges(self, *args, **kwargs): + self.edge_calls.append((args, kwargs)) + return [] + + +def _test_id(value: str) -> str: + try: + return str(uuid.UUID(value)) + except ValueError: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, value)) + + +def make_item( + item_id: str, + memory_type: str, + *, + knowledge_material_ids: list[str] | None = None, + knowledge_summary_ids: list[str] | None = None, + info: dict | None = None, + file_ids: list[str] | None = None, + summary_ids: list[str] | None = None, +) -> TextualMemoryItem: + metadata = TreeNodeTextualMemoryMetadata( + memory_type=memory_type, + knowledge_material_ids=[_test_id(value) for value in knowledge_material_ids or []], + knowledge_summary_ids=[_test_id(value) for value in knowledge_summary_ids or []], + info=( + { + **info, + "merged_from": [_test_id(value) for value in info.get("merged_from", [])], + } + if info and "merged_from" in info + else info + ), + file_ids=file_ids or [], + ) + if summary_ids is not None: + metadata.summary_ids = [_test_id(value) for value in summary_ids] + return TextualMemoryItem(id=_test_id(item_id), memory=item_id, metadata=metadata) + + +def make_tree(items: list[TextualMemoryItem]) -> tuple[TreeTextMemory, FakeGraphStore]: + tree = TreeTextMemory.__new__(TreeTextMemory) + graph_store = FakeGraphStore(items) + tree.graph_store = graph_store + + def add(memories, user_name=None, **kwargs): + for memory in memories: + graph_store.nodes[memory.id] = memory.model_dump() + return [memory.id for memory in memories] + + tree.add = Mock(side_effect=add) + return tree, graph_store + + +def test_add_rawfile_nodes_persists_bidirectional_ids_without_edges(): + summary = make_item("summary-1", "LongTermMemory") + material = make_item("material-1", "RawFileMemory", summary_ids=[summary.id]) + tree, graph_store = make_tree([summary]) + + tree.add_rawfile_nodes([material], [summary.id], user_name="cube-1") + + assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [summary.id] + assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [material.id] + assert graph_store.edge_calls == [] + + +def test_add_rawfile_nodes_reconnects_materials_when_summary_is_merged(): + old_material = make_item("material-old", "RawFileMemory", knowledge_summary_ids=["summary-old"]) + old_summary = make_item( + "summary-old", "LongTermMemory", knowledge_material_ids=[old_material.id] + ) + new_summary = make_item("summary-new", "LongTermMemory", info={"merged_from": [old_summary.id]}) + new_material = make_item("material-new", "RawFileMemory", summary_ids=[new_summary.id]) + tree, graph_store = make_tree([old_material, old_summary, new_summary]) + + tree.add_rawfile_nodes([new_material], [new_summary.id], user_name="cube-1") + + assert graph_store.nodes[new_summary.id]["metadata"]["knowledge_material_ids"] == [ + new_material.id, + old_material.id, + ] + assert graph_store.nodes[old_material.id]["metadata"]["knowledge_summary_ids"] == [ + new_summary.id + ] + + +def test_delete_summary_unlinks_material_without_deleting_it(): + material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"]) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.delete_by_memory_ids([summary.id]) + + assert summary.id not in graph_store.nodes + assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [] + + +def test_delete_material_unlinks_summary_without_deleting_it(): + material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"]) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.delete_by_memory_ids([material.id]) + + assert material.id not in graph_store.nodes + assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [] + + +def test_file_delete_unlinks_knowledge_relations_before_bulk_delete(): + material = make_item( + "material-1", + "RawFileMemory", + knowledge_summary_ids=["summary-1"], + file_ids=["file-1"], + ) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.delete_by_filter(writable_cube_ids=["cube-1"], file_ids=["file-1"]) + + assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [] + assert graph_store.deleted_by_params == [ + {"writable_cube_ids": ["cube-1"], "file_ids": ["file-1"], "filter": None} + ] + + +def test_soft_delete_unlinks_summary_before_marking_it_deleted(): + material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"]) + summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id]) + tree, graph_store = make_tree([material, summary]) + + tree.soft_delete([summary.id], user_name="cube-1") + + assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [] + assert graph_store.nodes[summary.id]["metadata"]["status"] == "deleted" diff --git a/tests/memories/textual/test_tree_searcher.py b/tests/memories/textual/test_tree_searcher.py index b79958ca1..fe40cbb7d 100644 --- a/tests/memories/textual/test_tree_searcher.py +++ b/tests/memories/textual/test_tree_searcher.py @@ -1,3 +1,5 @@ +import uuid + from unittest.mock import MagicMock import pytest @@ -143,3 +145,26 @@ def test_searcher_respects_memory_type(mock_searcher): ) # WorkingMemory triggers only once path A assert mock_searcher.graph_retriever.retrieve.call_args[1]["memory_scope"] == "WorkingMemory" + + +def test_deduplicate_rawfile_results_uses_knowledge_summary_ids(mock_searcher): + summary_id = str(uuid.uuid4()) + material_id = str(uuid.uuid4()) + summary = TextualMemoryItem( + id=summary_id, + memory="summary", + metadata=TreeNodeTextualMemoryMetadata(memory_type="LongTermMemory"), + ) + raw_file = TextualMemoryItem( + id=material_id, + memory="material", + metadata=TreeNodeTextualMemoryMetadata( + memory_type="RawFileMemory", + knowledge_summary_ids=[summary.id], + ), + ) + + result = mock_searcher._deduplicate_rawfile_results([summary, raw_file], user_name="cube-1") + + assert result == [raw_file] + mock_searcher.graph_store.get_edges.assert_not_called() diff --git a/uv.lock b/uv.lock index 385ef4382..d081fa9d5 100644 --- a/uv.lock +++ b/uv.lock @@ -132,11 +132,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.2.25" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -2069,9 +2069,10 @@ wheels = [ [[package]] name = "memoryos" -version = "2.0.14" +version = "2.0.33" source = { editable = "." } dependencies = [ + { name = "cachetools" }, { name = "concurrent-log-handler" }, { name = "fastapi", extra = ["all"] }, { name = "fastmcp" }, @@ -2090,7 +2091,6 @@ dependencies = [ [package.optional-dependencies] all = [ { name = "alibabacloud-oss-v2" }, - { name = "cachetools" }, { name = "chonkie" }, { name = "datasketch" }, { name = "jieba" }, @@ -2142,7 +2142,7 @@ tree-mem = [ requires-dist = [ { name = "alibabacloud-oss-v2", marker = "extra == 'all'", specifier = ">=1.2.2,<1.2.3" }, { name = "alibabacloud-oss-v2", marker = "extra == 'skill-mem'", specifier = ">=1.2.2,<1.2.3" }, - { name = "cachetools", marker = "extra == 'all'", specifier = ">=6.0.0" }, + { name = "cachetools", specifier = ">=6.0.0" }, { name = "chonkie", marker = "extra == 'all'", specifier = ">=1.0.7,<2.0.0" }, { name = "chonkie", marker = "extra == 'mem-reader'", specifier = ">=1.0.7,<2.0.0" }, { name = "concurrent-log-handler", specifier = ">=0.9.28,<1.0.0" },