From bac5e2d19cbe05e0ff6f14991bd969e43f41244d Mon Sep 17 00:00:00 2001 From: dingliang <2650876010@qq.com> Date: Wed, 21 Oct 2026 19:15:19 +0800 Subject: [PATCH 1/3] feat: align PostgreSQL graph storage with current v2.0.36 APIs Store the existing graph interface on MD5 tenant shards so this branch can run on PostgreSQL without attribute-tree tables. Co-authored-by: Cursor --- src/memos/api/config.py | 6 +- src/memos/configs/graph_db.py | 33 +- src/memos/graph_dbs/postgres.py | 2166 ++++++++++++++++--- tests/graph_dbs/test_postgres_400_compat.py | 391 ++++ tests/graph_dbs/test_postgres_pgvector.py | 463 ++++ 5 files changed, 2726 insertions(+), 333 deletions(-) create mode 100644 tests/graph_dbs/test_postgres_400_compat.py create mode 100644 tests/graph_dbs/test_postgres_pgvector.py diff --git a/src/memos/api/config.py b/src/memos/api/config.py index 0247dfa42..f09e51335 100644 --- a/src/memos/api/config.py +++ b/src/memos/api/config.py @@ -906,8 +906,8 @@ def get_polardb_config(user_id: str | None = None) -> dict[str, Any]: def get_postgres_config(user_id: str | None = None) -> dict[str, Any]: """Get PostgreSQL + pgvector configuration for MemOS graph storage. - Uses standard PostgreSQL with pgvector extension. - Schema: memos.memories, memos.edges + Uses standard PostgreSQL with pgvector. Shard count, index, and + full-text configuration keep the backend defaults unless overridden. """ user_name = os.getenv("MEMOS_USER_NAME", "default") if user_id: @@ -922,7 +922,7 @@ def get_postgres_config(user_id: str | None = None) -> dict[str, Any]: "schema_name": os.getenv("MEMOS_SCHEMA", "memos"), "user_name": user_name, "use_multi_db": False, - "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "384")), + "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "1024")), "maxconn": int(os.getenv("POSTGRES_MAX_CONN", "20")), } diff --git a/src/memos/configs/graph_db.py b/src/memos/configs/graph_db.py index 98de09812..984c95060 100644 --- a/src/memos/configs/graph_db.py +++ b/src/memos/configs/graph_db.py @@ -1,4 +1,4 @@ -from typing import Any, ClassVar +from typing import Any, ClassVar, Literal from pydantic import BaseModel, Field, field_validator, model_validator @@ -195,8 +195,7 @@ class PostgresGraphDBConfig(BaseConfig): Does NOT require Apache AGE or other graph extensions. Schema: - - memos_memories: Main table for memory nodes (id, memory, properties JSONB, embedding vector) - - memos_edges: Edge table for relationships (source_id, target_id, type) + - {schema}_graph_{n}."Memory" and "Edges", one physical shard per tenant Example: --- @@ -223,13 +222,33 @@ class PostgresGraphDBConfig(BaseConfig): default=False, description="If False: use single database with logical isolation by user_name", ) - embedding_dimension: int = Field( - default=768, description="Dimension of vector embedding (768 for all-mpnet-base-v2)" + embedding_dimension: int = Field(default=1024, description="Dimension of vector embedding") + shard_count: int = Field( + default=400, + ge=1, + le=4096, + description="Number of MD5 tenant shards", ) maxconn: int = Field( default=20, description="Maximum number of connections in the connection pool", ) + minconn: int = Field( + default=8, + ge=1, + description="Warm connections kept in the pool", + ) + vector_index_type: Literal["hnsw", "ivfflat"] = Field( + default="hnsw", + description="pgvector approximate index type", + ) + hnsw_m: int = Field(default=16, ge=2, le=100) + hnsw_ef_construction: int = Field(default=64, ge=4, le=1000) + ivfflat_lists: int = Field(default=100, ge=1) + fulltext_config: str = Field( + default="jiebacfg", + description="PostgreSQL text search configuration used to build search_vector", + ) @model_validator(mode="after") def validate_config(self): @@ -238,6 +257,10 @@ def validate_config(self): raise ValueError("`db_name` must be provided") if not self.use_multi_db and not self.user_name: raise ValueError("In single-database mode, `user_name` must be provided") + if self.minconn > self.maxconn: + raise ValueError(f"`minconn` ({self.minconn}) must be <= `maxconn` ({self.maxconn})") + if not self.fulltext_config.replace(".", "_").isidentifier(): + raise ValueError("`fulltext_config` must be a valid PostgreSQL configuration name") return self diff --git a/src/memos/graph_dbs/postgres.py b/src/memos/graph_dbs/postgres.py index 594f7e695..0a5460839 100644 --- a/src/memos/graph_dbs/postgres.py +++ b/src/memos/graph_dbs/postgres.py @@ -5,16 +5,18 @@ No Apache AGE or other graph extensions required. Tables: -- {schema}.memories: Memory nodes with JSONB properties and vector embeddings -- {schema}.edges: Relationships between memory nodes +- {schema}_graph_{n}."Memory": one tenant shard of memory nodes +- {schema}_graph_{n}."Edges": relationships for that same shard """ +import hashlib import json +import random import re import time from contextlib import suppress -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Literal from memos.configs.graph_db import PostgresGraphDBConfig @@ -28,7 +30,7 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: """Ensure metadata has proper datetime fields and normalized types.""" - now = datetime.utcnow().isoformat() + now = datetime.now(timezone.utc).isoformat() metadata.setdefault("created_at", now) metadata.setdefault("updated_at", now) @@ -43,6 +45,9 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: class PostgresGraphDB(BaseGraphDB): """PostgreSQL + pgvector implementation of a graph memory store.""" + supports_lightweight_vector_search = True + prefer_lightweight_vector_search = True + @require_python_package( import_name="psycopg2", install_command="pip install psycopg2-binary", @@ -50,26 +55,29 @@ class PostgresGraphDB(BaseGraphDB): ) def __init__(self, config: PostgresGraphDBConfig): """Initialize PostgreSQL connection pool.""" - import psycopg2 import psycopg2.pool self.config = config + if not self._is_safe_field_name(config.schema_name): + raise ValueError(f"Invalid PostgreSQL schema name: {config.schema_name}") self.schema = config.schema_name + self.db_name = config.db_name self.user_name = config.user_name self._pool_closed = False + self._partition_remainders: dict[str, int] = {} logger.info(f"Connecting to PostgreSQL: {config.host}:{config.port}/{config.db_name}") # Create connection pool self.pool = psycopg2.pool.ThreadedConnectionPool( - minconn=2, + minconn=config.minconn, maxconn=config.maxconn, host=config.host, port=config.port, user=config.user, password=config.password, dbname=config.db_name, - connect_timeout=30, + connect_timeout=10, keepalives_idle=30, keepalives_interval=10, keepalives_count=5, @@ -78,8 +86,39 @@ def __init__(self, config: PostgresGraphDBConfig): # Initialize schema and tables self._init_schema() + def warm_up_search_connections_by_full(self, user_name: str | None = None) -> None: + """Warm one pooled PostgreSQL connection; tenant routing needs no fan-out.""" + conn = self._get_conn() + self._put_conn(conn) + + def get_memory_graph_table_name(self, user_name: str | None) -> str: + """Return the one partition table for this tenant, matching PolarDB shard routing.""" + target = user_name or self.user_name + return self._physical_table("memories", target) + + def create_extension(self) -> None: + self._init_schema() + + def create_graph(self) -> None: + self._init_schema() + + def create_edge(self) -> None: + self._init_schema() + + def create_index( + self, + label: str = "Memory", + vector_property: str = "embedding", + dimensions: int = 1024, + index_name: str = "memory_vector_index", + ) -> None: + self._init_schema() + + def format_param_value(self, value: str | None) -> str: + return "null" if value is None else json.dumps(value, ensure_ascii=False) + def _get_conn(self): - """Get connection from pool with health check.""" + """Get a pooled connection without an extra round trip.""" if self._pool_closed: raise RuntimeError("Connection pool is closed") @@ -91,9 +130,6 @@ def _get_conn(self): self.pool.putconn(conn, close=True) continue conn.autocommit = True - # Health check - with conn.cursor() as cur: - cur.execute("SELECT 1") return conn except Exception as e: if conn: @@ -114,66 +150,147 @@ def _put_conn(self, conn): conn.close() def _init_schema(self): - """Create schema and tables if they don't exist.""" + """Create PolarDB-style MD5 shards with a halfvec Memory table in each.""" + fulltext_config = getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_.]*$", fulltext_config): + raise ValueError(f"Invalid PostgreSQL text search config: {fulltext_config}") + vector_index_type = getattr(self.config, "vector_index_type", "hnsw") + shard_count = int(getattr(self.config, "shard_count", 400)) + if shard_count < 1 or shard_count > 4096: + raise ValueError("PostgreSQL shard_count must be between 1 and 4096") + conn = self._get_conn() try: with conn.cursor() as cur: - # Create schema cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}") - - # Enable pgvector cur.execute("CREATE EXTENSION IF NOT EXISTS vector") - # Create memories table - dim = self.config.embedding_dimension - cur.execute(f""" - CREATE TABLE IF NOT EXISTS {self.schema}.memories ( - id TEXT PRIMARY KEY, - memory TEXT NOT NULL DEFAULT '', - properties JSONB NOT NULL DEFAULT '{{}}', - embedding vector({dim}), - user_name TEXT, - created_at TIMESTAMPTZ DEFAULT NOW(), - updated_at TIMESTAMPTZ DEFAULT NOW() - ) - """) - - # Create edges table - cur.execute(f""" - CREATE TABLE IF NOT EXISTS {self.schema}.edges ( - id SERIAL PRIMARY KEY, - source_id TEXT NOT NULL, - target_id TEXT NOT NULL, - edge_type TEXT NOT NULL, - created_at TIMESTAMPTZ DEFAULT NOW(), - UNIQUE(source_id, target_id, edge_type) + cur.execute( + """ + SELECT c.relname, c.relkind + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = %s + AND c.relname = ANY(%s) + """, + ( + self.schema, + ["memories", "edges"], + ), + ) + legacy_rows = cur.fetchall() + if isinstance(legacy_rows, list) and legacy_rows: + raise RuntimeError( + f"{self.schema} still has the old memories layout; " + "configure a new MEMOS_SCHEMA for the MD5 shard layout " + "or migrate the existing data" ) - """) - - # Create indexes - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_user - ON {self.schema}.memories(user_name) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_props - ON {self.schema}.memories USING GIN(properties) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_memories_embedding - ON {self.schema}.memories USING ivfflat(embedding vector_cosine_ops) - WITH (lists = 100) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_edges_source - ON {self.schema}.edges(source_id) - """) - cur.execute(f""" - CREATE INDEX IF NOT EXISTS idx_edges_target - ON {self.schema}.edges(target_id) - """) - - logger.info(f"Schema {self.schema} initialized successfully") + + dim = int(self.config.embedding_dimension) + cur.execute( + """ + SELECT COUNT(*) + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE c.relname = 'Memory' + AND c.relkind = 'r' + AND n.nspname ~ %s + """, + (rf"^{re.escape(self.schema)}_graph_[0-9]+$",), + ) + count_row = cur.fetchone() + memory_tables = ( + count_row[0] if count_row and isinstance(count_row[0], int) else -1 + ) + if memory_tables != shard_count: + hnsw_m = int(getattr(self.config, "hnsw_m", 16)) + ef_construction = int(getattr(self.config, "hnsw_ef_construction", 64)) + lists = int(getattr(self.config, "ivfflat_lists", 100)) + statements: list[str] = [] + for remainder in range(shard_count): + shard_schema = f"{self.schema}_graph_{remainder}" + memory = f'"{shard_schema}"."Memory"' + edges = f'"{shard_schema}"."Edges"' + if vector_index_type == "ivfflat": + vector_index = ( + f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec " + f"ON {memory} USING ivfflat (embedding halfvec_cosine_ops) " + f"WITH (lists = {lists})" + ) + else: + vector_index = ( + f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec " + f"ON {memory} USING hnsw (embedding halfvec_cosine_ops) " + f"WITH (m = {hnsw_m}, ef_construction = {ef_construction})" + ) + statements.extend( + [ + f'CREATE SCHEMA IF NOT EXISTS "{shard_schema}"', + f""" + CREATE TABLE IF NOT EXISTS {memory} ( + id TEXT NOT NULL, + memory TEXT NOT NULL DEFAULT '', + properties JSONB NOT NULL DEFAULT '{{}}', + embedding halfvec({dim}), + user_name TEXT NOT NULL, + search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector( + '{fulltext_config}', + coalesce(memory, '') || ' ' || coalesce(properties::text, '') + ) + ) STORED, + deleted_at TIMESTAMPTZ, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (user_name, id) + ) + """, + f""" + CREATE TABLE IF NOT EXISTS {edges} ( + id BIGSERIAL NOT NULL, + user_name TEXT NOT NULL, + source_id TEXT NOT NULL, + target_id TEXT NOT NULL, + edge_type TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + PRIMARY KEY (user_name, id), + UNIQUE(user_name, source_id, target_id, edge_type) + ) + """, + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_user " + f"ON {memory}(user_name)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_props " + f"ON {memory} USING GIN(properties)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_memory_search_vector " + f"ON {memory} USING GIN(search_vector)" + ), + vector_index, + ( + f"CREATE INDEX IF NOT EXISTS idx_edges_source " + f"ON {edges}(user_name, source_id)" + ), + ( + f"CREATE INDEX IF NOT EXISTS idx_edges_target " + f"ON {edges}(user_name, target_id)" + ), + ( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_tenant_unique " + f"ON {edges}(user_name, source_id, target_id, edge_type)" + ), + ] + ) + cur.execute(";\n".join(statements)) + + logger.info( + "Schema %s initialized successfully with %d MD5 shards", + self.schema, + shard_count, + ) except Exception as e: logger.error(f"Failed to init schema: {e}") raise @@ -197,6 +314,8 @@ def remove_oldest_memory( """ user_name = user_name or self.user_name keep_latest = int(keep_latest) + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: @@ -206,7 +325,7 @@ def remove_oldest_memory( f""" WITH ranked AS ( SELECT id, ROW_NUMBER() OVER (ORDER BY updated_at DESC) as rn - FROM {self.schema}.memories + FROM {memories} WHERE user_name = %s AND properties->>'memory_type' = %s ) @@ -221,101 +340,144 @@ def remove_oldest_memory( # Delete edges first cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) + DELETE FROM {edges} + WHERE user_name = %s + AND (source_id = ANY(%s) OR target_id = ANY(%s)) """, - (ids_to_delete, ids_to_delete), + (user_name, ids_to_delete, ids_to_delete), ) # Delete nodes cur.execute( f""" - DELETE FROM {self.schema}.memories - WHERE id = ANY(%s) + DELETE FROM {memories} + WHERE id = ANY(%s) AND user_name = %s """, - (ids_to_delete,), + (ids_to_delete, user_name), ) logger.info( - f"Removed {len(ids_to_delete)} oldest {memory_type} memories for user {user_name}" + "Removed %d oldest %s memories for user %s", + len(ids_to_delete), + memory_type, + user_name, ) finally: self._put_conn(conn) - def add_node( - self, id: str, memory: str, metadata: dict[str, Any], user_name: str | None = None - ) -> None: - """Add a memory node.""" - user_name = user_name or self.user_name - metadata = _prepare_node_metadata(metadata.copy()) - - # Extract embedding - embedding = metadata.pop("embedding", None) - created_at = metadata.pop("created_at", datetime.utcnow().isoformat()) - updated_at = metadata.pop("updated_at", datetime.utcnow().isoformat()) - - # Serialize sources if present - if metadata.get("sources"): - metadata["sources"] = [ - json.dumps(s) if not isinstance(s, str) else s for s in metadata["sources"] + def _prepare_node_for_insert( + self, + node_id: str, + memory: str, + metadata: dict[str, Any] | None, + user_name: str, + ) -> tuple[Any, ...]: + """Build one write row with the same property shape PolarDB stores.""" + fields = _prepare_node_metadata(dict(metadata or {})) + fields["user_name"] = user_name + fields.setdefault("delete_time", "") + fields.setdefault("delete_record_id", "") + now = datetime.now(timezone.utc).isoformat() + created_at = fields.pop("created_at", now) + updated_at = fields.pop("updated_at", now) + for field_name in ("sources", "usage"): + value = fields.get(field_name) + if isinstance(value, list): + fields[field_name] = [ + item if isinstance(item, str) else json.dumps(item) for item in value + ] + embedding = fields.pop("embedding", None) + if not isinstance(embedding, list) or not embedding: + embedding = [ + round(random.uniform(-0.2, 0.2), 6) for _ in range(self._vector_dim()) ] + else: + embedding = [float(item) for item in embedding] + properties = { + "id": node_id, + "memory": memory, + "created_at": created_at, + "updated_at": updated_at, + "delete_time": fields.get("delete_time", ""), + "delete_record_id": fields.get("delete_record_id", ""), + **fields, + } + return ( + node_id, + memory, + json.dumps(properties, ensure_ascii=False), + embedding, + user_name, + created_at, + updated_at, + ) + + def add_node(self, id: str, memory: str, metadata: dict[str, Any], user_name: str) -> None: + """Add a memory node.""" + if not user_name: + raise ValueError("add_node requires user_name") + self.add_nodes_batch( + [{"id": id, "memory": memory, "metadata": metadata}], + user_name, + ) + def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None: + """Insert every node for one tenant in a single statement.""" + if not nodes: + return + if not user_name: + raise ValueError("add_nodes_batch requires user_name") + rows: list[tuple[Any, ...]] = [] + for node in nodes: + try: + rows.append( + self._prepare_node_for_insert( + node["id"], + node["memory"], + node.get("metadata", {}), + user_name, + ) + ) + except Exception: + logger.error( + "add_nodes_batch failed to prepare node %s", + node.get("id", "unknown"), + exc_info=True, + ) + if not rows: + return + memories = self._physical_table("memories", user_name) + dim = self._vector_dim() + values_sql = ",".join( + [f"(%s, %s, %s, %s::halfvec({dim}), %s, %s, %s)"] * len(rows) + ) + params: list[Any] = [value for row in rows for value in row] conn = self._get_conn() try: with conn.cursor() as cur: - if embedding: - cur.execute( - f""" - INSERT INTO {self.schema}.memories - (id, memory, properties, embedding, user_name, created_at, updated_at) - VALUES (%s, %s, %s, %s::vector, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - memory = EXCLUDED.memory, - properties = EXCLUDED.properties, - embedding = EXCLUDED.embedding, - updated_at = EXCLUDED.updated_at - """, - ( - id, - memory, - json.dumps(metadata), - embedding, - user_name, - created_at, - updated_at, - ), - ) - else: - cur.execute( - f""" - INSERT INTO {self.schema}.memories - (id, memory, properties, user_name, created_at, updated_at) - VALUES (%s, %s, %s, %s, %s, %s) - ON CONFLICT (id) DO UPDATE SET - memory = EXCLUDED.memory, - properties = EXCLUDED.properties, - updated_at = EXCLUDED.updated_at + cur.execute( + f""" + INSERT INTO {memories} + (id, memory, properties, embedding, user_name, created_at, updated_at) + VALUES {values_sql} + ON CONFLICT (user_name, id) DO UPDATE SET + memory = EXCLUDED.memory, + properties = EXCLUDED.properties, + embedding = EXCLUDED.embedding, + updated_at = EXCLUDED.updated_at + WHERE "Memory".user_name = EXCLUDED.user_name """, - (id, memory, json.dumps(metadata), user_name, created_at, updated_at), - ) + params, + ) finally: self._put_conn(conn) - def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str | None = None) -> None: - """Batch add memory nodes.""" - for node in nodes: - self.add_node( - id=node["id"], - memory=node["memory"], - metadata=node.get("metadata", {}), - user_name=user_name, - ) - def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = None) -> None: """Update node fields.""" user_name = user_name or self.user_name if not fields: return + fields = fields.copy() # Get current node current = self.get_node(id, user_name=user_name) @@ -327,7 +489,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N embedding = fields.pop("embedding", None) memory = fields.pop("memory", current.get("memory", "")) props.update(fields) - props["updated_at"] = datetime.utcnow().isoformat() + props["updated_at"] = datetime.now(timezone.utc).isoformat() + memories = self._physical_table("memories", user_name) conn = self._get_conn() try: @@ -335,8 +498,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N if embedding: cur.execute( f""" - UPDATE {self.schema}.memories - SET memory = %s, properties = %s, embedding = %s::vector, updated_at = NOW() + UPDATE {memories} + SET memory = %s, properties = %s, embedding = %s::halfvec({self._vector_dim()}), updated_at = NOW() WHERE id = %s AND user_name = %s """, (memory, json.dumps(props), embedding, id, user_name), @@ -344,7 +507,7 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N else: cur.execute( f""" - UPDATE {self.schema}.memories + UPDATE {memories} SET memory = %s, properties = %s, updated_at = NOW() WHERE id = %s AND user_name = %s """, @@ -356,21 +519,23 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N def delete_node(self, id: str, user_name: str | None = None) -> None: """Delete a node and its edges.""" user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: # Delete edges cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = %s OR target_id = %s + DELETE FROM {edges} + WHERE user_name = %s AND (source_id = %s OR target_id = %s) """, - (id, id), + (user_name, id, id), ) # Delete node cur.execute( f""" - DELETE FROM {self.schema}.memories + DELETE FROM {memories} WHERE id = %s AND user_name = %s """, (id, user_name), @@ -378,22 +543,34 @@ def delete_node(self, id: str, user_name: str | None = None) -> None: finally: self._put_conn(conn) - def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[str, Any] | None: + def get_node( + self, id: str, include_embedding: bool = False, user_name: str | None = None + ) -> dict[str, Any] | None: """Get a single node by ID.""" - user_name = kwargs.get("user_name") or self.user_name + relation = self._tenant_relation("memories", [user_name] if user_name else []) conn = self._get_conn() try: with conn.cursor() as cur: cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" - cur.execute( - f""" - SELECT {cols} FROM {self.schema}.memories - WHERE id = %s AND user_name = %s - """, - (id, user_name), - ) + cols += ", user_name" + if user_name: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = %s AND user_name = %s + """, + (id, user_name), + ) + else: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = %s + """, + (id,), + ) row = cur.fetchone() if not row: return None @@ -402,32 +579,77 @@ def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[s self._put_conn(conn) def get_nodes( - self, ids: list, include_embedding: bool = False, **kwargs + self, ids: list[str], user_name: str | None = None, **kwargs ) -> list[dict[str, Any]]: """Get multiple nodes by IDs.""" if not ids: return [] - user_name = kwargs.get("user_name") or self.user_name + include_embedding = bool(kwargs.get("include_embedding", False)) + relation = self._tenant_relation("memories", [user_name] if user_name else []) conn = self._get_conn() try: with conn.cursor() as cur: cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" - cur.execute( - f""" - SELECT {cols} FROM {self.schema}.memories - WHERE id = ANY(%s) AND user_name = %s - """, - (ids, user_name), - ) + cols += ", user_name" + if user_name: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = ANY(%s) AND user_name = %s + AND COALESCE(properties->>'status', 'activated') <> 'archived' + """, + (ids, user_name), + ) + else: + cur.execute( + f""" + SELECT {cols} FROM {relation} + WHERE id = ANY(%s) + AND COALESCE(properties->>'status', 'activated') <> 'archived' + """, + (ids,), + ) return [self._parse_row(row, include_embedding) for row in cur.fetchall()] finally: self._put_conn(conn) + @staticmethod + def _normalize_node_properties( + raw_properties: dict[str, Any], user_name: str | None = None + ) -> dict[str, Any]: + """Restore the logical PolarDB metadata shape from relational storage.""" + properties = dict(raw_properties) + if user_name: + properties["user_name"] = user_name + + sources = properties.get("sources") + if isinstance(sources, list): + normalized_sources: list[Any] = [] + for source in sources: + if isinstance(source, str): + try: + parsed_source = json.loads(source) + except (json.JSONDecodeError, TypeError): + parsed_source = {"type": "doc", "content": source} + normalized_sources.append(parsed_source) + elif isinstance(source, dict): + normalized_sources.append(source) + else: + normalized_sources.append({"type": "doc", "content": str(source)}) + properties["sources"] = normalized_sources + + return properties + def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]: """Parse database row to node dict.""" - props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + raw_props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + user_name_index = 6 if include_embedding else 5 + user_name = row[user_name_index] if len(row) > user_name_index else None + props = self._normalize_node_properties(raw_props, user_name=user_name) + props["id"] = row[0] + props["memory"] = row[1] or "" props["created_at"] = row[3].isoformat() if row[3] else None props["updated_at"] = row[4].isoformat() if row[4] else None result = { @@ -436,9 +658,21 @@ def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]: "metadata": props, } if include_embedding and len(row) > 5: - result["metadata"]["embedding"] = row[5] + result["metadata"]["embedding"] = self._coerce_embedding(row[5]) return result + @staticmethod + def _coerce_embedding(value: Any) -> list[float]: + """Normalize pgvector values returned as lists, tuples, or text.""" + if value is None: + return [] + if isinstance(value, str): + value = value.strip().strip("[]") + if not value: + return [] + return [float(item) for item in value.split(",")] + return [float(item) for item in value] + @staticmethod def _is_safe_field_name(field: str) -> bool: """Validate field names used in dynamic SQL fragments.""" @@ -486,31 +720,40 @@ def _build_single_filter_condition( op_map = {"gt": ">", "lt": "<", "gte": ">=", "lte": "<="} sql_op = op_map[op] if raw_key in timestamp_fields or raw_key.endswith("_at"): - parts.append(f"({text_expr})::timestamptz {sql_op} %s::timestamptz") + parts.append(f"({text_expr})::timestamp {sql_op} %s::timestamp") params.append(op_value) else: parts.append(f"NULLIF({text_expr}, '')::numeric {sql_op} %s") params.append(op_value) elif op == "contains": - if raw_key in array_fields: - parts.append(f"{json_expr} @> %s::jsonb") - params.append(json.dumps([op_value])) - else: - parts.append(f"{text_expr} ILIKE %s") - params.append(f"%{op_value}%") + values = op_value if isinstance(op_value, list) else [op_value] + or_parts = [] + for item in values: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([item], ensure_ascii=False)) + parts.append("(" + " OR ".join(or_parts) + ")") elif op == "in": if not isinstance(op_value, list): raise ValueError( - f"in operator expects list for '{key}', got {type(op_value).__name__}" + f"in operator expects list for '{key}', " + f"got {type(op_value).__name__}" ) - if raw_key in array_fields: - parts.append(f"{json_expr} ?| %s") - params.append([str(v) for v in op_value]) + if len(op_value) == 0: + parts.append("FALSE") + elif raw_key in array_fields or raw_key == "related_id": + or_parts = [] + for item in op_value: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([str(item)], ensure_ascii=False)) + parts.append("(" + " OR ".join(or_parts) + ")") else: parts.append(f"{text_expr} = ANY(%s)") params.append([str(v) for v in op_value]) elif op == "like": - parts.append(f"{text_expr} ILIKE %s") + parts.append(f"{text_expr} LIKE %s") + params.append(f"%{op_value}%") + elif op == "nolike": + parts.append(f"{text_expr} NOT LIKE %s") params.append(f"%{op_value}%") else: raise ValueError(f"Unsupported filter operator: {op}") @@ -532,6 +775,7 @@ def _build_single_filter_condition( def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[Any]) -> str: """Build SQL WHERE fragment from filter dict.""" + filter_dict = self.parse_filter(filter_dict) if not filter_dict: return "" @@ -542,7 +786,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A parts: list[str] = [] for cond in and_conditions: if isinstance(cond, dict): - cond_sql = self._build_single_filter_condition(cond, params) + cond_sql = self._build_filter_where_clause(cond, params) if cond_sql: parts.append(f"({cond_sql})") return " AND ".join(parts) @@ -554,7 +798,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A parts: list[str] = [] for cond in or_conditions: if isinstance(cond, dict): - cond_sql = self._build_single_filter_condition(cond, params) + cond_sql = self._build_filter_where_clause(cond, params) if cond_sql: parts.append(f"({cond_sql})") return f"({' OR '.join(parts)})" if parts else "" @@ -562,6 +806,55 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A cond_sql = self._build_single_filter_condition(filter_dict, params) return cond_sql or "" + def parse_filter(self, filter_dict: dict | None = None): + """Apply the same implicit info.* field routing as the PolarDB backend.""" + if filter_dict is None: + return None + full_fields = { + "id", + "key", + "tags", + "type", + "usage", + "memory", + "status", + "sources", + "user_id", + "graph_id", + "user_name", + "background", + "confidence", + "created_at", + "session_id", + "updated_at", + "memory_type", + "node_type", + "info", + "source", + "file_ids", + "project_id", + "manager_user_id", + "delete_time", + "related_id", + "message_ids", + } + + def normalize(condition: Any) -> Any: + if not isinstance(condition, dict): + return condition + normalized: dict[str, Any] = {} + for key, value in condition.items(): + if key.lower() in {"and", "or"} and isinstance(value, list): + normalized[key] = [normalize(item) for item in value] + continue + normalized_key = ( + key if key in full_fields or key.startswith("info.") else f"info.{key}" + ) + normalized[normalized_key] = value + return normalized + + return normalize(filter_dict) + def delete_node_by_prams( self, writable_cube_ids: list[str] | None = None, @@ -571,7 +864,8 @@ def delete_node_by_prams( ) -> int: """Delete nodes by memory_ids, file_ids, or filter.""" logger.info( - "[delete_node_by_prams] memory_ids: %s, file_ids: %s, filter: %s, writable_cube_ids: %s", + "[delete_node_by_prams] memory_ids: %s, file_ids: %s, " + "filter: %s, writable_cube_ids: %s", memory_ids, file_ids, filter, @@ -600,7 +894,8 @@ def delete_node_by_prams( if not where_conditions: logger.warning( - "[delete_node_by_prams] No nodes to delete (no memory_ids, file_ids, or filter provided)" + "[delete_node_by_prams] No nodes to delete " + "(no memory_ids, file_ids, or filter provided)" ) return 0 @@ -609,32 +904,148 @@ def delete_node_by_prams( params.append(writable_cube_ids) where_clause = " AND ".join(where_conditions) + memory_tables = self._physical_tables("memories", writable_cube_ids) + edge_tables = self._physical_tables("edges", writable_cube_ids) conn = self._get_conn() try: with conn.cursor() as cur: - query = f""" - WITH to_delete AS ( - SELECT id - FROM {self.schema}.memories - WHERE {where_clause} - ), - deleted_edges AS ( - DELETE FROM {self.schema}.edges e + deleted_count = 0 + for memories, edges in zip(memory_tables, edge_tables, strict=True): + query = f""" + WITH to_delete AS ( + SELECT id, user_name + FROM {memories} + WHERE {where_clause} + ), + deleted_edges AS ( + DELETE FROM {edges} e + USING to_delete d + WHERE e.user_name = d.user_name + AND (e.source_id = d.id OR e.target_id = d.id) + ) + DELETE FROM {memories} m USING to_delete d - WHERE e.source_id = d.id OR e.target_id = d.id - ) - DELETE FROM {self.schema}.memories m - USING to_delete d - WHERE m.id = d.id - """ - cur.execute(query, params) - deleted_count = cur.rowcount if cur.rowcount is not None else 0 + WHERE m.id = d.id + """ + cur.execute(query, params) + deleted_count += cur.rowcount if cur.rowcount is not None else 0 logger.info("[delete_node_by_prams] Deleted %s nodes", deleted_count) return deleted_count finally: self._put_conn(conn) + @staticmethod + def _normalize_cube_memory_groups( + cube_memory_groups: list[dict[str, Any]], + ) -> list[tuple[str, str]]: + """Normalize and de-duplicate (tenant, memory-id) requests.""" + pairs: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for group in cube_memory_groups: + cube_id = str(group.get("cube_id") or "").strip() + if not cube_id: + continue + for raw_memory_id in group.get("memory_ids") or []: + memory_id = str(raw_memory_id).strip() + pair = (cube_id, memory_id) + if memory_id and pair not in seen: + seen.add(pair) + pairs.append(pair) + return pairs + + def delete_node_by_cube_memory_groups(self, cube_memory_groups: list[dict[str, Any]]) -> int: + """Delete exactly the requested (cube, memory-id) pairs. + + PolarDB locates these pairs in one of 400 schemas. PostgreSQL keeps the + same isolation rule in a shared table by joining both user_name and id. + """ + pairs = self._normalize_cube_memory_groups(cube_memory_groups) + if not pairs: + return 0 + user_names = [pair[0] for pair in pairs] + memory_ids = [pair[1] for pair in pairs] + tenant_names = list(dict.fromkeys(user_names)) + memory_tables = self._physical_tables("memories", tenant_names) + edge_tables = self._physical_tables("edges", tenant_names) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + deleted_count = 0 + for memories, edges in zip(memory_tables, edge_tables, strict=True): + cur.execute( + f""" + WITH requested(user_name, memory_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ), to_delete AS ( + SELECT m.id, m.user_name + FROM {memories} m + JOIN requested + ON requested.user_name = m.user_name + AND requested.memory_id = m.id + ), deleted_edges AS ( + DELETE FROM {edges} e + USING to_delete d + WHERE e.user_name = d.user_name + AND (e.source_id = d.id OR e.target_id = d.id) + ), deleted AS ( + DELETE FROM {memories} m + USING to_delete d + WHERE m.id = d.id AND m.user_name = d.user_name + RETURNING 1 + ) + SELECT COUNT(*) FROM deleted + """, + (user_names, memory_ids), + ) + row = cur.fetchone() + deleted_count += int(row[0]) if row and row[0] is not None else 0 + return deleted_count + finally: + self._put_conn(conn) + + def get_user_names_by_cube_memory_groups( + self, cube_memory_groups: list[dict[str, Any]] + ) -> dict[str, str | None]: + """Resolve memory ownership without scanning unrelated tenants.""" + raw_ids = [ + str(memory_id).strip() + for group in cube_memory_groups + for memory_id in (group.get("memory_ids") or []) + if str(memory_id).strip() + ] + result: dict[str, str | None] = dict.fromkeys(raw_ids) + pairs = self._normalize_cube_memory_groups(cube_memory_groups) + if not pairs: + return result + + user_names = [pair[0] for pair in pairs] + memory_ids = [pair[1] for pair in pairs] + memories = self._tenant_relation("memories", list(dict.fromkeys(user_names)), alias="m") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH requested(user_name, memory_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ) + SELECT requested.memory_id, m.user_name + FROM requested + LEFT JOIN {memories} + ON m.id = requested.memory_id + AND m.user_name = requested.user_name + """, + (user_names, memory_ids), + ) + for memory_id, owner in cur.fetchall(): + if owner: + result[str(memory_id)] = str(owner) + return result + finally: + self._put_conn(conn) + # ========================================================================= # Edge Management # ========================================================================= @@ -643,16 +1054,36 @@ def add_edge( self, source_id: str, target_id: str, type: str, user_name: str | None = None ) -> None: """Create an edge between nodes.""" + user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - INSERT INTO {self.schema}.edges (source_id, target_id, edge_type) - VALUES (%s, %s, %s) - ON CONFLICT (source_id, target_id, edge_type) DO NOTHING + INSERT INTO {edges} + (user_name, source_id, target_id, edge_type) + SELECT %s, %s, %s, %s + WHERE EXISTS ( + SELECT 1 FROM {memories} + WHERE id = %s AND user_name = %s + ) AND EXISTS ( + SELECT 1 FROM {memories} + WHERE id = %s AND user_name = %s + ) + ON CONFLICT (user_name, source_id, target_id, edge_type) DO NOTHING """, - (source_id, target_id, type), + ( + user_name, + source_id, + target_id, + type, + source_id, + user_name, + target_id, + user_name, + ), ) finally: self._put_conn(conn) @@ -661,31 +1092,62 @@ def delete_edge( self, source_id: str, target_id: str, type: str, user_name: str | None = None ) -> None: """Delete an edge.""" + user_name = user_name or self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = %s AND target_id = %s AND edge_type = %s + DELETE FROM {edges} + WHERE user_name = %s + AND source_id = %s AND target_id = %s AND edge_type = %s """, - (source_id, target_id, type), + (user_name, source_id, target_id, type), ) finally: self._put_conn(conn) - def edge_exists(self, source_id: str, target_id: str, type: str) -> bool: + def edge_exists( + self, + source_id: str, + target_id: str, + type: str = "ANY", + direction: str = "OUTGOING", + user_name: str | None = None, + ) -> bool: """Check if edge exists.""" + user_name = user_name or self.user_name + direction = direction.upper() + if direction not in {"OUTGOING", "INCOMING", "ANY"}: + raise ValueError("direction must be OUTGOING, INCOMING, or ANY") + if direction == "INCOMING": + source_id, target_id = target_id, source_id + + direction_sql = "source_id = %s AND target_id = %s" + params: list[Any] = [user_name, source_id, target_id] + if direction == "ANY": + direction_sql = ( + "((source_id = %s AND target_id = %s) OR (source_id = %s AND target_id = %s))" + ) + params = [user_name, source_id, target_id, target_id, source_id] + + type_sql = "" + if type != "ANY": + type_sql = " AND edge_type = %s" + params.append(type) + + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - SELECT 1 FROM {self.schema}.edges - WHERE source_id = %s AND target_id = %s AND edge_type = %s + SELECT 1 FROM {edges} + WHERE user_name = %s AND {direction_sql}{type_sql} LIMIT 1 """, - (source_id, target_id, type), + params, ) return cur.fetchone() is not None finally: @@ -696,43 +1158,132 @@ def edge_exists(self, source_id: str, target_id: str, type: str) -> bool: # ========================================================================= def get_neighbors( - self, id: str, type: str, direction: Literal["in", "out", "both"] = "out" + self, + id: str, + type: str, + direction: Literal["in", "out", "both"] = "out", ) -> list[str]: """Get neighboring node IDs.""" + user_name = self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: if direction == "out": cur.execute( f""" - SELECT target_id FROM {self.schema}.edges - WHERE source_id = %s AND edge_type = %s + SELECT target_id FROM {edges} + WHERE user_name = %s AND source_id = %s AND edge_type = %s """, - (id, type), + (user_name, id, type), ) elif direction == "in": cur.execute( f""" - SELECT source_id FROM {self.schema}.edges - WHERE target_id = %s AND edge_type = %s + SELECT source_id FROM {edges} + WHERE user_name = %s AND target_id = %s AND edge_type = %s """, - (id, type), + (user_name, id, type), ) else: # both cur.execute( f""" - SELECT target_id FROM {self.schema}.edges WHERE source_id = %s AND edge_type = %s + SELECT target_id FROM {edges} + WHERE user_name = %s AND source_id = %s AND edge_type = %s UNION - SELECT source_id FROM {self.schema}.edges WHERE target_id = %s AND edge_type = %s + SELECT source_id FROM {edges} + WHERE user_name = %s AND target_id = %s AND edge_type = %s """, - (id, type, id, type), + (user_name, id, type, user_name, id, type), ) return [row[0] for row in cur.fetchall()] finally: self._put_conn(conn) + def get_children_with_embeddings( + self, id: str, user_name: str | None = None + ) -> list[dict[str, Any]]: + """Return PARENT-edge children with vectors for structure optimization.""" + user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT child.id, child.embedding, child.memory + FROM {edges} e + JOIN {memories} child + ON child.id = e.target_id AND child.user_name = e.user_name + WHERE e.user_name = %s + AND e.source_id = %s + AND e.edge_type = 'PARENT' + AND child.deleted_at IS NULL + ORDER BY e.id + """, + (user_name, id), + ) + return [ + { + "id": row[0], + "embedding": self._coerce_embedding(row[1]), + "memory": row[2] or "", + } + for row in cur.fetchall() + ] + finally: + self._put_conn(conn) + + def get_edges( + self, + id: str, + type: str = "ANY", + direction: str = "ANY", + user_name: str | None = None, + ) -> list[dict[str, str]]: + """Return tenant-scoped edges connected to a node.""" + user_name = user_name or self.user_name + direction = direction.upper() + if direction not in {"OUTGOING", "INCOMING", "ANY"}: + raise ValueError("direction must be OUTGOING, INCOMING, or ANY") + + conditions = ["user_name = %s"] + params: list[Any] = [user_name] + if direction == "OUTGOING": + conditions.append("source_id = %s") + params.append(id) + elif direction == "INCOMING": + conditions.append("target_id = %s") + params.append(id) + else: + conditions.append("(source_id = %s OR target_id = %s)") + params.extend([id, id]) + if type != "ANY": + conditions.append("edge_type = %s") + params.append(type) + + edges = self._physical_table("edges", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT source_id, target_id, edge_type + FROM {edges} + WHERE {" AND ".join(conditions)} + ORDER BY id + """, + params, + ) + return [{"from": row[0], "to": row[1], "type": row[2]} for row in cur.fetchall()] + finally: + self._put_conn(conn) + def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[str]: """Get path between nodes using recursive CTE.""" + user_name = self.user_name + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: @@ -740,13 +1291,14 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s f""" WITH RECURSIVE path AS ( SELECT source_id, target_id, ARRAY[source_id] as nodes, 1 as depth - FROM {self.schema}.edges - WHERE source_id = %s + FROM {edges} + WHERE user_name = %s AND source_id = %s UNION ALL SELECT e.source_id, e.target_id, p.nodes || e.source_id, p.depth + 1 - FROM {self.schema}.edges e + FROM {edges} e JOIN path p ON e.source_id = p.target_id - WHERE p.depth < %s AND NOT e.source_id = ANY(p.nodes) + WHERE e.user_name = %s + AND p.depth < %s AND NOT e.source_id = ANY(p.nodes) ) SELECT nodes || target_id as full_path FROM path @@ -754,34 +1306,95 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s ORDER BY depth LIMIT 1 """, - (source_id, max_depth, target_id), + (user_name, source_id, user_name, max_depth, target_id), ) row = cur.fetchone() return row[0] if row else [] finally: self._put_conn(conn) - def get_subgraph(self, center_id: str, depth: int = 2) -> list[str]: - """Get subgraph around center node.""" + def get_subgraph( + self, + center_id: str, + depth: int = 2, + center_status: str = "activated", + user_name: str | None = None, + ) -> dict[str, Any]: + """Return the PolarDB-compatible local subgraph structure.""" + if not 1 <= depth <= 5: + raise ValueError("depth must be 1-5") + center_id = center_id.strip('"') + user_name = user_name or self.user_name + core_node = self.get_node(center_id, user_name=user_name) + if core_node is None: + return {"core_node": None, "neighbors": [], "edges": []} + actual_status = core_node.get("metadata", {}).get("status", "activated") + if center_status and actual_status != center_status: + return {"core_node": None, "neighbors": [], "edges": []} + + edges_table = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - WITH RECURSIVE subgraph AS ( - SELECT %s::text as node_id, 0 as level - UNION - SELECT CASE WHEN e.source_id = s.node_id THEN e.target_id ELSE e.source_id END, - s.level + 1 - FROM {self.schema}.edges e + WITH RECURSIVE subgraph(node_id, level, path) AS ( + SELECT %s::text, 0, ARRAY[%s::text] + UNION ALL + SELECT + CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END, + s.level + 1, + s.path || CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END + FROM {edges_table} e JOIN subgraph s ON (e.source_id = s.node_id OR e.target_id = s.node_id) - WHERE s.level < %s + WHERE e.user_name = %s + AND s.level < %s + AND NOT ( + CASE + WHEN e.source_id = s.node_id THEN e.target_id + ELSE e.source_id + END + ) = ANY(s.path) ) - SELECT DISTINCT node_id FROM subgraph - """, - (center_id, depth), + SELECT DISTINCT node_id FROM subgraph ORDER BY node_id + """, + (center_id, center_id, user_name, depth), ) - return [row[0] for row in cur.fetchall()] + node_ids = [str(row[0]) for row in cur.fetchall()] + neighbor_ids = [node_id for node_id in node_ids if node_id != center_id] + neighbors = ( + self.get_nodes(neighbor_ids, user_name=user_name) if neighbor_ids else [] + ) + + if node_ids: + cur.execute( + f""" + SELECT source_id, target_id, edge_type + FROM {edges_table} + WHERE user_name = %s + AND source_id = ANY(%s) + AND target_id = ANY(%s) + ORDER BY id + """, + (user_name, node_ids, node_ids), + ) + edges = [ + {"source": row[0], "target": row[1], "type": row[2]} + for row in cur.fetchall() + ] + else: + edges = [] + return { + "core_node": core_node, + "neighbors": neighbors, + "edges": edges, + } finally: self._put_conn(conn) @@ -789,95 +1402,642 @@ def get_context_chain(self, id: str, type: str = "FOLLOWS") -> list[str]: """Get ordered chain following relationship type.""" return self.get_neighbors(id, type, "out") + def get_neighbors_by_tag( + self, + tags: list[str], + exclude_ids: list[str], + top_k: int = 5, + min_overlap: int = 1, + include_embedding: bool = False, + user_name: str | None = None, + ) -> list[dict[str, Any]]: + """Find active tenant memories ranked by tag overlap.""" + if not tags or top_k <= 0: + return [] + user_name = user_name or self.user_name + params: list[Any] = [user_name] + exclude_sql = "" + if exclude_ids: + exclude_sql = "AND id <> ALL(%s)" + params.append(exclude_ids) + + cols = "id, memory, properties, created_at, updated_at" + if include_embedding: + cols += ", embedding" + cols += ", user_name" + memories = self._physical_table("memories", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT {cols} + FROM {memories} + WHERE user_name = %s + AND COALESCE(properties->>'status', 'activated') = 'activated' + AND COALESCE(properties->>'node_type', '') <> 'reasoning' + AND COALESCE(properties->>'memory_type', '') <> 'WorkingMemory' + {exclude_sql} + """, + params, + ) + ranked: list[tuple[int, dict[str, Any]]] = [] + requested_tags = set(tags) + for row in cur.fetchall(): + node = self._parse_row(row, include_embedding) + node_tags = node.get("metadata", {}).get("tags", []) + if isinstance(node_tags, str): + try: + node_tags = json.loads(node_tags) + except (TypeError, json.JSONDecodeError): + node_tags = [] + overlap = len(requested_tags.intersection(node_tags or [])) + if overlap >= min_overlap: + ranked.append((overlap, node)) + ranked.sort(key=lambda item: item[0], reverse=True) + return [node for _, node in ranked[:top_k]] + finally: + self._put_conn(conn) + # ========================================================================= # Search Operations # ========================================================================= - def search_by_embedding( + def _effective_user_names( self, - vector: list[float], - top_k: int = 5, - scope: str | None = None, - status: str | None = None, - threshold: float | None = None, - search_filter: dict | None = None, user_name: str | None = None, - filter: dict | None = None, knowledgebase_ids: list[str] | None = None, - **kwargs, - ) -> list[dict]: - """Search nodes by vector similarity using pgvector.""" - user_name = user_name or self.user_name + ) -> list[str]: + """Resolve the memory cubes that a query is allowed to read.""" + names: list[str] = [] + primary = user_name or self.user_name + if primary: + names.append(primary) + for knowledgebase_id in knowledgebase_ids or []: + if knowledgebase_id and knowledgebase_id not in names: + names.append(knowledgebase_id) + return names + + def _shard_count(self) -> int: + return int(getattr(self.config, "shard_count", 400)) + + def _remainder_cache(self) -> dict[str, int]: + cache = getattr(self, "_partition_remainders", None) + if cache is None: + cache = {} + self._partition_remainders = cache + return cache + + def _partition_remainder(self, user_name: str, conn=None) -> int: + """Return the PolarDB MD5 shard number for one tenant.""" + del conn + cache = self._remainder_cache() + cached = cache.get(user_name) + if cached is not None: + return cached + remainder = ( + int(hashlib.md5(user_name.encode("utf-8")).hexdigest(), 16) % self._shard_count() + ) + cache[user_name] = remainder + return remainder - # Build WHERE clause - conditions = ["embedding IS NOT NULL"] - params = [] + def _qualified_partition(self, table: str, remainder: int) -> str: + physical_names = { + "memories": "Memory", + "edges": "Edges", + } + physical = physical_names.get(table) + if physical is None: + raise ValueError(f"Invalid partitioned table: {table}") + schema = f"{self.schema}_graph_{remainder}" + if not self._is_safe_field_name(schema): + raise ValueError(f"Invalid partition schema: {schema}") + return f'"{schema}"."{physical}"' + + def _all_partitions_relation(self, table: str, alias: str | None = None) -> str: + """Read every hash partition explicitly, as PolarDB unions every shard.""" + branches = " UNION ALL ".join( + f"SELECT * FROM {self._qualified_partition(table, remainder)}" + for remainder in range(self._shard_count()) + ) + return f"({branches}) {alias or table}" - if user_name: + def _tenant_relation( + self, + table: str, + user_names: list[str] | None, + alias: str | None = None, + conn=None, + ) -> str: + """Return one child table, or a union of only the touched partitions.""" + names = [name for name in (user_names or []) if name] + if not names: + return self._all_partitions_relation(table, alias) + remainders = sorted({self._partition_remainder(name, conn) for name in names}) + if len(remainders) == 1: + relation = self._qualified_partition(table, remainders[0]) + else: + branches = " UNION ALL ".join( + f"SELECT * FROM {self._qualified_partition(table, remainder)}" + for remainder in remainders + ) + relation = f"({branches})" + if alias is None: + alias = table + return f"{relation} {alias}" if alias else relation + + def _physical_tables( + self, table: str, user_names: list[str] | None, conn=None + ) -> list[str]: + """Real tables a write can target. A union subquery cannot be updated.""" + names = [name for name in (user_names or []) if name] + if not names: + return [ + self._qualified_partition(table, remainder) + for remainder in range(self._shard_count()) + ] + remainders = sorted({self._partition_remainder(name, conn) for name in names}) + return [self._qualified_partition(table, remainder) for remainder in remainders] + + def _physical_table(self, table: str, user_name: str | None, conn=None) -> str: + """Return the one partition table PolarDB would address for this tenant.""" + if not user_name: + return f"{self.schema}.{table}" + return self._qualified_partition(table, self._partition_remainder(user_name, conn)) + + def _vector_dim(self) -> int: + dim = int(getattr(self.config, "embedding_dimension", 1024)) + if dim < 1 or dim > 4096: + raise ValueError(f"Invalid embedding dimension: {dim}") + return dim + + def _ensure_hnsw_search_settings(self, conn) -> None: + """Match PolarDB's HNSW session settings once for each pooled connection.""" + ready = getattr(self, "_hnsw_ready_ids", None) + if ready is None: + ready = set() + self._hnsw_ready_ids = ready + marker = id(conn) + if marker in ready: + return + with conn.cursor() as cur: + cur.execute( + """ + SELECT set_config('hnsw.ef_search', '100', false), + set_config('hnsw.iterative_scan', 'relaxed_order', false) + """ + ) + ready.add(marker) + + def _projected_field_exprs(self, return_fields: list[str] | None) -> list[tuple[str, str]]: + projected: list[tuple[str, str]] = [] + for field in self._validate_return_fields(return_fields): + if field in {"id", "memory", "score", "user_name", "created_at", "updated_at"}: + continue + if field == "embedding": + projected.append((field, "embedding")) + else: + projected.append((field, f"properties->'{field}'")) + return projected + + @staticmethod + def _decode_projected_value(value: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return value + + def _build_projected_search_result( + self, row: tuple[Any, ...], extra_fields: list[str] + ) -> dict[str, Any]: + created_at, updated_at = row[3], row[4] + result: dict[str, Any] = { + "id": row[0], + "score": float(row[1]), + "memory": row[2] or "", + "user_name": row[5], + } + if created_at is not None: + result["created_at"] = ( + created_at.isoformat() if hasattr(created_at, "isoformat") else created_at + ) + if updated_at is not None: + result["updated_at"] = ( + updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at + ) + for offset, field in enumerate(extra_fields): + result[field] = self._decode_projected_value(row[6 + offset]) + return result + + def _append_user_name_condition( + self, conditions: list[str], params: list[Any], user_names: list[str] + ) -> None: + if len(user_names) == 1: conditions.append("user_name = %s") - params.append(user_name) + params.append(user_names[0]) + elif user_names: + conditions.append("user_name = ANY(%s)") + params.append(user_names) + def _build_search_conditions( + self, + params: list[Any], + *, + user_name: str | None = None, + knowledgebase_ids: list[str] | None = None, + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + filter: dict | None = None, + default_activated: bool = True, + ) -> list[str]: + conditions: list[str] = [] + user_names = self._effective_user_names(user_name, knowledgebase_ids) + self._append_user_name_condition(conditions, params, user_names) if scope: conditions.append("properties->>'memory_type' = %s") params.append(scope) - if status: conditions.append("properties->>'status' = %s") params.append(status) - else: - conditions.append( - "(properties->>'status' = 'activated' OR properties->>'status' IS NULL)" - ) + elif default_activated: + conditions.append("properties->>'status' = 'activated'") + + for filter_dict in (search_filter, filter): + if filter_dict: + filter_sql = self._build_filter_where_clause(filter_dict, params) + if filter_sql: + conditions.append(f"({filter_sql})") + return conditions + + def _build_search_result( + self, row: tuple[Any, ...], return_fields: list[str] | None + ) -> dict[str, Any]: + result: dict[str, Any] = {"id": row[0], "score": float(row[1])} + fields = self._validate_return_fields(return_fields) + if not fields: + return result + + memory = row[2] or "" + created_at, updated_at, user_name = row[4], row[5], row[6] + raw_properties = row[3] if isinstance(row[3], dict) else json.loads(row[3] or "{}") + properties = dict(raw_properties) + properties["user_name"] = user_name + properties.update( + { + "id": row[0], + "memory": memory, + "created_at": created_at.isoformat() if created_at else None, + "updated_at": updated_at.isoformat() if updated_at else None, + "user_name": user_name, + } + ) + for field in fields: + if field != "id" and field in properties: + result[field] = properties[field] + return result - if search_filter: - for k, v in search_filter.items(): - conditions.append(f"properties->>'{k}' = %s") - params.append(str(v)) + def _build_keyword_result( + self, row: tuple[Any, ...], return_fields: list[str] | None + ) -> dict[str, Any]: + result: dict[str, Any] = {"id": row[0]} + fields = self._validate_return_fields(return_fields) + if not fields: + return result + + raw_properties = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}") + properties = dict(raw_properties) + properties["user_name"] = row[5] + properties.update( + { + "id": row[0], + "memory": row[1] or "", + "created_at": row[3].isoformat() if row[3] else None, + "updated_at": row[4].isoformat() if row[4] else None, + "user_name": row[5], + } + ) + for field in fields: + if field != "id" and field in properties: + result[field] = properties[field] + return result + def search_by_embedding( + self, + vector: list[float], + user_name: str, + top_k: int = 5, + scope: str | None = None, + status: str | None = None, + threshold: float | None = None, + search_filter: dict | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + return_fields: list[str] | None = None, + light_weight_mode: bool = False, + **kwargs, + ) -> list[dict]: + """Search nodes by vector similarity using pgvector.""" + if light_weight_mode and not return_fields: + return_fields = [ + "memory", + "memory_type", + "user_name", + "status", + "key", + "tags", + "confidence", + "source", + ] + params: list[Any] = [] + conditions = ["embedding IS NOT NULL"] + conditions.extend( + self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + ) where_clause = " AND ".join(conditions) + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) - # pgvector cosine distance: 1 - (a <=> b) gives similarity score + # Match PolarDB: one distance, index-ordered LIMIT, then drop rows at or past 0.9. + dim = self._vector_dim() + distance_sql = "embedding <=> (SELECT query_vector FROM q)" conn = self._get_conn() try: + self._ensure_hnsw_search_settings(conn) with conn.cursor() as cur: + projected = ( + self._projected_field_exprs(return_fields) if light_weight_mode else [] + ) + if light_weight_mode: + inner_columns = "memory, created_at, updated_at, user_name" + extra_inner = "".join( + f', {expression} AS "{field}"' for field, expression in projected + ) + extra_outer = "".join(f', "{field}"' for field, _ in projected) + else: + inner_columns = "memory, properties, created_at, updated_at, user_name" + extra_inner = "" + extra_outer = "" cur.execute( f""" - SELECT id, 1 - (embedding <=> %s::vector) as score - FROM {self.schema}.memories - WHERE {where_clause} - ORDER BY embedding <=> %s::vector - LIMIT %s - """, - (vector, *params, vector, top_k), + WITH q AS ( + SELECT %s::halfvec({dim}) AS query_vector + ), + t AS MATERIALIZED ( + SELECT COALESCE(properties->>'id', id) AS id, + ({distance_sql}) AS scope_distance, + {inner_columns} + {extra_inner} + FROM {relation} + WHERE {where_clause} + ORDER BY {distance_sql} + LIMIT %s + ) + SELECT id, + ((1 - scope_distance) + 1) / 2.0 AS score, + {inner_columns} + {extra_outer} + FROM t + WHERE scope_distance < 0.9 + """, + (vector, *params, top_k), ) results = [] for row in cur.fetchall(): score = float(row[1]) if threshold is None or score >= threshold: - results.append({"id": row[0], "score": score}) + if light_weight_mode: + results.append( + self._build_projected_search_result( + row, [field for field, _ in projected] + ) + ) + else: + results.append(self._build_search_result(row, return_fields)) return results finally: self._put_conn(conn) - def get_by_metadata( + def search_by_fulltext( self, - filters: list[dict[str, Any]], + query_words: list[str], + top_k: int = 10, + scope: str | None = None, status: str | None = None, + threshold: float | None = None, + search_filter: dict | None = None, user_name: str | None = None, filter: dict | None = None, knowledgebase_ids: list[str] | None = None, + tsvector_field: str = "properties_tsvector_zh", + tsquery_config: str = "jiebacfg", + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Search with PolarDB's AND tsquery on one tenant partition.""" + terms = [str(word).strip() for word in query_words if str(word).strip()] + if not terms: + return [] + + query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config): + raise ValueError(f"Invalid PostgreSQL text search config: {query_config}") + quoted_terms = [] + for term in terms: + if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}: + term = term[1:-1] + quoted_terms.append("'" + term.replace("'", "''") + "'") + params: list[Any] = [" & ".join(quoted_terms)] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH q AS ( + SELECT to_tsquery('{query_config}', %s) AS query + ) + SELECT COALESCE(properties->>'id', id) AS id, + ts_rank(search_vector, q.query) AS score, + memory, properties, created_at, updated_at, user_name + FROM {relation} + CROSS JOIN q + WHERE {where_clause} + AND search_vector @@ q.query + ORDER BY score DESC, updated_at DESC + LIMIT %s + """, + (*params, top_k), + ) + output = [] + for row in cur.fetchall(): + result = self._build_search_result(row, return_fields) + if threshold is None or result["score"] >= threshold: + output.append(result) + return output + finally: + self._put_conn(conn) + + def search_by_keywords_tfidf( + self, + query_words: list[str], + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + tsvector_field: str = "properties_tsvector_zh", + tsquery_config: str = "jiebaqry", + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Return keyword matches with PolarDB's OR tsquery and no score.""" + if not user_name: + return [] + terms = [str(word).strip() for word in query_words if str(word).strip()] + if not terms: + return [] + + query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple") + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config): + raise ValueError(f"Invalid PostgreSQL text search config: {query_config}") + quoted_terms = [] + for term in terms: + if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}: + term = term[1:-1] + quoted_terms.append("'" + term.replace("'", "''") + "'") + params: list[Any] = [" | ".join(quoted_terms)] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + search_filter=search_filter, + filter=filter, + ) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + WITH q AS ( + SELECT to_tsquery('{query_config}', %s) AS query + ) + SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name + FROM {relation} + CROSS JOIN q + WHERE {where_clause} + AND search_vector @@ q.query + """, + (*params,), + ) + return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()] + finally: + self._put_conn(conn) + + def search_by_keywords_like( + self, + query_word: str, + scope: str | None = None, + status: str | None = None, + search_filter: dict | None = None, + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list[str] | None = None, + return_fields: list[str] | None = None, + **kwargs, + ) -> list[dict]: + """Fallback substring search compatible with the PolarDB adapter.""" + if not user_name: + return [] + pattern = f"%{query_word}%" + params: list[Any] = [] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + search_filter=search_filter, + filter=filter, + status=status, + scope=scope, + ) + conditions.append("COALESCE(properties->>'memory', '') LIKE %s") + params.append(pattern) + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) + + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name + FROM {relation} + WHERE {" AND ".join(conditions)} + """, + params, + ) + return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()] + finally: + self._put_conn(conn) + + def get_by_metadata( + self, + filters: list[dict[str, Any]], + user_name: str | None = None, + filter: dict | None = None, + knowledgebase_ids: list | None = None, user_name_flag: bool = True, + status: str | None = None, + **kwargs, ) -> list[str]: """Get node IDs matching metadata filters.""" - user_name = user_name or self.user_name + if not user_name: + raise ValueError("get_by_metadata requires user_name") + conditions: list[str] = [] + params: list[Any] = [] + user_names: list[str] = [] - conditions = [] - params = [] - - if user_name_flag and user_name: - conditions.append("user_name = %s") - params.append(user_name) + if user_name_flag: + user_names = self._effective_user_names(user_name, knowledgebase_ids) + self._append_user_name_condition(conditions, params, user_names) if status: conditions.append("properties->>'status' = %s") @@ -887,29 +2047,52 @@ def get_by_metadata( field = f["field"] op = f.get("op", "=") value = f["value"] + text_expr, json_expr = self._field_expr(field) if op == "=": - conditions.append(f"properties->>'{field}' = %s") + conditions.append(f"{text_expr} = %s") params.append(str(value)) elif op == "in": - placeholders = ",".join(["%s"] * len(value)) - conditions.append(f"properties->>'{field}' IN ({placeholders})") - params.extend([str(v) for v in value]) + if not isinstance(value, list): + raise ValueError("in operator expects a list") + conditions.append(f"{text_expr} = ANY(%s)") + params.append([str(item) for item in value]) elif op in (">", ">=", "<", "<="): - conditions.append(f"(properties->>'{field}')::numeric {op} %s") + conditions.append(f"NULLIF({text_expr}, '')::numeric {op} %s") params.append(value) elif op == "contains": - conditions.append(f"properties->'{field}' @> %s::jsonb") - params.append(json.dumps([value])) + values = value if isinstance(value, list) else [value] + or_parts = [] + for item in values: + or_parts.append(f"{json_expr} @> %s::jsonb") + params.append(json.dumps([item], ensure_ascii=False)) + conditions.append("(" + " OR ".join(or_parts) + ")") + elif op == "like": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"%{value}%") + elif op == "starts_with": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"{value}%") + elif op == "ends_with": + conditions.append(f"{text_expr} LIKE %s") + params.append(f"%{value}") + else: + raise ValueError(f"Unsupported operator: {op}") + + if filter: + filter_sql = self._build_filter_where_clause(filter, params) + if filter_sql: + conditions.append(f"({filter_sql})") where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation("memories", user_names) conn = self._get_conn() try: with conn.cursor() as cur: cur.execute( f""" - SELECT id FROM {self.schema}.memories + SELECT id FROM {relation} WHERE {where_clause} """, params, @@ -921,23 +2104,34 @@ def get_by_metadata( def get_all_memory_items( self, scope: str, + user_name: str | None = None, include_embedding: bool = False, - status: str | None = None, filter: dict | None = None, - knowledgebase_ids: list[str] | None = None, - **kwargs, + knowledgebase_ids: list | None = None, + status: str | None = None, ) -> list[dict]: """Get all memory items of a specific type.""" - user_name = kwargs.get("user_name") or self.user_name - - conditions = ["properties->>'memory_type' = %s", "user_name = %s"] - params = [scope, user_name] + if not user_name: + return [] + if scope not in {"WorkingMemory", "LongTermMemory", "UserMemory", "OuterMemory"}: + raise ValueError(f"Unsupported memory type scope: {scope}") - if status: - conditions.append("properties->>'status' = %s") - params.append(status) + params: list[Any] = [] + conditions = self._build_search_conditions( + params, + user_name=user_name, + knowledgebase_ids=knowledgebase_ids, + scope=scope, + status=status, + filter=filter, + default_activated=False, + ) - where_clause = " AND ".join(conditions) + where_clause = " AND ".join(conditions) if conditions else "TRUE" + relation = self._tenant_relation( + "memories", + self._effective_user_names(user_name, knowledgebase_ids), + ) conn = self._get_conn() try: @@ -945,10 +2139,12 @@ def get_all_memory_items( cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" + cols += ", user_name" cur.execute( f""" - SELECT {cols} FROM {self.schema}.memories + SELECT {cols} FROM {relation} WHERE {where_clause} + LIMIT 100 """, params, ) @@ -957,29 +2153,44 @@ def get_all_memory_items( self._put_conn(conn) def get_structure_optimization_candidates( - self, scope: str, include_embedding: bool = False + self, + scope: str, + include_embedding: bool = False, + user_name: str | None = None, ) -> list[dict]: """Find isolated nodes (no edges).""" - user_name = self.user_name + if not user_name: + raise ValueError("get_structure_optimization_candidates requires user_name") + memories = self._tenant_relation("memories", [user_name]) + edges = self._tenant_relation("edges", [user_name]) conn = self._get_conn() try: with conn.cursor() as cur: cols = "m.id, m.memory, m.properties, m.created_at, m.updated_at" + if include_embedding: + cols += ", m.embedding" + cols += ", m.user_name" cur.execute( f""" SELECT {cols} - FROM {self.schema}.memories m - LEFT JOIN {self.schema}.edges e1 ON m.id = e1.source_id - LEFT JOIN {self.schema}.edges e2 ON m.id = e2.target_id + FROM {memories} m + LEFT JOIN {edges} e1 + ON m.id = e1.source_id + AND e1.user_name = m.user_name + AND e1.edge_type = 'PARENT' + LEFT JOIN {edges} e2 + ON m.id = e2.target_id + AND e2.user_name = m.user_name + AND e2.edge_type = 'PARENT' WHERE m.properties->>'memory_type' = %s AND m.user_name = %s AND m.properties->>'status' = 'activated' AND e1.id IS NULL AND e2.id IS NULL - """, + """, (scope, user_name), ) - return [self._parse_row(row, False) for row in cur.fetchall()] + return [self._parse_row(row, include_embedding) for row in cur.fetchall()] finally: self._put_conn(conn) @@ -987,6 +2198,190 @@ def get_structure_optimization_candidates( # Maintenance # ========================================================================= + def get_memory_count(self, memory_type: str, user_name: str | None = None) -> int: + """Count active memories of one type for a tenant.""" + user_name = user_name or self.user_name + relation = self._tenant_relation("memories", [user_name] if user_name else []) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT COUNT(*) + FROM {relation} + WHERE user_name = %s + AND properties->>'memory_type' = %s + AND COALESCE(properties->>'status', 'activated') = 'activated' + """, + (user_name, memory_type), + ) + row = cur.fetchone() + return int(row[0]) if row else 0 + finally: + self._put_conn(conn) + + def node_not_exist(self, scope: str, user_name: str | None = None) -> int: + """Return 1 when no active node exists for the requested scope.""" + return int(self.get_memory_count(scope, user_name=user_name) == 0) + + def count_nodes(self, scope: str, user_name: str | None = None) -> int: + """Compatibility alias for memory counts.""" + return self.get_memory_count(scope, user_name=user_name) + + def get_user_names_by_memory_ids(self, memory_ids: list[str]) -> dict[str, str | None]: + normalized_ids = list( + dict.fromkeys( + str(memory_id).strip() for memory_id in memory_ids if str(memory_id).strip() + ) + ) + if not normalized_ids: + return {} + result: dict[str, str | None] = dict.fromkeys(normalized_ids) + memories = self._all_partitions_relation("memories") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, user_name + FROM {memories} + WHERE id = ANY(%s) + """, + (normalized_ids,), + ) + result.update({str(row[0]): row[1] for row in cur.fetchall()}) + return result + finally: + self._put_conn(conn) + + def exist_user_name(self, user_name: str) -> dict[str, bool]: + memories = self._physical_table("memories", user_name) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f"SELECT EXISTS(SELECT 1 FROM {memories} WHERE user_name = %s)", + (user_name,), + ) + row = cur.fetchone() + return {user_name: bool(row and row[0])} + finally: + self._put_conn(conn) + + def delete_node_by_mem_cube_id( + self, + mem_cube_id: str, + delete_record_id: str, + hard_delete: bool = False, + ) -> int: + """Soft-delete a tenant's memories, or purge a previously deleted batch.""" + if not mem_cube_id: + raise ValueError("delete_node_by_mem_cube_id mem_cube_id is required") + if not delete_record_id: + raise ValueError("delete_node_by_mem_cube_id delete_record_id is required") + + memories = self._physical_table("memories", mem_cube_id) + edges = self._physical_table("edges", mem_cube_id) + conn = self._get_conn() + try: + with conn.cursor() as cur: + if hard_delete: + cur.execute( + f""" + WITH deleted AS ( + DELETE FROM {memories} + WHERE user_name = %s + AND properties->>'delete_record_id' = %s + RETURNING id + ), deleted_edges AS ( + DELETE FROM {edges} e + USING deleted d + WHERE e.user_name = %s + AND (e.source_id = d.id OR e.target_id = d.id) + RETURNING e.id + ) + SELECT COUNT(*) FROM deleted + """, + (mem_cube_id, delete_record_id, mem_cube_id), + ) + row = cur.fetchone() + return int(row[0]) if row else 0 + else: + deleted_at = datetime.now(timezone.utc).isoformat() + cur.execute( + f""" + UPDATE {memories} + SET properties = properties || %s::jsonb, + deleted_at = %s::timestamptz, + updated_at = NOW() + WHERE user_name = %s + AND COALESCE(properties->>'status', 'activated') <> 'deleted' + """, + ( + json.dumps( + { + "status": "deleted", + "delete_time": deleted_at, + "delete_record_id": delete_record_id, + } + ), + deleted_at, + mem_cube_id, + ), + ) + return max(int(cur.rowcount or 0), 0) + finally: + self._put_conn(conn) + + def recover_memory_by_mem_cube_id( + self, + mem_cube_id: str, + delete_record_id: str, + ) -> int: + """Recover one tenant soft-delete batch.""" + if not mem_cube_id: + raise ValueError("recover_memory_by_mem_cube_id mem_cube_id is required") + if not delete_record_id: + raise ValueError("recover_memory_by_mem_cube_id delete_record_id is required") + + memories = self._physical_table("memories", mem_cube_id) + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + f""" + UPDATE {memories} + SET properties = properties || %s::jsonb, + deleted_at = NULL, + updated_at = NOW() + WHERE user_name = %s + AND properties->>'delete_record_id' = %s + AND properties->>'status' = 'deleted' + """, + ( + json.dumps( + { + "status": "activated", + "delete_time": "", + "delete_record_id": "", + } + ), + mem_cube_id, + delete_record_id, + ), + ) + return max(int(cur.rowcount or 0), 0) + finally: + self._put_conn(conn) + + def drop_database(self) -> None: + """Compatibility operation: clear only the configured tenant. + + The standard PostgreSQL backend is shared, so dropping the physical + database or schema would delete other tenants. + """ + self.clear(user_name=self.user_name) + def deduplicate_nodes(self) -> None: """Not implemented - handled at application level.""" @@ -1012,6 +2407,8 @@ def get_grouped_counts( user_name = user_name or self.user_name if not group_fields: raise ValueError("group_fields cannot be empty") + if any(not self._is_safe_field_name(field) for field in group_fields): + raise ValueError("group_fields contains an invalid field name") # Build SELECT and GROUP BY clauses # Fields come from JSONB properties column @@ -1033,10 +2430,11 @@ def get_grouped_counts( query_params.extend(params.values()) where_sql = " AND ".join(conditions) + memories = self._physical_table("memories", user_name) query = f""" SELECT {select_fields}, COUNT(*) AS count - FROM {self.schema}.memories + FROM {memories} WHERE {where_sql} GROUP BY {group_by} """ @@ -1067,13 +2465,15 @@ def merge_nodes(self, id1: str, id2: str) -> str: def clear(self, user_name: str | None = None) -> None: """Clear all data for user.""" user_name = user_name or self.user_name + memories = self._physical_table("memories", user_name) + edges = self._physical_table("edges", user_name) conn = self._get_conn() try: with conn.cursor() as cur: # Get all node IDs for user cur.execute( f""" - SELECT id FROM {self.schema}.memories WHERE user_name = %s + SELECT id FROM {memories} WHERE user_name = %s """, (user_name,), ) @@ -1083,16 +2483,17 @@ def clear(self, user_name: str | None = None) -> None: # Delete edges cur.execute( f""" - DELETE FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) + DELETE FROM {edges} + WHERE user_name = %s + AND (source_id = ANY(%s) OR target_id = ANY(%s)) """, - (ids, ids), + (user_name, ids, ids), ) # Delete nodes cur.execute( f""" - DELETE FROM {self.schema}.memories WHERE user_name = %s + DELETE FROM {memories} WHERE user_name = %s """, (user_name,), ) @@ -1100,36 +2501,148 @@ def clear(self, user_name: str | None = None) -> None: finally: self._put_conn(conn) - def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, Any]: - """Export all data.""" - user_name = kwargs.get("user_name") or self.user_name + def export_graph( + self, + user_name: str | None = None, + include_embedding: bool = False, + user_id: str | None = None, + page: int | None = None, + page_size: int | None = None, + filter: dict | None = None, + memory_type: list[str] | None = None, + status: list[str] | None = None, + **kwargs, + ) -> dict[str, Any]: + """Export memories with the same filters and pagination as PolarDB.""" + + extracted_object_type: str | None = None + extracted_mem_cube_id: str | None = None + + def extract_special_values(value: Any) -> Any: + nonlocal extracted_object_type, extracted_mem_cube_id + if not isinstance(value, dict): + return value + cleaned: dict[str, Any] = {} + for key, item in value.items(): + if key in {"and", "or"} and isinstance(item, list): + children = [extract_special_values(child) for child in item] + children = [child for child in children if child not in (None, {}, [])] + if children: + cleaned[key] = children + elif key == "object_type" and isinstance(item, str): + extracted_object_type = extracted_object_type or item + elif key == "mem_cube_id" and isinstance(item, str): + extracted_mem_cube_id = extracted_mem_cube_id or item + else: + cleaned[key] = item + return cleaned or None + + filter_for_sql = extract_special_values(filter) + conditions: list[str] = [] + params: list[Any] = [] + has_object_filter = bool(extracted_object_type and extracted_mem_cube_id) + object_type = (extracted_object_type or "").strip().lower() + + if has_object_filter: + if object_type == "user": + conditions.append("user_name <> %s") + params.append(extracted_mem_cube_id) + elif object_type == "public": + conditions.append("user_name = %s") + params.append(extracted_mem_cube_id) + elif user_name: + conditions.append("user_name = %s") + params.append(user_name) + + effective_user_id = user_id or getattr(self.config, "user_id", None) + if effective_user_id: + conditions.append("properties->>'user_id' = %s") + params.append(effective_user_id) + if memory_type: + conditions.append("properties->>'memory_type' = ANY(%s)") + params.append([str(item) for item in memory_type]) + if status: + conditions.append("properties->>'status' = ANY(%s)") + params.append([str(item) for item in status]) + else: + conditions.append("COALESCE(properties->>'status', 'activated') <> 'deleted'") + conditions.append("deleted_at IS NULL") + if filter_for_sql: + filter_sql = self._build_filter_where_clause(filter_for_sql, params) + if filter_sql: + conditions.append(f"({filter_sql})") + + where_clause = " AND ".join(conditions) if conditions else "TRUE" + target_user = user_name + if has_object_filter and object_type == "public": + target_user = extracted_mem_cube_id + elif has_object_filter and object_type == "user": + target_user = None + memories = self._physical_table("memories", target_user) + use_pagination = page is not None and page_size is not None + if use_pagination: + page = max(int(page), 1) + page_size = max(int(page_size), 1) + offset = (page - 1) * page_size + conn = self._get_conn() try: with conn.cursor() as cur: - # Get nodes + cur.execute( + f"SELECT COUNT(*) FROM {memories} WHERE {where_clause}", + params, + ) + count_row = cur.fetchone() + total_nodes = int(count_row[0]) if count_row else 0 + cols = "id, memory, properties, created_at, updated_at" if include_embedding: cols += ", embedding" + cols += ", user_name" + pagination_sql = "" + data_params = list(params) + if use_pagination: + pagination_sql = "LIMIT %s OFFSET %s" + data_params.extend([page_size, offset]) cur.execute( f""" - SELECT {cols} FROM {self.schema}.memories - WHERE user_name = %s - ORDER BY created_at DESC - """, - (user_name,), + SELECT {cols} + FROM {memories} + WHERE {where_clause} + ORDER BY created_at DESC NULLS LAST, id DESC + {pagination_sql} + """, + data_params, ) nodes = [self._parse_row(row, include_embedding) for row in cur.fetchall()] - # Get edges node_ids = [n["id"] for n in nodes] - if node_ids: + node_user_names = [ + n.get("metadata", {}).get("user_name") or user_name for n in nodes + ] + if node_ids and all(node_user_names): + edges_table = self._tenant_relation( + "edges", + list(dict.fromkeys(str(name) for name in node_user_names if name)), + alias="e", + conn=conn, + ) cur.execute( f""" - SELECT source_id, target_id, edge_type - FROM {self.schema}.edges - WHERE source_id = ANY(%s) OR target_id = ANY(%s) - """, - (node_ids, node_ids), + WITH selected_nodes(user_name, node_id) AS ( + SELECT * FROM unnest(%s::text[], %s::text[]) + ) + SELECT e.source_id, e.target_id, e.edge_type + FROM {edges_table} + JOIN selected_nodes AS source_node + ON source_node.user_name = e.user_name + AND source_node.node_id = e.source_id + JOIN selected_nodes AS target_node + ON target_node.user_name = e.user_name + AND target_node.node_id = e.target_id + ORDER BY e.id + """, + (node_user_names, node_ids), ) edges = [ {"source": row[0], "target": row[1], "type": row[2]} @@ -1141,7 +2654,7 @@ def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, A return { "nodes": nodes, "edges": edges, - "total_nodes": len(nodes), + "total_nodes": total_nodes, "total_edges": len(edges), } finally: @@ -1164,10 +2677,13 @@ def import_graph(self, data: dict[str, Any], user_name: str | None = None) -> No source_id=edge["source"], target_id=edge["target"], type=edge["type"], + user_name=user_name, ) + def close(self): """Close connection pool.""" if not self._pool_closed: self._pool_closed = True self.pool.closeall() + diff --git a/tests/graph_dbs/test_postgres_400_compat.py b/tests/graph_dbs/test_postgres_400_compat.py new file mode 100644 index 000000000..bd23fd8ff --- /dev/null +++ b/tests/graph_dbs/test_postgres_400_compat.py @@ -0,0 +1,391 @@ +import json + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def postgres_db(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db.schema = "memos" + db.db_name = "memos" + db.user_name = "tenant-a" + db._pool_closed = False + db.config = SimpleNamespace( + embedding_dimension=3, + fulltext_config="simple", + vector_index_type="hnsw", + hnsw_m=16, + hnsw_ef_construction=64, + ivfflat_lists=100, + shard_count=400, + ) + + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + db._get_conn = MagicMock(return_value=connection) + db._put_conn = MagicMock() + db._partition_remainders = { + "tenant-a": 17, + "tenant-b": 17, + "kb-1": 17, + } + return db, cursor + + +def _executed_sql(cursor: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_schema_does_not_create_profile_storage(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "profile_templates" not in sql + assert "profile_instances" not in sql + + +def test_schema_creates_400_md5_memory_shards(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "PARTITION BY HASH" not in sql + assert '"Memory"' in sql + assert "embedding halfvec(" in sql + assert "memos_graph_0" in sql + assert "memos_graph_399" in sql + assert "MODULUS" not in sql + assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 401 + assert "PRIMARY KEY (user_name, id)" in sql + + +def test_schema_rejects_legacy_nonpartitioned_parent(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("memories", "r")] + + with pytest.raises(RuntimeError, match="configure a new MEMOS_SCHEMA"): + db._init_schema() + + +def test_schema_skips_partition_ddl_when_layout_is_complete(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (400,) + + db._init_schema() + + sql = _executed_sql(cursor) + assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 1 + assert "memos_graph_0" not in sql + + +def test_node_upsert_targets_composite_partition_key(postgres_db): + db, cursor = postgres_db + + db.add_node( + "memory-1", + "memory", + {"embedding": [1.0, 0.0, 0.0]}, + user_name="tenant-a", + ) + + assert "ON CONFLICT (user_name, id)" in _executed_sql(cursor) + + +def test_batch_insert_uses_one_statement_and_polardb_properties(postgres_db): + db, cursor = postgres_db + + db.add_nodes_batch( + [ + {"id": "m1", "memory": "one", "metadata": {"embedding": [1.0, 0.0, 0.0]}}, + {"id": "m2", "memory": "two", "metadata": {"embedding": [0.0, 1.0, 0.0]}}, + ], + user_name="tenant-a", + ) + + assert cursor.execute.call_count == 1 + query, params = cursor.execute.call_args.args + assert query.count("::halfvec(") == 2 + assert "ON CONFLICT (user_name, id)" in query + first_properties = json.loads(params[2]) + second_properties = json.loads(params[9]) + assert first_properties["id"] == "m1" + assert first_properties["memory"] == "one" + assert second_properties["id"] == "m2" + assert second_properties["memory"] == "two" + assert "embedding" not in first_properties + + +def test_node_upsert_persists_tenant_in_metadata(postgres_db): + db, cursor = postgres_db + + db.add_node( + "memory-1", + "memory", + {"embedding": [1.0, 0.0, 0.0]}, + user_name="tenant-a", + ) + + _, params = cursor.execute.call_args.args + properties = json.loads(params[2]) + assert properties["user_name"] == "tenant-a" + assert properties["delete_time"] == "" + assert properties["delete_record_id"] == "" + + +def test_batch_node_metadata_matches_polardb_shape(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + + node = db._parse_row( + ( + "memory-1", + "service HTTP comparison memory", + {"status": "activated"}, + now, + now, + ) + ) + + assert node["metadata"]["id"] == "memory-1" + assert node["metadata"]["memory"] == "service HTTP comparison memory" + + +def test_parse_row_restores_tenant_and_deserializes_sources(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + source = {"type": "chat", "role": "user", "content": "hello"} + + node = db._parse_row( + ( + "memory-1", + "memory", + {"sources": [json.dumps(source)]}, + now, + now, + "tenant-a", + ) + ) + + assert node["metadata"]["user_name"] == "tenant-a" + assert node["metadata"]["sources"] == [source] + + +def test_new_polardb_business_methods_are_available(): + required_methods = { + "delete_node_by_cube_memory_groups", + "get_user_names_by_cube_memory_groups", + "get_children_with_embeddings", + "get_neighbors_by_tag", + } + + missing = sorted(name for name in required_methods if not hasattr(PostgresGraphDB, name)) + assert missing == [] + + +def test_get_subgraph_matches_polardb_return_shape(postgres_db): + db, cursor = postgres_db + core = { + "id": "core", + "memory": "core memory", + "metadata": {"status": "activated"}, + } + neighbor = { + "id": "neighbor", + "memory": "neighbor memory", + "metadata": {"status": "activated"}, + } + db.get_node = MagicMock(return_value=core) + db.get_nodes = MagicMock(return_value=[neighbor]) + cursor.fetchall.side_effect = [ + [("core",), ("neighbor",)], + [("core", "neighbor", "RELATE_TO")], + ] + + result = db.get_subgraph( + "core", + depth=2, + center_status="activated", + user_name="tenant-a", + ) + + assert result == { + "core_node": core, + "neighbors": [neighbor], + "edges": [{"source": "core", "target": "neighbor", "type": "RELATE_TO"}], + } + assert "user_name = %s" in _executed_sql(cursor) + db.get_node.assert_called_once_with("core", user_name="tenant-a") + db.get_nodes.assert_called_once_with(["neighbor"], user_name="tenant-a") + + +def test_cube_group_delete_pairs_ids_with_tenants(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (2,) + + deleted = db.delete_node_by_cube_memory_groups( + [ + {"cube_id": "tenant-a", "memory_ids": ["m1"]}, + {"cube_id": "tenant-b", "memory_ids": ["m2"]}, + ] + ) + + query, params = cursor.execute.call_args.args + assert deleted == 2 + assert "unnest" in query.lower() + assert "requested.user_name" in query + assert params == (["tenant-a", "tenant-b"], ["m1", "m2"]) + + +def test_memory_owner_lookup_preserves_missing_ids(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("m1", "tenant-a")] + + owners = db.get_user_names_by_memory_ids(["m1", "missing"]) + + assert owners == {"m1": "tenant-a", "missing": None} + + +def test_export_graph_supports_latest_filters_and_pagination(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchone.return_value = (3,) + cursor.fetchall.side_effect = [ + [ + ( + "m1", + "memory", + {"memory_type": "LongTermMemory", "status": "activated"}, + now, + now, + ) + ], + [("m1", "m2", "RELATE_TO")], + ] + + result = db.export_graph( + user_name="tenant-a", + page=2, + page_size=1, + filter={"agent_id": "agent-1"}, + memory_type=["LongTermMemory"], + status=["activated"], + ) + + sql = _executed_sql(cursor) + assert result["total_nodes"] == 3 + assert result["nodes"][0]["id"] == "m1" + assert "properties->'info'->>'agent_id'" in sql + assert "LIMIT %s OFFSET %s" in sql + assert "properties->>'memory_type' = ANY(%s)" in sql + assert "JOIN selected_nodes AS source_node" in sql + assert "source_node.user_name = e.user_name" in sql + + +def test_keyword_like_matches_polardb_filters_and_parameter_name(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "memory-1", + "needle memory", + {"source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + result = db.search_by_keywords_like( + query_word="needle", + scope="LongTermMemory", + status="activated", + search_filter={"source": "conversation"}, + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + return_fields=["memory", "source"], + ) + + query, params = cursor.execute.call_args.args + assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}] + assert "user_name = ANY(%s)" in query + assert ["tenant-a", "kb-1"] in params + assert "properties->>'source'" in query + assert "properties->>'memory', '') LIKE %s" in query + + +def test_keyword_tfidf_matches_polardb_result_shape(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "memory-1", + "needle memory", + {"source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + result = db.search_by_keywords_tfidf( + ["needle", "memory"], + scope="LongTermMemory", + user_name="tenant-a", + return_fields=["memory", "source"], + ) + + assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}] + query, params = cursor.execute.call_args.args + assert params[0] == "'needle' | 'memory'" + assert "to_tsquery('jiebaqry'" in query + assert "ILIKE" not in query + + +def test_keyword_searches_require_explicit_user_name(postgres_db): + db, cursor = postgres_db + + assert db.search_by_keywords_like("needle") == [] + assert db.search_by_keywords_tfidf(["needle"]) == [] + cursor.execute.assert_not_called() + + +def test_get_all_memory_items_matches_polardb_defaults(postgres_db): + db, cursor = postgres_db + + assert db.get_all_memory_items("LongTermMemory") == [] + cursor.execute.assert_not_called() + + with pytest.raises(ValueError, match="Unsupported memory type scope"): + db.get_all_memory_items("Context", user_name="tenant-a") + + cursor.fetchall.return_value = [] + db.get_all_memory_items("LongTermMemory", user_name="tenant-a") + query, _ = cursor.execute.call_args.args + assert "properties->>'status'" not in query + + +def test_metadata_contains_list_uses_json_array_overlap(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [("m1",)] + + result = db.get_by_metadata( + [{"field": "tags", "op": "contains", "value": ["blue", "green"]}], + user_name="tenant-a", + ) + + query, params = cursor.execute.call_args.args + assert result == ["m1"] + assert "properties->'tags' @> %s::jsonb" in query + assert " OR " in query + assert json.dumps(["blue"], ensure_ascii=False) in params + assert json.dumps(["green"], ensure_ascii=False) in params diff --git a/tests/graph_dbs/test_postgres_pgvector.py b/tests/graph_dbs/test_postgres_pgvector.py new file mode 100644 index 000000000..69a590e3c --- /dev/null +++ b/tests/graph_dbs/test_postgres_pgvector.py @@ -0,0 +1,463 @@ +import hashlib +import json + +from datetime import datetime, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from memos.configs.graph_db import PostgresGraphDBConfig +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def postgres_db(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db.schema = "memos" + db.db_name = "memos" + db.user_name = "tenant-a" + db._pool_closed = False + db.config = SimpleNamespace( + embedding_dimension=3, + fulltext_config="simple", + vector_index_type="hnsw", + hnsw_m=16, + hnsw_ef_construction=64, + ivfflat_lists=100, + shard_count=400, + ) + db._partition_remainders = { + "tenant-a": 17, + "tenant-b": 17, + "kb-1": 17, + } + + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + db._get_conn = MagicMock(return_value=connection) + db._put_conn = MagicMock() + return db, cursor + + +def _executed_sql(cursor: MagicMock) -> str: + return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list) + + +def test_postgres_pool_keeps_eight_warm_connections(): + config = PostgresGraphDBConfig( + host="postgres", + user="postgres", + password="secret", + db_name="memos", + user_name="tenant-a", + ) + assert config.minconn == 8 + assert config.maxconn == 20 + + +def test_get_conn_does_not_ping_before_the_query(): + db = PostgresGraphDB.__new__(PostgresGraphDB) + db._pool_closed = False + cursor = MagicMock() + connection = MagicMock() + connection.closed = 0 + connection.cursor.return_value.__enter__.return_value = cursor + db.pool = MagicMock() + db.pool.getconn.return_value = connection + + assert db._get_conn() is connection + cursor.execute.assert_not_called() + + +def test_postgres_config_defaults_to_current_embedding_dimension(): + config = PostgresGraphDBConfig( + host="postgres", + user="memos", + password="secret", + db_name="memos", + user_name="tenant-a", + ) + + assert config.embedding_dimension == 1024 + assert config.shard_count == 400 + + +def test_init_schema_creates_tenant_edges_fulltext_and_hnsw(postgres_db): + db, cursor = postgres_db + + db._init_schema() + + sql = _executed_sql(cursor) + assert "user_name TEXT NOT NULL" in sql + assert "UNIQUE(user_name, source_id, target_id, edge_type)" in sql + assert "search_vector tsvector" in sql + assert "properties::text" in sql + assert "USING GIN(search_vector)" in sql + assert "halfvec_cosine_ops" in sql + assert "idx_memory_embedding_halfvec" in sql + assert "embedding halfvec(" in sql + assert '"Memory"' in sql + assert "USING ivfflat" not in sql + + +def test_edge_operations_are_tenant_scoped_and_support_any_direction(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = (1,) + cursor.fetchall.return_value = [("source", "target", "RELATE_TO")] + + db.add_edge("source", "target", "RELATE_TO", user_name="tenant-a") + assert db.edge_exists( + "source", + "target", + "RELATE_TO", + direction="ANY", + user_name="tenant-a", + ) + edges = db.get_edges("source", type="ANY", direction="ANY", user_name="tenant-a") + + sql = _executed_sql(cursor) + assert "user_name" in sql + assert edges == [{"from": "source", "to": "target", "type": "RELATE_TO"}] + assert any( + "tenant-a" in call.args[1] for call in cursor.execute.call_args_list if len(call.args) > 1 + ) + + db.edge_exists("source", "target", type="ANY", user_name="tenant-a") + query, params = cursor.execute.call_args.args + assert "edge_type = %s" not in query + assert params == ["tenant-a", "source", "target"] + + +def test_vector_search_applies_filters_kb_ids_and_return_fields(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.95, + "remember this", + {"status": "activated", "source": "conversation"}, + now, + now, + "tenant-a", + ) + ] + + results = db.search_by_embedding( + vector=[0.1, 0.2, 0.3], + top_k=5, + scope="LongTermMemory", + status="activated", + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + filter={"and": [{"info.agent_id": "agent-1"}, {"tags": {"contains": "blue"}}]}, + return_fields=["memory", "source", "created_at"], + ) + + query, params = cursor.execute.call_args.args + assert "user_name = ANY(%s)" in query + assert "COALESCE(properties->>'id', id)" in query + assert "properties->'info'->>'agent_id'" in query + assert "properties->'tags' @> %s::jsonb" in query + assert ["tenant-a", "kb-1"] in params + assert results[0]["memory"] == "remember this" + assert results[0]["source"] == "conversation" + assert results[0]["created_at"] == now.isoformat() + + +def test_lightweight_vector_search_returns_memory_without_extra_lookup(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.95, + "remember this", + now, + now, + "tenant-a", + '"LongTermMemory"', + '"activated"', + None, + None, + None, + None, + ) + ] + + results = db.search_by_embedding( + vector=[0.1, 0.2, 0.3], + top_k=5, + scope="LongTermMemory", + user_name="tenant-a", + light_weight_mode=True, + ) + + assert results[0]["memory"] == "remember this" + assert results[0]["memory_type"] == "LongTermMemory" + assert PostgresGraphDB.supports_lightweight_vector_search is True + assert PostgresGraphDB.prefer_lightweight_vector_search is True + + +def test_search_result_does_not_add_postgres_only_property_aliases(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + row = ( + "node-1", + 0.95, + "remember this", + {"status": "activated"}, + now, + now, + "tenant-a", + ) + + result = db._build_search_result(row, ["metadata", "properties"]) + + assert result == {"id": "node-1", "score": 0.95} + + +def test_search_result_keeps_polardb_serialized_source_shape(postgres_db): + db, _ = postgres_db + now = datetime.now(timezone.utc) + source = json.dumps({"type": "chat", "role": "user", "content": "hello"}) + row = ( + "node-1", + 0.95, + "remember this", + {"sources": [source]}, + now, + now, + "tenant-a", + ) + + result = db._build_search_result(row, ["sources"]) + + assert result["sources"] == [source] + + +def test_fulltext_search_uses_polardb_and_tsquery(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + 0.8, + "中文记忆内容", + {"status": "activated"}, + now, + now, + "tenant-a", + ) + ] + + results = db.search_by_fulltext( + query_words=["中文", "记忆"], + top_k=10, + scope="LongTermMemory", + user_name="tenant-a", + filter={"info.agent_id": "agent-1"}, + return_fields=["memory"], + ) + + query, params = cursor.execute.call_args.args + assert "to_tsquery('jiebacfg'" in query + assert "search_vector @@" in query + assert "ILIKE" not in query + assert "properties->'info'->>'agent_id'" in query + assert params[0] == "'中文' & '记忆'" + assert results == [{"id": "node-1", "score": 0.8, "memory": "中文记忆内容"}] + + +def test_get_node_without_tenant_searches_all_partitions(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchone.return_value = ( + "node-1", + "remember this", + {"status": "activated", "user_name": "tenant-b"}, + now, + now, + ) + + result = db.get_node("node-1") + + query, params = cursor.execute.call_args.args + assert "WHERE id = %s" in query + assert "user_name = %s" not in query + assert '"memos_graph_0"."Memory"' in query + assert '"memos_graph_399"."Memory"' in query + assert "FROM memos.memories" not in query + assert params == ("node-1",) + assert result["id"] == "node-1" + + +def test_get_node_with_tenant_remains_tenant_scoped(postgres_db): + db, cursor = postgres_db + cursor.fetchone.return_value = None + + assert db.get_node("node-1", user_name="tenant-b") is None + + query, params = cursor.execute.call_args.args + assert "WHERE id = %s AND user_name = %s" in query + assert params == ("node-1", "tenant-b") + + +def test_get_nodes_without_tenant_searches_all_partitions(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + assert db.get_nodes(["node-1", "node-2"]) == [] + + query, params = cursor.execute.call_args.args + assert "WHERE id = ANY(%s)" in query + assert "user_name = %s" not in query + assert '"memos_graph_0"."Memory"' in query + assert '"memos_graph_399"."Memory"' in query + assert "FROM memos.memories" not in query + assert params == (["node-1", "node-2"],) + + +def test_cube_soft_delete_recover_and_counts_are_available(postgres_db): + db, cursor = postgres_db + cursor.rowcount = 3 + cursor.fetchone.side_effect = [(3,), (0,)] + + assert db.delete_node_by_mem_cube_id("tenant-a", "delete-1") == 3 + assert db.recover_memory_by_mem_cube_id("tenant-a", "delete-1") == 3 + assert db.get_memory_count("LongTermMemory", user_name="tenant-a") == 3 + assert db.node_not_exist("UserMemory", user_name="tenant-a") == 1 + + sql = _executed_sql(cursor) + assert "delete_record_id" in sql + assert "status" in sql + assert "user_name = %s" in sql + + +def test_drop_database_is_scoped_to_current_tenant(postgres_db): + db, _cursor = postgres_db + db.clear = MagicMock() + + db.drop_database() + + db.clear.assert_called_once_with(user_name="tenant-a") + + +def test_structure_candidates_accept_tenant_and_optional_embedding(postgres_db): + db, cursor = postgres_db + now = datetime.now(timezone.utc) + cursor.fetchall.return_value = [ + ( + "node-1", + "remember this", + {"status": "activated", "memory_type": "LongTermMemory"}, + now, + now, + [0.1, 0.2, 0.3], + ) + ] + + results = db.get_structure_optimization_candidates( + "LongTermMemory", + include_embedding=True, + user_name="tenant-b", + ) + + query, params = cursor.execute.call_args.args + assert "m.embedding" in query + assert params == ("LongTermMemory", "tenant-b") + assert results[0]["metadata"]["embedding"] == [0.1, 0.2, 0.3] + + +def test_structure_candidates_only_exclude_parent_edges(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + db.get_structure_optimization_candidates( + "LongTermMemory", + user_name="tenant-a", + ) + + query, _ = cursor.execute.call_args.args + assert "e1.edge_type = 'PARENT'" in query + assert "e2.edge_type = 'PARENT'" in query + + +def test_time_filter_uses_timestamp_comparison(postgres_db): + db, cursor = postgres_db + cursor.fetchall.return_value = [] + + db.search_by_embedding( + [0.1, 0.2, 0.3], + user_name="tenant-a", + top_k=5, + filter={"created_at": {"gt": "2026-01-01T00:00:00"}}, + ) + + query, _params = cursor.execute.call_args.args + assert "::timestamp >" in query + assert "timestamptz" not in query + + +def test_single_tenant_vector_search_reads_one_hash_partition(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {"tenant-a": 17} + cursor.fetchall.return_value = [] + + db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5) + + query, params = cursor.execute.call_args.args + assert 'FROM "memos_graph_17"."Memory"' in query + assert "FROM memos.memories" not in query + assert query.count("%s::halfvec(3)") == 1 + assert "AS MATERIALIZED" in query + assert "ORDER BY embedding <=> (SELECT query_vector FROM q)" in query + materialized_and_score, outer_filter = query.split("FROM t", 1) + materialized_body = materialized_and_score.split("AS MATERIALIZED", 1)[1] + assert "< 0.9" not in materialized_body + assert "scope_distance < 0.9" in outer_filter + assert "((1 - scope_distance) + 1) / 2.0" in materialized_and_score + assert "embedding::halfvec" not in query + assert "user_name = %s" in query + assert "user_name = ANY(%s)" not in query + assert params[0] == [0.1, 0.2, 0.3] + assert params[1] == "tenant-a" + assert params[-1] == 5 + assert any( + "hnsw.iterative_scan" in str(call.args[0]) for call in cursor.execute.call_args_list + ) + + +def test_cross_shard_vector_search_reads_only_those_partitions(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {"tenant-a": 17, "kb-1": 19} + cursor.fetchall.return_value = [] + + db.search_by_embedding( + [0.1, 0.2, 0.3], + user_name="tenant-a", + knowledgebase_ids=["kb-1"], + top_k=5, + ) + + query, _params = cursor.execute.call_args.args + assert '"memos_graph_17"."Memory"' in query + assert '"memos_graph_19"."Memory"' in query + assert "UNION ALL" in query + assert "FROM memos.memories" not in query + + +def test_partition_remainder_uses_polardb_md5(postgres_db): + db, cursor = postgres_db + db._partition_remainders = {} + cursor.fetchall.return_value = [] + shard = int(hashlib.md5(b"tenant-a").hexdigest(), 16) % 400 + + db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5) + + query = cursor.execute.call_args.args[0] + assert f'"memos_graph_{shard}"."Memory"' in query + assert "satisfies_hash_partition" not in _executed_sql(cursor) + assert db._partition_remainder("tenant-a") == shard From 8a0a6fb5e5391b0771e98b3bdef26885f5471f64 Mon Sep 17 00:00:00 2001 From: dingliang <2650876010@qq.com> Date: Wed, 21 Oct 2026 19:59:26 +0800 Subject: [PATCH 2/3] style: format PostgreSQL graph files for Ruff The upstream check stops on ruff format before the tests run. Co-authored-by: Cursor --- src/memos/graph_dbs/postgres.py | 22 +++++----------------- tests/graph_dbs/test_postgres_pgvector.py | 4 +--- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/src/memos/graph_dbs/postgres.py b/src/memos/graph_dbs/postgres.py index 0a5460839..2820a8b54 100644 --- a/src/memos/graph_dbs/postgres.py +++ b/src/memos/graph_dbs/postgres.py @@ -199,9 +199,7 @@ def _init_schema(self): (rf"^{re.escape(self.schema)}_graph_[0-9]+$",), ) count_row = cur.fetchone() - memory_tables = ( - count_row[0] if count_row and isinstance(count_row[0], int) else -1 - ) + memory_tables = count_row[0] if count_row and isinstance(count_row[0], int) else -1 if memory_tables != shard_count: hnsw_m = int(getattr(self.config, "hnsw_m", 16)) ef_construction = int(getattr(self.config, "hnsw_ef_construction", 64)) @@ -388,9 +386,7 @@ def _prepare_node_for_insert( ] embedding = fields.pop("embedding", None) if not isinstance(embedding, list) or not embedding: - embedding = [ - round(random.uniform(-0.2, 0.2), 6) for _ in range(self._vector_dim()) - ] + embedding = [round(random.uniform(-0.2, 0.2), 6) for _ in range(self._vector_dim())] else: embedding = [float(item) for item in embedding] properties = { @@ -448,9 +444,7 @@ def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None: return memories = self._physical_table("memories", user_name) dim = self._vector_dim() - values_sql = ",".join( - [f"(%s, %s, %s, %s::halfvec({dim}), %s, %s, %s)"] * len(rows) - ) + values_sql = ",".join([f"(%s, %s, %s, %s::halfvec({dim}), %s, %s, %s)"] * len(rows)) params: list[Any] = [value for row in rows for value in row] conn = self._get_conn() try: @@ -1546,9 +1540,7 @@ def _tenant_relation( alias = table return f"{relation} {alias}" if alias else relation - def _physical_tables( - self, table: str, user_names: list[str] | None, conn=None - ) -> list[str]: + def _physical_tables(self, table: str, user_names: list[str] | None, conn=None) -> list[str]: """Real tables a write can target. A union subquery cannot be updated.""" names = [name for name in (user_names or []) if name] if not names: @@ -1777,9 +1769,7 @@ def search_by_embedding( try: self._ensure_hnsw_search_settings(conn) with conn.cursor() as cur: - projected = ( - self._projected_field_exprs(return_fields) if light_weight_mode else [] - ) + projected = self._projected_field_exprs(return_fields) if light_weight_mode else [] if light_weight_mode: inner_columns = "memory, created_at, updated_at, user_name" extra_inner = "".join( @@ -2680,10 +2670,8 @@ def import_graph(self, data: dict[str, Any], user_name: str | None = None) -> No user_name=user_name, ) - def close(self): """Close connection pool.""" if not self._pool_closed: self._pool_closed = True self.pool.closeall() - diff --git a/tests/graph_dbs/test_postgres_pgvector.py b/tests/graph_dbs/test_postgres_pgvector.py index 69a590e3c..d404cf230 100644 --- a/tests/graph_dbs/test_postgres_pgvector.py +++ b/tests/graph_dbs/test_postgres_pgvector.py @@ -425,9 +425,7 @@ def test_single_tenant_vector_search_reads_one_hash_partition(postgres_db): assert params[0] == [0.1, 0.2, 0.3] assert params[1] == "tenant-a" assert params[-1] == 5 - assert any( - "hnsw.iterative_scan" in str(call.args[0]) for call in cursor.execute.call_args_list - ) + assert any("hnsw.iterative_scan" in str(call.args[0]) for call in cursor.execute.call_args_list) def test_cross_shard_vector_search_reads_only_those_partitions(postgres_db): From c5d3ace1f7e65f943b262cd406a3313039b28d18 Mon Sep 17 00:00:00 2001 From: dingliang <2650876010@qq.com> Date: Wed, 21 Oct 2026 20:34:48 +0800 Subject: [PATCH 3/3] ci: retrigger pull request checks The Windows Python 3.11 dependency install failed once and needs another run. Co-authored-by: Cursor