(
+ `/api/v1/import?${query}`,
form,
);
+ if (dryRun) { setPreview(r); setResult(null); } else { setResult(r); setPreview(null); }
setStatus({
- kind: "ok",
- text: `Imported ${r.imported} / skipped ${r.skipped}`,
+ kind: r.invalid || r.failed ? "error" : "ok",
+ text: t(dryRun ? "import.preview.done" : "import.result.done", {
+ imported: r.imported, duplicates: r.duplicates, invalid: r.invalid, failed: r.failed,
+ visible: r.visible, hidden: r.hidden,
+ }),
});
} catch (err) {
setStatus({ kind: "error", text: (err as Error).message });
@@ -206,6 +219,22 @@ function ImportCard() {
+
+
+ {ownership === "target" &&
}
+
{t("import.visibility.hint")}
+ {file &&
+ {file.name}
+
+
+
}
+
{status && (
)}
- {status?.kind === "ok" &&
}
+ {(preview ?? result)?.issues?.length ?
+ {(preview ?? result)!.issues.map(issue => -
+ {issue.kind}[{issue.index + 1}]{issue.field ? ` · ${issue.field}` : ""}: {issue.message} ({issue.code})
+
)}
+
: null}
+ {result && result.imported > 0 &&
}
);
}
diff --git a/apps/memos-local-plugin/viewer/src/views/SettingsView.tsx b/apps/memos-local-plugin/viewer/src/views/SettingsView.tsx
index d3e62fdd5..43d2efb1a 100644
--- a/apps/memos-local-plugin/viewer/src/views/SettingsView.tsx
+++ b/apps/memos-local-plugin/viewer/src/views/SettingsView.tsx
@@ -39,6 +39,13 @@ interface ProviderBlock {
temperature?: number;
maxInputTokens?: number;
batchSize?: number;
+ local?: {
+ remoteHost?: string;
+ modelPath?: string;
+ cacheDir?: string;
+ offline?: boolean;
+ loadTimeoutMs?: number;
+ };
}
interface AlgorithmBlock {
@@ -408,6 +415,7 @@ function ModelCard({
provider: block.provider,
endpoint: block.endpoint,
model: block.model,
+ local: block.local,
apiKey: sanitizeApiKey(block.apiKey),
});
setResult(r);
@@ -472,6 +480,7 @@ function ModelCard({
{localHint}
)}
+ {isLocal && testing && {t("settings.embedding.loading")}
}
onPatch({ model: (e.target as HTMLInputElement).value })}
/>
-
+ {!isLocal &&
onPatch({ endpoint: (e.target as HTMLInputElement).value })}
/>
-
-
+ }
+ {!isLocal &&
onPatch({ apiKey: (e.target as HTMLInputElement).value })}
/>
-
+ }
+ {isLocal && ([
+ ["remoteHost", "settings.embedding.remoteHost"],
+ ["modelPath", "settings.embedding.modelPath"],
+ ["cacheDir", "settings.embedding.cacheDir"],
+ ] as const).map(([key, label]) =>
+ onPatch({ local: { ...block.local, [key]: (event.target as HTMLInputElement).value } })} />
+ )}
+ {isLocal &&
+ onPatch({ local: { ...block.local, offline: (event.target as HTMLInputElement).checked } })} />
+ }
+ {isLocal &&
+ onPatch({ local: { ...block.local, loadTimeoutMs: Math.min(3600, Math.max(1, Number((event.target as HTMLInputElement).value) || 120)) * 1000 } })} />
+ }
{type === "embedding" && (
dict[str, Any]:
def get_postgres_config(user_id: str | None = None) -> dict[str, Any]:
"""Get PostgreSQL + pgvector configuration for MemOS graph storage.
- Uses standard PostgreSQL with pgvector extension.
- Schema: memos.memories, memos.edges
+ Uses standard PostgreSQL with pgvector. Shard count, index, and
+ full-text configuration keep the backend defaults unless overridden.
"""
user_name = os.getenv("MEMOS_USER_NAME", "default")
if user_id:
@@ -922,7 +922,7 @@ def get_postgres_config(user_id: str | None = None) -> dict[str, Any]:
"schema_name": os.getenv("MEMOS_SCHEMA", "memos"),
"user_name": user_name,
"use_multi_db": False,
- "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "384")),
+ "embedding_dimension": int(os.getenv("EMBEDDING_DIMENSION", "1024")),
"maxconn": int(os.getenv("POSTGRES_MAX_CONN", "20")),
}
diff --git a/src/memos/api/handlers/chat_handler.py b/src/memos/api/handlers/chat_handler.py
index 58a96cd75..31acec6d0 100644
--- a/src/memos/api/handlers/chat_handler.py
+++ b/src/memos/api/handlers/chat_handler.py
@@ -482,12 +482,8 @@ def generate_chat_response() -> Generator[str, None, None]:
yield f"data: {json.dumps({'type': 'status', 'data': '1'})}\n\n"
- # Extract memories from search results (first search)
- memories_list = []
- if search_response.data and search_response.data.get("text_mem"):
- text_mem_results = search_response.data["text_mem"]
- if text_mem_results and text_mem_results[0].get("memories"):
- memories_list = text_mem_results[0]["memories"]
+ # Extract memories from every searched cube (first search)
+ memories_list = self._memories_from_cube_results(search_response.data)
# Filter memories by threshold
filtered_memories = self._filter_memories_by_threshold(memories_list)[:5]
@@ -577,12 +573,8 @@ def generate_chat_response() -> Generator[str, None, None]:
project_id=chat_req.project_id,
)
- # Extract memories from search results (second search)
- memories_list = []
- if search_response.data and search_response.data.get("text_mem"):
- text_mem_results = search_response.data["text_mem"]
- if text_mem_results and text_mem_results[0].get("memories"):
- memories_list = text_mem_results[0]["memories"]
+ # Extract memories from every searched cube (second search)
+ memories_list = self._memories_from_cube_results(search_response.data)
# Filter memories by threshold, min_num is the min number of memories for playground
second_filtered_memories = self._filter_memories_by_threshold(
@@ -1161,6 +1153,14 @@ def _format_mem_block(
return "\n".join(lines_o), "\n".join(lines_p)
+ @staticmethod
+ def _memories_from_cube_results(search_data: dict[str, Any] | None) -> list[Any]:
+ """Collect textual memories from every cube in a search response."""
+ memories_list: list[Any] = []
+ for cube_result in (search_data or {}).get("text_mem") or []:
+ memories_list.extend(cube_result.get("memories") or [])
+ return memories_list
+
def _filter_memories_by_threshold(
self,
memories: list,
diff --git a/src/memos/api/middleware/auth.py b/src/memos/api/middleware/auth.py
index 15b217651..6d812337b 100644
--- a/src/memos/api/middleware/auth.py
+++ b/src/memos/api/middleware/auth.py
@@ -17,6 +17,13 @@
import memos.log
+# Canonical format check; re-exported here so the middleware shares the
+# same logic as ``src/memos/api/utils/api_keys.py``. Keeping one source of
+# truth is what fixes issue #2427 (previously the middleware copy and the
+# utils copy could drift independently).
+from memos.api.utils.api_keys import validate_key_format as validate_key_format_canonical
+
+
logger = memos.log.get_logger(__name__)
# API key header configuration
@@ -63,17 +70,16 @@ def hash_api_key(key: str) -> str:
def validate_key_format(key: str) -> bool:
- """Validate API key format: krlk_<64-hex>."""
- if not key or not key.startswith("krlk_"):
- return False
- hex_part = key[5:] # Remove 'krlk_' prefix
- if len(hex_part) != 64:
- return False
- try:
- int(hex_part, 16)
- return True
- except ValueError:
- return False
+ """Validate API key format: krlk_<64-hex>.
+
+ Re-exported from ``memos.api.utils.api_keys.validate_key_format`` for
+ backwards compatibility with callers that imported this symbol from
+ the middleware path. Issue #2427: the previous copy used ``int(...)``
+ which accepted non-canonical hex (leading signs, ``0x`` prefix, ``_``
+ separators, whitespace) — see the canonical implementation for the
+ fix.
+ """
+ return validate_key_format_canonical(key)
def get_key_prefix(key: str) -> str:
diff --git a/src/memos/api/middleware/rate_limit.py b/src/memos/api/middleware/rate_limit.py
index c547378ca..72808a13d 100644
--- a/src/memos/api/middleware/rate_limit.py
+++ b/src/memos/api/middleware/rate_limit.py
@@ -57,8 +57,13 @@ def _get_client_key(request: Request) -> str:
Uses API key if available, otherwise falls back to IP.
"""
- # Try to get API key from header
+ # Try to get API key from header. Real clients send a Bearer scheme
+ # per RFC 6750 (``Authorization: Bearer krlk_``); we strip the
+ # scheme before the prefix check so keyed requests actually land on
+ # a per-key bucket (issue #2428).
auth_header = request.headers.get("Authorization", "")
+ if auth_header.lower().startswith("bearer "):
+ auth_header = auth_header[len("bearer ") :].lstrip()
if auth_header.startswith("krlk_"):
# Use first 20 chars of key as identifier
return f"ratelimit:key:{auth_header[:20]}"
diff --git a/src/memos/api/utils/api_keys.py b/src/memos/api/utils/api_keys.py
index 29b493fd0..123420513 100644
--- a/src/memos/api/utils/api_keys.py
+++ b/src/memos/api/utils/api_keys.py
@@ -5,6 +5,7 @@
"""
import hashlib
+import re
import secrets
from dataclasses import dataclass
@@ -45,11 +46,19 @@ def hash_key(key: str) -> str:
return hashlib.sha256(key.encode()).hexdigest()
+# Match the only form ``generate_api_key()`` produces: ``krlk_`` followed by
+# exactly 64 lowercase hex characters. Using ``int(hex_part, 16)`` would also
+# accept leading signs, ``0x`` prefixes, ``_`` separators, and surrounding
+# whitespace, none of which a real key ever carries (issue #2427).
+_KEY_FORMAT_HEX_RE = re.compile(r"[0-9a-f]{64}")
+
+
def validate_key_format(key: str) -> bool:
"""
Validate API key format.
- Valid format: krlk_<64-hex-chars>
+ Valid format: ``krlk_<64-hex-chars>`` where the hex part is exactly 64
+ characters from ``[0-9a-f]`` — the only form ``generate_api_key()`` produces.
"""
if not key or not isinstance(key, str):
return False
@@ -58,14 +67,7 @@ def validate_key_format(key: str) -> bool:
return False
hex_part = key[5:]
- if len(hex_part) != 64:
- return False
-
- try:
- int(hex_part, 16)
- return True
- except ValueError:
- return False
+ return _KEY_FORMAT_HEX_RE.fullmatch(hex_part) is not None
def generate_master_key() -> tuple[str, str]:
diff --git a/src/memos/configs/graph_db.py b/src/memos/configs/graph_db.py
index 98de09812..984c95060 100644
--- a/src/memos/configs/graph_db.py
+++ b/src/memos/configs/graph_db.py
@@ -1,4 +1,4 @@
-from typing import Any, ClassVar
+from typing import Any, ClassVar, Literal
from pydantic import BaseModel, Field, field_validator, model_validator
@@ -195,8 +195,7 @@ class PostgresGraphDBConfig(BaseConfig):
Does NOT require Apache AGE or other graph extensions.
Schema:
- - memos_memories: Main table for memory nodes (id, memory, properties JSONB, embedding vector)
- - memos_edges: Edge table for relationships (source_id, target_id, type)
+ - {schema}_graph_{n}."Memory" and "Edges", one physical shard per tenant
Example:
---
@@ -223,13 +222,33 @@ class PostgresGraphDBConfig(BaseConfig):
default=False,
description="If False: use single database with logical isolation by user_name",
)
- embedding_dimension: int = Field(
- default=768, description="Dimension of vector embedding (768 for all-mpnet-base-v2)"
+ embedding_dimension: int = Field(default=1024, description="Dimension of vector embedding")
+ shard_count: int = Field(
+ default=400,
+ ge=1,
+ le=4096,
+ description="Number of MD5 tenant shards",
)
maxconn: int = Field(
default=20,
description="Maximum number of connections in the connection pool",
)
+ minconn: int = Field(
+ default=8,
+ ge=1,
+ description="Warm connections kept in the pool",
+ )
+ vector_index_type: Literal["hnsw", "ivfflat"] = Field(
+ default="hnsw",
+ description="pgvector approximate index type",
+ )
+ hnsw_m: int = Field(default=16, ge=2, le=100)
+ hnsw_ef_construction: int = Field(default=64, ge=4, le=1000)
+ ivfflat_lists: int = Field(default=100, ge=1)
+ fulltext_config: str = Field(
+ default="jiebacfg",
+ description="PostgreSQL text search configuration used to build search_vector",
+ )
@model_validator(mode="after")
def validate_config(self):
@@ -238,6 +257,10 @@ def validate_config(self):
raise ValueError("`db_name` must be provided")
if not self.use_multi_db and not self.user_name:
raise ValueError("In single-database mode, `user_name` must be provided")
+ if self.minconn > self.maxconn:
+ raise ValueError(f"`minconn` ({self.minconn}) must be <= `maxconn` ({self.maxconn})")
+ if not self.fulltext_config.replace(".", "_").isidentifier():
+ raise ValueError("`fulltext_config` must be a valid PostgreSQL configuration name")
return self
diff --git a/src/memos/graph_dbs/polardb.py b/src/memos/graph_dbs/polardb.py
index bf74fbb8b..8bd9fe76d 100644
--- a/src/memos/graph_dbs/polardb.py
+++ b/src/memos/graph_dbs/polardb.py
@@ -20,6 +20,19 @@
logger = get_logger(__name__)
+_LIGHTWEIGHT_IDENTIFIER_FIELDS = frozenset(
+ {
+ "user_id",
+ "user_name",
+ "session_id",
+ "mem_cube_id",
+ "dialog_id",
+ "working_binding",
+ "project_id",
+ "manager_user_id",
+ }
+)
+
def _build_lightweight_return_columns(return_fields: list[str]) -> str:
columns = []
@@ -33,9 +46,20 @@ def _build_lightweight_return_columns(return_fields: list[str]) -> str:
return "".join(f",\n {column}" for column in columns)
-def _decode_lightweight_return_value(value: Any) -> Any:
+def _decode_lightweight_return_value(value: Any, field: str | None = None) -> Any:
if hasattr(value, "value"):
value = value.value
+ # Drivers may return agtype identifiers as already-decoded scalar strings.
+ # Parsing a numeric ID again would change its type.
+ if field in _LIGHTWEIGHT_IDENTIFIER_FIELDS:
+ if isinstance(value, str):
+ try:
+ decoded = json.loads(value)
+ except (json.JSONDecodeError, TypeError):
+ return value
+ return decoded if decoded is None or isinstance(decoded, str) else value
+ if isinstance(value, int) and not isinstance(value, bool):
+ return str(value)
if isinstance(value, str):
try:
return json.loads(value)
@@ -1990,7 +2014,7 @@ def search_by_embedding(
item = {"id": id_val, "score": score_val}
if light_weight_mode:
for offset, field in enumerate(validated_return_fields, start=5):
- item[field] = _decode_lightweight_return_value(row[offset])
+ item[field] = _decode_lightweight_return_value(row[offset], field=field)
elif validated_return_fields:
properties = row[1] # properties column
item.update(
diff --git a/src/memos/graph_dbs/postgres.py b/src/memos/graph_dbs/postgres.py
index 594f7e695..2820a8b54 100644
--- a/src/memos/graph_dbs/postgres.py
+++ b/src/memos/graph_dbs/postgres.py
@@ -5,16 +5,18 @@
No Apache AGE or other graph extensions required.
Tables:
-- {schema}.memories: Memory nodes with JSONB properties and vector embeddings
-- {schema}.edges: Relationships between memory nodes
+- {schema}_graph_{n}."Memory": one tenant shard of memory nodes
+- {schema}_graph_{n}."Edges": relationships for that same shard
"""
+import hashlib
import json
+import random
import re
import time
from contextlib import suppress
-from datetime import datetime
+from datetime import datetime, timezone
from typing import Any, Literal
from memos.configs.graph_db import PostgresGraphDBConfig
@@ -28,7 +30,7 @@
def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
"""Ensure metadata has proper datetime fields and normalized types."""
- now = datetime.utcnow().isoformat()
+ now = datetime.now(timezone.utc).isoformat()
metadata.setdefault("created_at", now)
metadata.setdefault("updated_at", now)
@@ -43,6 +45,9 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
class PostgresGraphDB(BaseGraphDB):
"""PostgreSQL + pgvector implementation of a graph memory store."""
+ supports_lightweight_vector_search = True
+ prefer_lightweight_vector_search = True
+
@require_python_package(
import_name="psycopg2",
install_command="pip install psycopg2-binary",
@@ -50,26 +55,29 @@ class PostgresGraphDB(BaseGraphDB):
)
def __init__(self, config: PostgresGraphDBConfig):
"""Initialize PostgreSQL connection pool."""
- import psycopg2
import psycopg2.pool
self.config = config
+ if not self._is_safe_field_name(config.schema_name):
+ raise ValueError(f"Invalid PostgreSQL schema name: {config.schema_name}")
self.schema = config.schema_name
+ self.db_name = config.db_name
self.user_name = config.user_name
self._pool_closed = False
+ self._partition_remainders: dict[str, int] = {}
logger.info(f"Connecting to PostgreSQL: {config.host}:{config.port}/{config.db_name}")
# Create connection pool
self.pool = psycopg2.pool.ThreadedConnectionPool(
- minconn=2,
+ minconn=config.minconn,
maxconn=config.maxconn,
host=config.host,
port=config.port,
user=config.user,
password=config.password,
dbname=config.db_name,
- connect_timeout=30,
+ connect_timeout=10,
keepalives_idle=30,
keepalives_interval=10,
keepalives_count=5,
@@ -78,8 +86,39 @@ def __init__(self, config: PostgresGraphDBConfig):
# Initialize schema and tables
self._init_schema()
+ def warm_up_search_connections_by_full(self, user_name: str | None = None) -> None:
+ """Warm one pooled PostgreSQL connection; tenant routing needs no fan-out."""
+ conn = self._get_conn()
+ self._put_conn(conn)
+
+ def get_memory_graph_table_name(self, user_name: str | None) -> str:
+ """Return the one partition table for this tenant, matching PolarDB shard routing."""
+ target = user_name or self.user_name
+ return self._physical_table("memories", target)
+
+ def create_extension(self) -> None:
+ self._init_schema()
+
+ def create_graph(self) -> None:
+ self._init_schema()
+
+ def create_edge(self) -> None:
+ self._init_schema()
+
+ def create_index(
+ self,
+ label: str = "Memory",
+ vector_property: str = "embedding",
+ dimensions: int = 1024,
+ index_name: str = "memory_vector_index",
+ ) -> None:
+ self._init_schema()
+
+ def format_param_value(self, value: str | None) -> str:
+ return "null" if value is None else json.dumps(value, ensure_ascii=False)
+
def _get_conn(self):
- """Get connection from pool with health check."""
+ """Get a pooled connection without an extra round trip."""
if self._pool_closed:
raise RuntimeError("Connection pool is closed")
@@ -91,9 +130,6 @@ def _get_conn(self):
self.pool.putconn(conn, close=True)
continue
conn.autocommit = True
- # Health check
- with conn.cursor() as cur:
- cur.execute("SELECT 1")
return conn
except Exception as e:
if conn:
@@ -114,66 +150,145 @@ def _put_conn(self, conn):
conn.close()
def _init_schema(self):
- """Create schema and tables if they don't exist."""
+ """Create PolarDB-style MD5 shards with a halfvec Memory table in each."""
+ fulltext_config = getattr(self.config, "fulltext_config", "simple")
+ if not re.match(r"^[A-Za-z_][A-Za-z0-9_.]*$", fulltext_config):
+ raise ValueError(f"Invalid PostgreSQL text search config: {fulltext_config}")
+ vector_index_type = getattr(self.config, "vector_index_type", "hnsw")
+ shard_count = int(getattr(self.config, "shard_count", 400))
+ if shard_count < 1 or shard_count > 4096:
+ raise ValueError("PostgreSQL shard_count must be between 1 and 4096")
+
conn = self._get_conn()
try:
with conn.cursor() as cur:
- # Create schema
cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")
-
- # Enable pgvector
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
- # Create memories table
- dim = self.config.embedding_dimension
- cur.execute(f"""
- CREATE TABLE IF NOT EXISTS {self.schema}.memories (
- id TEXT PRIMARY KEY,
- memory TEXT NOT NULL DEFAULT '',
- properties JSONB NOT NULL DEFAULT '{{}}',
- embedding vector({dim}),
- user_name TEXT,
- created_at TIMESTAMPTZ DEFAULT NOW(),
- updated_at TIMESTAMPTZ DEFAULT NOW()
- )
- """)
-
- # Create edges table
- cur.execute(f"""
- CREATE TABLE IF NOT EXISTS {self.schema}.edges (
- id SERIAL PRIMARY KEY,
- source_id TEXT NOT NULL,
- target_id TEXT NOT NULL,
- edge_type TEXT NOT NULL,
- created_at TIMESTAMPTZ DEFAULT NOW(),
- UNIQUE(source_id, target_id, edge_type)
+ cur.execute(
+ """
+ SELECT c.relname, c.relkind
+ FROM pg_class AS c
+ JOIN pg_namespace AS n ON n.oid = c.relnamespace
+ WHERE n.nspname = %s
+ AND c.relname = ANY(%s)
+ """,
+ (
+ self.schema,
+ ["memories", "edges"],
+ ),
+ )
+ legacy_rows = cur.fetchall()
+ if isinstance(legacy_rows, list) and legacy_rows:
+ raise RuntimeError(
+ f"{self.schema} still has the old memories layout; "
+ "configure a new MEMOS_SCHEMA for the MD5 shard layout "
+ "or migrate the existing data"
)
- """)
-
- # Create indexes
- cur.execute(f"""
- CREATE INDEX IF NOT EXISTS idx_memories_user
- ON {self.schema}.memories(user_name)
- """)
- cur.execute(f"""
- CREATE INDEX IF NOT EXISTS idx_memories_props
- ON {self.schema}.memories USING GIN(properties)
- """)
- cur.execute(f"""
- CREATE INDEX IF NOT EXISTS idx_memories_embedding
- ON {self.schema}.memories USING ivfflat(embedding vector_cosine_ops)
- WITH (lists = 100)
- """)
- cur.execute(f"""
- CREATE INDEX IF NOT EXISTS idx_edges_source
- ON {self.schema}.edges(source_id)
- """)
- cur.execute(f"""
- CREATE INDEX IF NOT EXISTS idx_edges_target
- ON {self.schema}.edges(target_id)
- """)
-
- logger.info(f"Schema {self.schema} initialized successfully")
+
+ dim = int(self.config.embedding_dimension)
+ cur.execute(
+ """
+ SELECT COUNT(*)
+ FROM pg_class AS c
+ JOIN pg_namespace AS n ON n.oid = c.relnamespace
+ WHERE c.relname = 'Memory'
+ AND c.relkind = 'r'
+ AND n.nspname ~ %s
+ """,
+ (rf"^{re.escape(self.schema)}_graph_[0-9]+$",),
+ )
+ count_row = cur.fetchone()
+ memory_tables = count_row[0] if count_row and isinstance(count_row[0], int) else -1
+ if memory_tables != shard_count:
+ hnsw_m = int(getattr(self.config, "hnsw_m", 16))
+ ef_construction = int(getattr(self.config, "hnsw_ef_construction", 64))
+ lists = int(getattr(self.config, "ivfflat_lists", 100))
+ statements: list[str] = []
+ for remainder in range(shard_count):
+ shard_schema = f"{self.schema}_graph_{remainder}"
+ memory = f'"{shard_schema}"."Memory"'
+ edges = f'"{shard_schema}"."Edges"'
+ if vector_index_type == "ivfflat":
+ vector_index = (
+ f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec "
+ f"ON {memory} USING ivfflat (embedding halfvec_cosine_ops) "
+ f"WITH (lists = {lists})"
+ )
+ else:
+ vector_index = (
+ f"CREATE INDEX IF NOT EXISTS idx_memory_embedding_halfvec "
+ f"ON {memory} USING hnsw (embedding halfvec_cosine_ops) "
+ f"WITH (m = {hnsw_m}, ef_construction = {ef_construction})"
+ )
+ statements.extend(
+ [
+ f'CREATE SCHEMA IF NOT EXISTS "{shard_schema}"',
+ f"""
+ CREATE TABLE IF NOT EXISTS {memory} (
+ id TEXT NOT NULL,
+ memory TEXT NOT NULL DEFAULT '',
+ properties JSONB NOT NULL DEFAULT '{{}}',
+ embedding halfvec({dim}),
+ user_name TEXT NOT NULL,
+ search_vector tsvector GENERATED ALWAYS AS (
+ to_tsvector(
+ '{fulltext_config}',
+ coalesce(memory, '') || ' ' || coalesce(properties::text, '')
+ )
+ ) STORED,
+ deleted_at TIMESTAMPTZ,
+ created_at TIMESTAMPTZ DEFAULT NOW(),
+ updated_at TIMESTAMPTZ DEFAULT NOW(),
+ PRIMARY KEY (user_name, id)
+ )
+ """,
+ f"""
+ CREATE TABLE IF NOT EXISTS {edges} (
+ id BIGSERIAL NOT NULL,
+ user_name TEXT NOT NULL,
+ source_id TEXT NOT NULL,
+ target_id TEXT NOT NULL,
+ edge_type TEXT NOT NULL,
+ created_at TIMESTAMPTZ DEFAULT NOW(),
+ PRIMARY KEY (user_name, id),
+ UNIQUE(user_name, source_id, target_id, edge_type)
+ )
+ """,
+ (
+ f"CREATE INDEX IF NOT EXISTS idx_memory_user "
+ f"ON {memory}(user_name)"
+ ),
+ (
+ f"CREATE INDEX IF NOT EXISTS idx_memory_props "
+ f"ON {memory} USING GIN(properties)"
+ ),
+ (
+ f"CREATE INDEX IF NOT EXISTS idx_memory_search_vector "
+ f"ON {memory} USING GIN(search_vector)"
+ ),
+ vector_index,
+ (
+ f"CREATE INDEX IF NOT EXISTS idx_edges_source "
+ f"ON {edges}(user_name, source_id)"
+ ),
+ (
+ f"CREATE INDEX IF NOT EXISTS idx_edges_target "
+ f"ON {edges}(user_name, target_id)"
+ ),
+ (
+ "CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_tenant_unique "
+ f"ON {edges}(user_name, source_id, target_id, edge_type)"
+ ),
+ ]
+ )
+ cur.execute(";\n".join(statements))
+
+ logger.info(
+ "Schema %s initialized successfully with %d MD5 shards",
+ self.schema,
+ shard_count,
+ )
except Exception as e:
logger.error(f"Failed to init schema: {e}")
raise
@@ -197,6 +312,8 @@ def remove_oldest_memory(
"""
user_name = user_name or self.user_name
keep_latest = int(keep_latest)
+ memories = self._physical_table("memories", user_name)
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
@@ -206,7 +323,7 @@ def remove_oldest_memory(
f"""
WITH ranked AS (
SELECT id, ROW_NUMBER() OVER (ORDER BY updated_at DESC) as rn
- FROM {self.schema}.memories
+ FROM {memories}
WHERE user_name = %s
AND properties->>'memory_type' = %s
)
@@ -221,101 +338,140 @@ def remove_oldest_memory(
# Delete edges first
cur.execute(
f"""
- DELETE FROM {self.schema}.edges
- WHERE source_id = ANY(%s) OR target_id = ANY(%s)
+ DELETE FROM {edges}
+ WHERE user_name = %s
+ AND (source_id = ANY(%s) OR target_id = ANY(%s))
""",
- (ids_to_delete, ids_to_delete),
+ (user_name, ids_to_delete, ids_to_delete),
)
# Delete nodes
cur.execute(
f"""
- DELETE FROM {self.schema}.memories
- WHERE id = ANY(%s)
+ DELETE FROM {memories}
+ WHERE id = ANY(%s) AND user_name = %s
""",
- (ids_to_delete,),
+ (ids_to_delete, user_name),
)
logger.info(
- f"Removed {len(ids_to_delete)} oldest {memory_type} memories for user {user_name}"
+ "Removed %d oldest %s memories for user %s",
+ len(ids_to_delete),
+ memory_type,
+ user_name,
)
finally:
self._put_conn(conn)
- def add_node(
- self, id: str, memory: str, metadata: dict[str, Any], user_name: str | None = None
- ) -> None:
- """Add a memory node."""
- user_name = user_name or self.user_name
- metadata = _prepare_node_metadata(metadata.copy())
-
- # Extract embedding
- embedding = metadata.pop("embedding", None)
- created_at = metadata.pop("created_at", datetime.utcnow().isoformat())
- updated_at = metadata.pop("updated_at", datetime.utcnow().isoformat())
+ def _prepare_node_for_insert(
+ self,
+ node_id: str,
+ memory: str,
+ metadata: dict[str, Any] | None,
+ user_name: str,
+ ) -> tuple[Any, ...]:
+ """Build one write row with the same property shape PolarDB stores."""
+ fields = _prepare_node_metadata(dict(metadata or {}))
+ fields["user_name"] = user_name
+ fields.setdefault("delete_time", "")
+ fields.setdefault("delete_record_id", "")
+ now = datetime.now(timezone.utc).isoformat()
+ created_at = fields.pop("created_at", now)
+ updated_at = fields.pop("updated_at", now)
+ for field_name in ("sources", "usage"):
+ value = fields.get(field_name)
+ if isinstance(value, list):
+ fields[field_name] = [
+ item if isinstance(item, str) else json.dumps(item) for item in value
+ ]
+ embedding = fields.pop("embedding", None)
+ if not isinstance(embedding, list) or not embedding:
+ embedding = [round(random.uniform(-0.2, 0.2), 6) for _ in range(self._vector_dim())]
+ else:
+ embedding = [float(item) for item in embedding]
+ properties = {
+ "id": node_id,
+ "memory": memory,
+ "created_at": created_at,
+ "updated_at": updated_at,
+ "delete_time": fields.get("delete_time", ""),
+ "delete_record_id": fields.get("delete_record_id", ""),
+ **fields,
+ }
+ return (
+ node_id,
+ memory,
+ json.dumps(properties, ensure_ascii=False),
+ embedding,
+ user_name,
+ created_at,
+ updated_at,
+ )
- # Serialize sources if present
- if metadata.get("sources"):
- metadata["sources"] = [
- json.dumps(s) if not isinstance(s, str) else s for s in metadata["sources"]
- ]
+ def add_node(self, id: str, memory: str, metadata: dict[str, Any], user_name: str) -> None:
+ """Add a memory node."""
+ if not user_name:
+ raise ValueError("add_node requires user_name")
+ self.add_nodes_batch(
+ [{"id": id, "memory": memory, "metadata": metadata}],
+ user_name,
+ )
+ def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None:
+ """Insert every node for one tenant in a single statement."""
+ if not nodes:
+ return
+ if not user_name:
+ raise ValueError("add_nodes_batch requires user_name")
+ rows: list[tuple[Any, ...]] = []
+ for node in nodes:
+ try:
+ rows.append(
+ self._prepare_node_for_insert(
+ node["id"],
+ node["memory"],
+ node.get("metadata", {}),
+ user_name,
+ )
+ )
+ except Exception:
+ logger.error(
+ "add_nodes_batch failed to prepare node %s",
+ node.get("id", "unknown"),
+ exc_info=True,
+ )
+ if not rows:
+ return
+ memories = self._physical_table("memories", user_name)
+ dim = self._vector_dim()
+ values_sql = ",".join([f"(%s, %s, %s, %s::halfvec({dim}), %s, %s, %s)"] * len(rows))
+ params: list[Any] = [value for row in rows for value in row]
conn = self._get_conn()
try:
with conn.cursor() as cur:
- if embedding:
- cur.execute(
- f"""
- INSERT INTO {self.schema}.memories
- (id, memory, properties, embedding, user_name, created_at, updated_at)
- VALUES (%s, %s, %s, %s::vector, %s, %s, %s)
- ON CONFLICT (id) DO UPDATE SET
- memory = EXCLUDED.memory,
- properties = EXCLUDED.properties,
- embedding = EXCLUDED.embedding,
- updated_at = EXCLUDED.updated_at
- """,
- (
- id,
- memory,
- json.dumps(metadata),
- embedding,
- user_name,
- created_at,
- updated_at,
- ),
- )
- else:
- cur.execute(
- f"""
- INSERT INTO {self.schema}.memories
- (id, memory, properties, user_name, created_at, updated_at)
- VALUES (%s, %s, %s, %s, %s, %s)
- ON CONFLICT (id) DO UPDATE SET
- memory = EXCLUDED.memory,
- properties = EXCLUDED.properties,
- updated_at = EXCLUDED.updated_at
+ cur.execute(
+ f"""
+ INSERT INTO {memories}
+ (id, memory, properties, embedding, user_name, created_at, updated_at)
+ VALUES {values_sql}
+ ON CONFLICT (user_name, id) DO UPDATE SET
+ memory = EXCLUDED.memory,
+ properties = EXCLUDED.properties,
+ embedding = EXCLUDED.embedding,
+ updated_at = EXCLUDED.updated_at
+ WHERE "Memory".user_name = EXCLUDED.user_name
""",
- (id, memory, json.dumps(metadata), user_name, created_at, updated_at),
- )
+ params,
+ )
finally:
self._put_conn(conn)
- def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str | None = None) -> None:
- """Batch add memory nodes."""
- for node in nodes:
- self.add_node(
- id=node["id"],
- memory=node["memory"],
- metadata=node.get("metadata", {}),
- user_name=user_name,
- )
-
def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = None) -> None:
"""Update node fields."""
user_name = user_name or self.user_name
if not fields:
return
+ fields = fields.copy()
# Get current node
current = self.get_node(id, user_name=user_name)
@@ -327,7 +483,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N
embedding = fields.pop("embedding", None)
memory = fields.pop("memory", current.get("memory", ""))
props.update(fields)
- props["updated_at"] = datetime.utcnow().isoformat()
+ props["updated_at"] = datetime.now(timezone.utc).isoformat()
+ memories = self._physical_table("memories", user_name)
conn = self._get_conn()
try:
@@ -335,8 +492,8 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N
if embedding:
cur.execute(
f"""
- UPDATE {self.schema}.memories
- SET memory = %s, properties = %s, embedding = %s::vector, updated_at = NOW()
+ UPDATE {memories}
+ SET memory = %s, properties = %s, embedding = %s::halfvec({self._vector_dim()}), updated_at = NOW()
WHERE id = %s AND user_name = %s
""",
(memory, json.dumps(props), embedding, id, user_name),
@@ -344,7 +501,7 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N
else:
cur.execute(
f"""
- UPDATE {self.schema}.memories
+ UPDATE {memories}
SET memory = %s, properties = %s, updated_at = NOW()
WHERE id = %s AND user_name = %s
""",
@@ -356,21 +513,23 @@ def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = N
def delete_node(self, id: str, user_name: str | None = None) -> None:
"""Delete a node and its edges."""
user_name = user_name or self.user_name
+ memories = self._physical_table("memories", user_name)
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
# Delete edges
cur.execute(
f"""
- DELETE FROM {self.schema}.edges
- WHERE source_id = %s OR target_id = %s
+ DELETE FROM {edges}
+ WHERE user_name = %s AND (source_id = %s OR target_id = %s)
""",
- (id, id),
+ (user_name, id, id),
)
# Delete node
cur.execute(
f"""
- DELETE FROM {self.schema}.memories
+ DELETE FROM {memories}
WHERE id = %s AND user_name = %s
""",
(id, user_name),
@@ -378,22 +537,34 @@ def delete_node(self, id: str, user_name: str | None = None) -> None:
finally:
self._put_conn(conn)
- def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[str, Any] | None:
+ def get_node(
+ self, id: str, include_embedding: bool = False, user_name: str | None = None
+ ) -> dict[str, Any] | None:
"""Get a single node by ID."""
- user_name = kwargs.get("user_name") or self.user_name
+ relation = self._tenant_relation("memories", [user_name] if user_name else [])
conn = self._get_conn()
try:
with conn.cursor() as cur:
cols = "id, memory, properties, created_at, updated_at"
if include_embedding:
cols += ", embedding"
- cur.execute(
- f"""
- SELECT {cols} FROM {self.schema}.memories
- WHERE id = %s AND user_name = %s
- """,
- (id, user_name),
- )
+ cols += ", user_name"
+ if user_name:
+ cur.execute(
+ f"""
+ SELECT {cols} FROM {relation}
+ WHERE id = %s AND user_name = %s
+ """,
+ (id, user_name),
+ )
+ else:
+ cur.execute(
+ f"""
+ SELECT {cols} FROM {relation}
+ WHERE id = %s
+ """,
+ (id,),
+ )
row = cur.fetchone()
if not row:
return None
@@ -402,32 +573,77 @@ def get_node(self, id: str, include_embedding: bool = False, **kwargs) -> dict[s
self._put_conn(conn)
def get_nodes(
- self, ids: list, include_embedding: bool = False, **kwargs
+ self, ids: list[str], user_name: str | None = None, **kwargs
) -> list[dict[str, Any]]:
"""Get multiple nodes by IDs."""
if not ids:
return []
- user_name = kwargs.get("user_name") or self.user_name
+ include_embedding = bool(kwargs.get("include_embedding", False))
+ relation = self._tenant_relation("memories", [user_name] if user_name else [])
conn = self._get_conn()
try:
with conn.cursor() as cur:
cols = "id, memory, properties, created_at, updated_at"
if include_embedding:
cols += ", embedding"
- cur.execute(
- f"""
- SELECT {cols} FROM {self.schema}.memories
- WHERE id = ANY(%s) AND user_name = %s
- """,
- (ids, user_name),
- )
+ cols += ", user_name"
+ if user_name:
+ cur.execute(
+ f"""
+ SELECT {cols} FROM {relation}
+ WHERE id = ANY(%s) AND user_name = %s
+ AND COALESCE(properties->>'status', 'activated') <> 'archived'
+ """,
+ (ids, user_name),
+ )
+ else:
+ cur.execute(
+ f"""
+ SELECT {cols} FROM {relation}
+ WHERE id = ANY(%s)
+ AND COALESCE(properties->>'status', 'activated') <> 'archived'
+ """,
+ (ids,),
+ )
return [self._parse_row(row, include_embedding) for row in cur.fetchall()]
finally:
self._put_conn(conn)
+ @staticmethod
+ def _normalize_node_properties(
+ raw_properties: dict[str, Any], user_name: str | None = None
+ ) -> dict[str, Any]:
+ """Restore the logical PolarDB metadata shape from relational storage."""
+ properties = dict(raw_properties)
+ if user_name:
+ properties["user_name"] = user_name
+
+ sources = properties.get("sources")
+ if isinstance(sources, list):
+ normalized_sources: list[Any] = []
+ for source in sources:
+ if isinstance(source, str):
+ try:
+ parsed_source = json.loads(source)
+ except (json.JSONDecodeError, TypeError):
+ parsed_source = {"type": "doc", "content": source}
+ normalized_sources.append(parsed_source)
+ elif isinstance(source, dict):
+ normalized_sources.append(source)
+ else:
+ normalized_sources.append({"type": "doc", "content": str(source)})
+ properties["sources"] = normalized_sources
+
+ return properties
+
def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]:
"""Parse database row to node dict."""
- props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}")
+ raw_props = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}")
+ user_name_index = 6 if include_embedding else 5
+ user_name = row[user_name_index] if len(row) > user_name_index else None
+ props = self._normalize_node_properties(raw_props, user_name=user_name)
+ props["id"] = row[0]
+ props["memory"] = row[1] or ""
props["created_at"] = row[3].isoformat() if row[3] else None
props["updated_at"] = row[4].isoformat() if row[4] else None
result = {
@@ -436,9 +652,21 @@ def _parse_row(self, row, include_embedding: bool = False) -> dict[str, Any]:
"metadata": props,
}
if include_embedding and len(row) > 5:
- result["metadata"]["embedding"] = row[5]
+ result["metadata"]["embedding"] = self._coerce_embedding(row[5])
return result
+ @staticmethod
+ def _coerce_embedding(value: Any) -> list[float]:
+ """Normalize pgvector values returned as lists, tuples, or text."""
+ if value is None:
+ return []
+ if isinstance(value, str):
+ value = value.strip().strip("[]")
+ if not value:
+ return []
+ return [float(item) for item in value.split(",")]
+ return [float(item) for item in value]
+
@staticmethod
def _is_safe_field_name(field: str) -> bool:
"""Validate field names used in dynamic SQL fragments."""
@@ -486,31 +714,40 @@ def _build_single_filter_condition(
op_map = {"gt": ">", "lt": "<", "gte": ">=", "lte": "<="}
sql_op = op_map[op]
if raw_key in timestamp_fields or raw_key.endswith("_at"):
- parts.append(f"({text_expr})::timestamptz {sql_op} %s::timestamptz")
+ parts.append(f"({text_expr})::timestamp {sql_op} %s::timestamp")
params.append(op_value)
else:
parts.append(f"NULLIF({text_expr}, '')::numeric {sql_op} %s")
params.append(op_value)
elif op == "contains":
- if raw_key in array_fields:
- parts.append(f"{json_expr} @> %s::jsonb")
- params.append(json.dumps([op_value]))
- else:
- parts.append(f"{text_expr} ILIKE %s")
- params.append(f"%{op_value}%")
+ values = op_value if isinstance(op_value, list) else [op_value]
+ or_parts = []
+ for item in values:
+ or_parts.append(f"{json_expr} @> %s::jsonb")
+ params.append(json.dumps([item], ensure_ascii=False))
+ parts.append("(" + " OR ".join(or_parts) + ")")
elif op == "in":
if not isinstance(op_value, list):
raise ValueError(
- f"in operator expects list for '{key}', got {type(op_value).__name__}"
+ f"in operator expects list for '{key}', "
+ f"got {type(op_value).__name__}"
)
- if raw_key in array_fields:
- parts.append(f"{json_expr} ?| %s")
- params.append([str(v) for v in op_value])
+ if len(op_value) == 0:
+ parts.append("FALSE")
+ elif raw_key in array_fields or raw_key == "related_id":
+ or_parts = []
+ for item in op_value:
+ or_parts.append(f"{json_expr} @> %s::jsonb")
+ params.append(json.dumps([str(item)], ensure_ascii=False))
+ parts.append("(" + " OR ".join(or_parts) + ")")
else:
parts.append(f"{text_expr} = ANY(%s)")
params.append([str(v) for v in op_value])
elif op == "like":
- parts.append(f"{text_expr} ILIKE %s")
+ parts.append(f"{text_expr} LIKE %s")
+ params.append(f"%{op_value}%")
+ elif op == "nolike":
+ parts.append(f"{text_expr} NOT LIKE %s")
params.append(f"%{op_value}%")
else:
raise ValueError(f"Unsupported filter operator: {op}")
@@ -532,6 +769,7 @@ def _build_single_filter_condition(
def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[Any]) -> str:
"""Build SQL WHERE fragment from filter dict."""
+ filter_dict = self.parse_filter(filter_dict)
if not filter_dict:
return ""
@@ -542,7 +780,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A
parts: list[str] = []
for cond in and_conditions:
if isinstance(cond, dict):
- cond_sql = self._build_single_filter_condition(cond, params)
+ cond_sql = self._build_filter_where_clause(cond, params)
if cond_sql:
parts.append(f"({cond_sql})")
return " AND ".join(parts)
@@ -554,7 +792,7 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A
parts: list[str] = []
for cond in or_conditions:
if isinstance(cond, dict):
- cond_sql = self._build_single_filter_condition(cond, params)
+ cond_sql = self._build_filter_where_clause(cond, params)
if cond_sql:
parts.append(f"({cond_sql})")
return f"({' OR '.join(parts)})" if parts else ""
@@ -562,6 +800,55 @@ def _build_filter_where_clause(self, filter_dict: dict[str, Any], params: list[A
cond_sql = self._build_single_filter_condition(filter_dict, params)
return cond_sql or ""
+ def parse_filter(self, filter_dict: dict | None = None):
+ """Apply the same implicit info.* field routing as the PolarDB backend."""
+ if filter_dict is None:
+ return None
+ full_fields = {
+ "id",
+ "key",
+ "tags",
+ "type",
+ "usage",
+ "memory",
+ "status",
+ "sources",
+ "user_id",
+ "graph_id",
+ "user_name",
+ "background",
+ "confidence",
+ "created_at",
+ "session_id",
+ "updated_at",
+ "memory_type",
+ "node_type",
+ "info",
+ "source",
+ "file_ids",
+ "project_id",
+ "manager_user_id",
+ "delete_time",
+ "related_id",
+ "message_ids",
+ }
+
+ def normalize(condition: Any) -> Any:
+ if not isinstance(condition, dict):
+ return condition
+ normalized: dict[str, Any] = {}
+ for key, value in condition.items():
+ if key.lower() in {"and", "or"} and isinstance(value, list):
+ normalized[key] = [normalize(item) for item in value]
+ continue
+ normalized_key = (
+ key if key in full_fields or key.startswith("info.") else f"info.{key}"
+ )
+ normalized[normalized_key] = value
+ return normalized
+
+ return normalize(filter_dict)
+
def delete_node_by_prams(
self,
writable_cube_ids: list[str] | None = None,
@@ -571,7 +858,8 @@ def delete_node_by_prams(
) -> int:
"""Delete nodes by memory_ids, file_ids, or filter."""
logger.info(
- "[delete_node_by_prams] memory_ids: %s, file_ids: %s, filter: %s, writable_cube_ids: %s",
+ "[delete_node_by_prams] memory_ids: %s, file_ids: %s, "
+ "filter: %s, writable_cube_ids: %s",
memory_ids,
file_ids,
filter,
@@ -600,7 +888,8 @@ def delete_node_by_prams(
if not where_conditions:
logger.warning(
- "[delete_node_by_prams] No nodes to delete (no memory_ids, file_ids, or filter provided)"
+ "[delete_node_by_prams] No nodes to delete "
+ "(no memory_ids, file_ids, or filter provided)"
)
return 0
@@ -609,32 +898,148 @@ def delete_node_by_prams(
params.append(writable_cube_ids)
where_clause = " AND ".join(where_conditions)
+ memory_tables = self._physical_tables("memories", writable_cube_ids)
+ edge_tables = self._physical_tables("edges", writable_cube_ids)
conn = self._get_conn()
try:
with conn.cursor() as cur:
- query = f"""
- WITH to_delete AS (
- SELECT id
- FROM {self.schema}.memories
- WHERE {where_clause}
- ),
- deleted_edges AS (
- DELETE FROM {self.schema}.edges e
+ deleted_count = 0
+ for memories, edges in zip(memory_tables, edge_tables, strict=True):
+ query = f"""
+ WITH to_delete AS (
+ SELECT id, user_name
+ FROM {memories}
+ WHERE {where_clause}
+ ),
+ deleted_edges AS (
+ DELETE FROM {edges} e
+ USING to_delete d
+ WHERE e.user_name = d.user_name
+ AND (e.source_id = d.id OR e.target_id = d.id)
+ )
+ DELETE FROM {memories} m
USING to_delete d
- WHERE e.source_id = d.id OR e.target_id = d.id
- )
- DELETE FROM {self.schema}.memories m
- USING to_delete d
- WHERE m.id = d.id
- """
- cur.execute(query, params)
- deleted_count = cur.rowcount if cur.rowcount is not None else 0
+ WHERE m.id = d.id
+ """
+ cur.execute(query, params)
+ deleted_count += cur.rowcount if cur.rowcount is not None else 0
logger.info("[delete_node_by_prams] Deleted %s nodes", deleted_count)
return deleted_count
finally:
self._put_conn(conn)
+ @staticmethod
+ def _normalize_cube_memory_groups(
+ cube_memory_groups: list[dict[str, Any]],
+ ) -> list[tuple[str, str]]:
+ """Normalize and de-duplicate (tenant, memory-id) requests."""
+ pairs: list[tuple[str, str]] = []
+ seen: set[tuple[str, str]] = set()
+ for group in cube_memory_groups:
+ cube_id = str(group.get("cube_id") or "").strip()
+ if not cube_id:
+ continue
+ for raw_memory_id in group.get("memory_ids") or []:
+ memory_id = str(raw_memory_id).strip()
+ pair = (cube_id, memory_id)
+ if memory_id and pair not in seen:
+ seen.add(pair)
+ pairs.append(pair)
+ return pairs
+
+ def delete_node_by_cube_memory_groups(self, cube_memory_groups: list[dict[str, Any]]) -> int:
+ """Delete exactly the requested (cube, memory-id) pairs.
+
+ PolarDB locates these pairs in one of 400 schemas. PostgreSQL keeps the
+ same isolation rule in a shared table by joining both user_name and id.
+ """
+ pairs = self._normalize_cube_memory_groups(cube_memory_groups)
+ if not pairs:
+ return 0
+ user_names = [pair[0] for pair in pairs]
+ memory_ids = [pair[1] for pair in pairs]
+ tenant_names = list(dict.fromkeys(user_names))
+ memory_tables = self._physical_tables("memories", tenant_names)
+ edge_tables = self._physical_tables("edges", tenant_names)
+
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ deleted_count = 0
+ for memories, edges in zip(memory_tables, edge_tables, strict=True):
+ cur.execute(
+ f"""
+ WITH requested(user_name, memory_id) AS (
+ SELECT * FROM unnest(%s::text[], %s::text[])
+ ), to_delete AS (
+ SELECT m.id, m.user_name
+ FROM {memories} m
+ JOIN requested
+ ON requested.user_name = m.user_name
+ AND requested.memory_id = m.id
+ ), deleted_edges AS (
+ DELETE FROM {edges} e
+ USING to_delete d
+ WHERE e.user_name = d.user_name
+ AND (e.source_id = d.id OR e.target_id = d.id)
+ ), deleted AS (
+ DELETE FROM {memories} m
+ USING to_delete d
+ WHERE m.id = d.id AND m.user_name = d.user_name
+ RETURNING 1
+ )
+ SELECT COUNT(*) FROM deleted
+ """,
+ (user_names, memory_ids),
+ )
+ row = cur.fetchone()
+ deleted_count += int(row[0]) if row and row[0] is not None else 0
+ return deleted_count
+ finally:
+ self._put_conn(conn)
+
+ def get_user_names_by_cube_memory_groups(
+ self, cube_memory_groups: list[dict[str, Any]]
+ ) -> dict[str, str | None]:
+ """Resolve memory ownership without scanning unrelated tenants."""
+ raw_ids = [
+ str(memory_id).strip()
+ for group in cube_memory_groups
+ for memory_id in (group.get("memory_ids") or [])
+ if str(memory_id).strip()
+ ]
+ result: dict[str, str | None] = dict.fromkeys(raw_ids)
+ pairs = self._normalize_cube_memory_groups(cube_memory_groups)
+ if not pairs:
+ return result
+
+ user_names = [pair[0] for pair in pairs]
+ memory_ids = [pair[1] for pair in pairs]
+ memories = self._tenant_relation("memories", list(dict.fromkeys(user_names)), alias="m")
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ WITH requested(user_name, memory_id) AS (
+ SELECT * FROM unnest(%s::text[], %s::text[])
+ )
+ SELECT requested.memory_id, m.user_name
+ FROM requested
+ LEFT JOIN {memories}
+ ON m.id = requested.memory_id
+ AND m.user_name = requested.user_name
+ """,
+ (user_names, memory_ids),
+ )
+ for memory_id, owner in cur.fetchall():
+ if owner:
+ result[str(memory_id)] = str(owner)
+ return result
+ finally:
+ self._put_conn(conn)
+
# =========================================================================
# Edge Management
# =========================================================================
@@ -643,16 +1048,36 @@ def add_edge(
self, source_id: str, target_id: str, type: str, user_name: str | None = None
) -> None:
"""Create an edge between nodes."""
+ user_name = user_name or self.user_name
+ memories = self._physical_table("memories", user_name)
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(
f"""
- INSERT INTO {self.schema}.edges (source_id, target_id, edge_type)
- VALUES (%s, %s, %s)
- ON CONFLICT (source_id, target_id, edge_type) DO NOTHING
+ INSERT INTO {edges}
+ (user_name, source_id, target_id, edge_type)
+ SELECT %s, %s, %s, %s
+ WHERE EXISTS (
+ SELECT 1 FROM {memories}
+ WHERE id = %s AND user_name = %s
+ ) AND EXISTS (
+ SELECT 1 FROM {memories}
+ WHERE id = %s AND user_name = %s
+ )
+ ON CONFLICT (user_name, source_id, target_id, edge_type) DO NOTHING
""",
- (source_id, target_id, type),
+ (
+ user_name,
+ source_id,
+ target_id,
+ type,
+ source_id,
+ user_name,
+ target_id,
+ user_name,
+ ),
)
finally:
self._put_conn(conn)
@@ -661,31 +1086,62 @@ def delete_edge(
self, source_id: str, target_id: str, type: str, user_name: str | None = None
) -> None:
"""Delete an edge."""
+ user_name = user_name or self.user_name
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(
f"""
- DELETE FROM {self.schema}.edges
- WHERE source_id = %s AND target_id = %s AND edge_type = %s
+ DELETE FROM {edges}
+ WHERE user_name = %s
+ AND source_id = %s AND target_id = %s AND edge_type = %s
""",
- (source_id, target_id, type),
+ (user_name, source_id, target_id, type),
)
finally:
self._put_conn(conn)
- def edge_exists(self, source_id: str, target_id: str, type: str) -> bool:
+ def edge_exists(
+ self,
+ source_id: str,
+ target_id: str,
+ type: str = "ANY",
+ direction: str = "OUTGOING",
+ user_name: str | None = None,
+ ) -> bool:
"""Check if edge exists."""
+ user_name = user_name or self.user_name
+ direction = direction.upper()
+ if direction not in {"OUTGOING", "INCOMING", "ANY"}:
+ raise ValueError("direction must be OUTGOING, INCOMING, or ANY")
+ if direction == "INCOMING":
+ source_id, target_id = target_id, source_id
+
+ direction_sql = "source_id = %s AND target_id = %s"
+ params: list[Any] = [user_name, source_id, target_id]
+ if direction == "ANY":
+ direction_sql = (
+ "((source_id = %s AND target_id = %s) OR (source_id = %s AND target_id = %s))"
+ )
+ params = [user_name, source_id, target_id, target_id, source_id]
+
+ type_sql = ""
+ if type != "ANY":
+ type_sql = " AND edge_type = %s"
+ params.append(type)
+
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(
f"""
- SELECT 1 FROM {self.schema}.edges
- WHERE source_id = %s AND target_id = %s AND edge_type = %s
+ SELECT 1 FROM {edges}
+ WHERE user_name = %s AND {direction_sql}{type_sql}
LIMIT 1
""",
- (source_id, target_id, type),
+ params,
)
return cur.fetchone() is not None
finally:
@@ -696,43 +1152,132 @@ def edge_exists(self, source_id: str, target_id: str, type: str) -> bool:
# =========================================================================
def get_neighbors(
- self, id: str, type: str, direction: Literal["in", "out", "both"] = "out"
+ self,
+ id: str,
+ type: str,
+ direction: Literal["in", "out", "both"] = "out",
) -> list[str]:
"""Get neighboring node IDs."""
+ user_name = self.user_name
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
if direction == "out":
cur.execute(
f"""
- SELECT target_id FROM {self.schema}.edges
- WHERE source_id = %s AND edge_type = %s
+ SELECT target_id FROM {edges}
+ WHERE user_name = %s AND source_id = %s AND edge_type = %s
""",
- (id, type),
+ (user_name, id, type),
)
elif direction == "in":
cur.execute(
f"""
- SELECT source_id FROM {self.schema}.edges
- WHERE target_id = %s AND edge_type = %s
+ SELECT source_id FROM {edges}
+ WHERE user_name = %s AND target_id = %s AND edge_type = %s
""",
- (id, type),
+ (user_name, id, type),
)
else: # both
cur.execute(
f"""
- SELECT target_id FROM {self.schema}.edges WHERE source_id = %s AND edge_type = %s
+ SELECT target_id FROM {edges}
+ WHERE user_name = %s AND source_id = %s AND edge_type = %s
UNION
- SELECT source_id FROM {self.schema}.edges WHERE target_id = %s AND edge_type = %s
+ SELECT source_id FROM {edges}
+ WHERE user_name = %s AND target_id = %s AND edge_type = %s
""",
- (id, type, id, type),
+ (user_name, id, type, user_name, id, type),
)
return [row[0] for row in cur.fetchall()]
finally:
self._put_conn(conn)
+ def get_children_with_embeddings(
+ self, id: str, user_name: str | None = None
+ ) -> list[dict[str, Any]]:
+ """Return PARENT-edge children with vectors for structure optimization."""
+ user_name = user_name or self.user_name
+ memories = self._physical_table("memories", user_name)
+ edges = self._physical_table("edges", user_name)
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ SELECT child.id, child.embedding, child.memory
+ FROM {edges} e
+ JOIN {memories} child
+ ON child.id = e.target_id AND child.user_name = e.user_name
+ WHERE e.user_name = %s
+ AND e.source_id = %s
+ AND e.edge_type = 'PARENT'
+ AND child.deleted_at IS NULL
+ ORDER BY e.id
+ """,
+ (user_name, id),
+ )
+ return [
+ {
+ "id": row[0],
+ "embedding": self._coerce_embedding(row[1]),
+ "memory": row[2] or "",
+ }
+ for row in cur.fetchall()
+ ]
+ finally:
+ self._put_conn(conn)
+
+ def get_edges(
+ self,
+ id: str,
+ type: str = "ANY",
+ direction: str = "ANY",
+ user_name: str | None = None,
+ ) -> list[dict[str, str]]:
+ """Return tenant-scoped edges connected to a node."""
+ user_name = user_name or self.user_name
+ direction = direction.upper()
+ if direction not in {"OUTGOING", "INCOMING", "ANY"}:
+ raise ValueError("direction must be OUTGOING, INCOMING, or ANY")
+
+ conditions = ["user_name = %s"]
+ params: list[Any] = [user_name]
+ if direction == "OUTGOING":
+ conditions.append("source_id = %s")
+ params.append(id)
+ elif direction == "INCOMING":
+ conditions.append("target_id = %s")
+ params.append(id)
+ else:
+ conditions.append("(source_id = %s OR target_id = %s)")
+ params.extend([id, id])
+ if type != "ANY":
+ conditions.append("edge_type = %s")
+ params.append(type)
+
+ edges = self._physical_table("edges", user_name)
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ SELECT source_id, target_id, edge_type
+ FROM {edges}
+ WHERE {" AND ".join(conditions)}
+ ORDER BY id
+ """,
+ params,
+ )
+ return [{"from": row[0], "to": row[1], "type": row[2]} for row in cur.fetchall()]
+ finally:
+ self._put_conn(conn)
+
def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[str]:
"""Get path between nodes using recursive CTE."""
+ user_name = self.user_name
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
@@ -740,13 +1285,14 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s
f"""
WITH RECURSIVE path AS (
SELECT source_id, target_id, ARRAY[source_id] as nodes, 1 as depth
- FROM {self.schema}.edges
- WHERE source_id = %s
+ FROM {edges}
+ WHERE user_name = %s AND source_id = %s
UNION ALL
SELECT e.source_id, e.target_id, p.nodes || e.source_id, p.depth + 1
- FROM {self.schema}.edges e
+ FROM {edges} e
JOIN path p ON e.source_id = p.target_id
- WHERE p.depth < %s AND NOT e.source_id = ANY(p.nodes)
+ WHERE e.user_name = %s
+ AND p.depth < %s AND NOT e.source_id = ANY(p.nodes)
)
SELECT nodes || target_id as full_path
FROM path
@@ -754,34 +1300,95 @@ def get_path(self, source_id: str, target_id: str, max_depth: int = 3) -> list[s
ORDER BY depth
LIMIT 1
""",
- (source_id, max_depth, target_id),
+ (user_name, source_id, user_name, max_depth, target_id),
)
row = cur.fetchone()
return row[0] if row else []
finally:
self._put_conn(conn)
- def get_subgraph(self, center_id: str, depth: int = 2) -> list[str]:
- """Get subgraph around center node."""
+ def get_subgraph(
+ self,
+ center_id: str,
+ depth: int = 2,
+ center_status: str = "activated",
+ user_name: str | None = None,
+ ) -> dict[str, Any]:
+ """Return the PolarDB-compatible local subgraph structure."""
+ if not 1 <= depth <= 5:
+ raise ValueError("depth must be 1-5")
+ center_id = center_id.strip('"')
+ user_name = user_name or self.user_name
+ core_node = self.get_node(center_id, user_name=user_name)
+ if core_node is None:
+ return {"core_node": None, "neighbors": [], "edges": []}
+ actual_status = core_node.get("metadata", {}).get("status", "activated")
+ if center_status and actual_status != center_status:
+ return {"core_node": None, "neighbors": [], "edges": []}
+
+ edges_table = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(
f"""
- WITH RECURSIVE subgraph AS (
- SELECT %s::text as node_id, 0 as level
- UNION
- SELECT CASE WHEN e.source_id = s.node_id THEN e.target_id ELSE e.source_id END,
- s.level + 1
- FROM {self.schema}.edges e
+ WITH RECURSIVE subgraph(node_id, level, path) AS (
+ SELECT %s::text, 0, ARRAY[%s::text]
+ UNION ALL
+ SELECT
+ CASE
+ WHEN e.source_id = s.node_id THEN e.target_id
+ ELSE e.source_id
+ END,
+ s.level + 1,
+ s.path || CASE
+ WHEN e.source_id = s.node_id THEN e.target_id
+ ELSE e.source_id
+ END
+ FROM {edges_table} e
JOIN subgraph s ON (e.source_id = s.node_id OR e.target_id = s.node_id)
- WHERE s.level < %s
+ WHERE e.user_name = %s
+ AND s.level < %s
+ AND NOT (
+ CASE
+ WHEN e.source_id = s.node_id THEN e.target_id
+ ELSE e.source_id
+ END
+ ) = ANY(s.path)
)
- SELECT DISTINCT node_id FROM subgraph
- """,
- (center_id, depth),
+ SELECT DISTINCT node_id FROM subgraph ORDER BY node_id
+ """,
+ (center_id, center_id, user_name, depth),
)
- return [row[0] for row in cur.fetchall()]
+ node_ids = [str(row[0]) for row in cur.fetchall()]
+ neighbor_ids = [node_id for node_id in node_ids if node_id != center_id]
+ neighbors = (
+ self.get_nodes(neighbor_ids, user_name=user_name) if neighbor_ids else []
+ )
+
+ if node_ids:
+ cur.execute(
+ f"""
+ SELECT source_id, target_id, edge_type
+ FROM {edges_table}
+ WHERE user_name = %s
+ AND source_id = ANY(%s)
+ AND target_id = ANY(%s)
+ ORDER BY id
+ """,
+ (user_name, node_ids, node_ids),
+ )
+ edges = [
+ {"source": row[0], "target": row[1], "type": row[2]}
+ for row in cur.fetchall()
+ ]
+ else:
+ edges = []
+ return {
+ "core_node": core_node,
+ "neighbors": neighbors,
+ "edges": edges,
+ }
finally:
self._put_conn(conn)
@@ -789,95 +1396,638 @@ def get_context_chain(self, id: str, type: str = "FOLLOWS") -> list[str]:
"""Get ordered chain following relationship type."""
return self.get_neighbors(id, type, "out")
+ def get_neighbors_by_tag(
+ self,
+ tags: list[str],
+ exclude_ids: list[str],
+ top_k: int = 5,
+ min_overlap: int = 1,
+ include_embedding: bool = False,
+ user_name: str | None = None,
+ ) -> list[dict[str, Any]]:
+ """Find active tenant memories ranked by tag overlap."""
+ if not tags or top_k <= 0:
+ return []
+ user_name = user_name or self.user_name
+ params: list[Any] = [user_name]
+ exclude_sql = ""
+ if exclude_ids:
+ exclude_sql = "AND id <> ALL(%s)"
+ params.append(exclude_ids)
+
+ cols = "id, memory, properties, created_at, updated_at"
+ if include_embedding:
+ cols += ", embedding"
+ cols += ", user_name"
+ memories = self._physical_table("memories", user_name)
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ SELECT {cols}
+ FROM {memories}
+ WHERE user_name = %s
+ AND COALESCE(properties->>'status', 'activated') = 'activated'
+ AND COALESCE(properties->>'node_type', '') <> 'reasoning'
+ AND COALESCE(properties->>'memory_type', '') <> 'WorkingMemory'
+ {exclude_sql}
+ """,
+ params,
+ )
+ ranked: list[tuple[int, dict[str, Any]]] = []
+ requested_tags = set(tags)
+ for row in cur.fetchall():
+ node = self._parse_row(row, include_embedding)
+ node_tags = node.get("metadata", {}).get("tags", [])
+ if isinstance(node_tags, str):
+ try:
+ node_tags = json.loads(node_tags)
+ except (TypeError, json.JSONDecodeError):
+ node_tags = []
+ overlap = len(requested_tags.intersection(node_tags or []))
+ if overlap >= min_overlap:
+ ranked.append((overlap, node))
+ ranked.sort(key=lambda item: item[0], reverse=True)
+ return [node for _, node in ranked[:top_k]]
+ finally:
+ self._put_conn(conn)
+
# =========================================================================
# Search Operations
# =========================================================================
- def search_by_embedding(
+ def _effective_user_names(
self,
- vector: list[float],
- top_k: int = 5,
- scope: str | None = None,
- status: str | None = None,
- threshold: float | None = None,
- search_filter: dict | None = None,
user_name: str | None = None,
- filter: dict | None = None,
knowledgebase_ids: list[str] | None = None,
- **kwargs,
- ) -> list[dict]:
- """Search nodes by vector similarity using pgvector."""
- user_name = user_name or self.user_name
+ ) -> list[str]:
+ """Resolve the memory cubes that a query is allowed to read."""
+ names: list[str] = []
+ primary = user_name or self.user_name
+ if primary:
+ names.append(primary)
+ for knowledgebase_id in knowledgebase_ids or []:
+ if knowledgebase_id and knowledgebase_id not in names:
+ names.append(knowledgebase_id)
+ return names
+
+ def _shard_count(self) -> int:
+ return int(getattr(self.config, "shard_count", 400))
+
+ def _remainder_cache(self) -> dict[str, int]:
+ cache = getattr(self, "_partition_remainders", None)
+ if cache is None:
+ cache = {}
+ self._partition_remainders = cache
+ return cache
+
+ def _partition_remainder(self, user_name: str, conn=None) -> int:
+ """Return the PolarDB MD5 shard number for one tenant."""
+ del conn
+ cache = self._remainder_cache()
+ cached = cache.get(user_name)
+ if cached is not None:
+ return cached
+ remainder = (
+ int(hashlib.md5(user_name.encode("utf-8")).hexdigest(), 16) % self._shard_count()
+ )
+ cache[user_name] = remainder
+ return remainder
- # Build WHERE clause
- conditions = ["embedding IS NOT NULL"]
- params = []
+ def _qualified_partition(self, table: str, remainder: int) -> str:
+ physical_names = {
+ "memories": "Memory",
+ "edges": "Edges",
+ }
+ physical = physical_names.get(table)
+ if physical is None:
+ raise ValueError(f"Invalid partitioned table: {table}")
+ schema = f"{self.schema}_graph_{remainder}"
+ if not self._is_safe_field_name(schema):
+ raise ValueError(f"Invalid partition schema: {schema}")
+ return f'"{schema}"."{physical}"'
+
+ def _all_partitions_relation(self, table: str, alias: str | None = None) -> str:
+ """Read every hash partition explicitly, as PolarDB unions every shard."""
+ branches = " UNION ALL ".join(
+ f"SELECT * FROM {self._qualified_partition(table, remainder)}"
+ for remainder in range(self._shard_count())
+ )
+ return f"({branches}) {alias or table}"
- if user_name:
+ def _tenant_relation(
+ self,
+ table: str,
+ user_names: list[str] | None,
+ alias: str | None = None,
+ conn=None,
+ ) -> str:
+ """Return one child table, or a union of only the touched partitions."""
+ names = [name for name in (user_names or []) if name]
+ if not names:
+ return self._all_partitions_relation(table, alias)
+ remainders = sorted({self._partition_remainder(name, conn) for name in names})
+ if len(remainders) == 1:
+ relation = self._qualified_partition(table, remainders[0])
+ else:
+ branches = " UNION ALL ".join(
+ f"SELECT * FROM {self._qualified_partition(table, remainder)}"
+ for remainder in remainders
+ )
+ relation = f"({branches})"
+ if alias is None:
+ alias = table
+ return f"{relation} {alias}" if alias else relation
+
+ def _physical_tables(self, table: str, user_names: list[str] | None, conn=None) -> list[str]:
+ """Real tables a write can target. A union subquery cannot be updated."""
+ names = [name for name in (user_names or []) if name]
+ if not names:
+ return [
+ self._qualified_partition(table, remainder)
+ for remainder in range(self._shard_count())
+ ]
+ remainders = sorted({self._partition_remainder(name, conn) for name in names})
+ return [self._qualified_partition(table, remainder) for remainder in remainders]
+
+ def _physical_table(self, table: str, user_name: str | None, conn=None) -> str:
+ """Return the one partition table PolarDB would address for this tenant."""
+ if not user_name:
+ return f"{self.schema}.{table}"
+ return self._qualified_partition(table, self._partition_remainder(user_name, conn))
+
+ def _vector_dim(self) -> int:
+ dim = int(getattr(self.config, "embedding_dimension", 1024))
+ if dim < 1 or dim > 4096:
+ raise ValueError(f"Invalid embedding dimension: {dim}")
+ return dim
+
+ def _ensure_hnsw_search_settings(self, conn) -> None:
+ """Match PolarDB's HNSW session settings once for each pooled connection."""
+ ready = getattr(self, "_hnsw_ready_ids", None)
+ if ready is None:
+ ready = set()
+ self._hnsw_ready_ids = ready
+ marker = id(conn)
+ if marker in ready:
+ return
+ with conn.cursor() as cur:
+ cur.execute(
+ """
+ SELECT set_config('hnsw.ef_search', '100', false),
+ set_config('hnsw.iterative_scan', 'relaxed_order', false)
+ """
+ )
+ ready.add(marker)
+
+ def _projected_field_exprs(self, return_fields: list[str] | None) -> list[tuple[str, str]]:
+ projected: list[tuple[str, str]] = []
+ for field in self._validate_return_fields(return_fields):
+ if field in {"id", "memory", "score", "user_name", "created_at", "updated_at"}:
+ continue
+ if field == "embedding":
+ projected.append((field, "embedding"))
+ else:
+ projected.append((field, f"properties->'{field}'"))
+ return projected
+
+ @staticmethod
+ def _decode_projected_value(value: Any) -> Any:
+ if isinstance(value, str):
+ try:
+ return json.loads(value)
+ except (json.JSONDecodeError, TypeError):
+ return value
+ return value
+
+ def _build_projected_search_result(
+ self, row: tuple[Any, ...], extra_fields: list[str]
+ ) -> dict[str, Any]:
+ created_at, updated_at = row[3], row[4]
+ result: dict[str, Any] = {
+ "id": row[0],
+ "score": float(row[1]),
+ "memory": row[2] or "",
+ "user_name": row[5],
+ }
+ if created_at is not None:
+ result["created_at"] = (
+ created_at.isoformat() if hasattr(created_at, "isoformat") else created_at
+ )
+ if updated_at is not None:
+ result["updated_at"] = (
+ updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at
+ )
+ for offset, field in enumerate(extra_fields):
+ result[field] = self._decode_projected_value(row[6 + offset])
+ return result
+
+ def _append_user_name_condition(
+ self, conditions: list[str], params: list[Any], user_names: list[str]
+ ) -> None:
+ if len(user_names) == 1:
conditions.append("user_name = %s")
- params.append(user_name)
+ params.append(user_names[0])
+ elif user_names:
+ conditions.append("user_name = ANY(%s)")
+ params.append(user_names)
+ def _build_search_conditions(
+ self,
+ params: list[Any],
+ *,
+ user_name: str | None = None,
+ knowledgebase_ids: list[str] | None = None,
+ scope: str | None = None,
+ status: str | None = None,
+ search_filter: dict | None = None,
+ filter: dict | None = None,
+ default_activated: bool = True,
+ ) -> list[str]:
+ conditions: list[str] = []
+ user_names = self._effective_user_names(user_name, knowledgebase_ids)
+ self._append_user_name_condition(conditions, params, user_names)
if scope:
conditions.append("properties->>'memory_type' = %s")
params.append(scope)
-
if status:
conditions.append("properties->>'status' = %s")
params.append(status)
- else:
- conditions.append(
- "(properties->>'status' = 'activated' OR properties->>'status' IS NULL)"
- )
+ elif default_activated:
+ conditions.append("properties->>'status' = 'activated'")
+
+ for filter_dict in (search_filter, filter):
+ if filter_dict:
+ filter_sql = self._build_filter_where_clause(filter_dict, params)
+ if filter_sql:
+ conditions.append(f"({filter_sql})")
+ return conditions
+
+ def _build_search_result(
+ self, row: tuple[Any, ...], return_fields: list[str] | None
+ ) -> dict[str, Any]:
+ result: dict[str, Any] = {"id": row[0], "score": float(row[1])}
+ fields = self._validate_return_fields(return_fields)
+ if not fields:
+ return result
+
+ memory = row[2] or ""
+ created_at, updated_at, user_name = row[4], row[5], row[6]
+ raw_properties = row[3] if isinstance(row[3], dict) else json.loads(row[3] or "{}")
+ properties = dict(raw_properties)
+ properties["user_name"] = user_name
+ properties.update(
+ {
+ "id": row[0],
+ "memory": memory,
+ "created_at": created_at.isoformat() if created_at else None,
+ "updated_at": updated_at.isoformat() if updated_at else None,
+ "user_name": user_name,
+ }
+ )
+ for field in fields:
+ if field != "id" and field in properties:
+ result[field] = properties[field]
+ return result
- if search_filter:
- for k, v in search_filter.items():
- conditions.append(f"properties->>'{k}' = %s")
- params.append(str(v))
+ def _build_keyword_result(
+ self, row: tuple[Any, ...], return_fields: list[str] | None
+ ) -> dict[str, Any]:
+ result: dict[str, Any] = {"id": row[0]}
+ fields = self._validate_return_fields(return_fields)
+ if not fields:
+ return result
+
+ raw_properties = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}")
+ properties = dict(raw_properties)
+ properties["user_name"] = row[5]
+ properties.update(
+ {
+ "id": row[0],
+ "memory": row[1] or "",
+ "created_at": row[3].isoformat() if row[3] else None,
+ "updated_at": row[4].isoformat() if row[4] else None,
+ "user_name": row[5],
+ }
+ )
+ for field in fields:
+ if field != "id" and field in properties:
+ result[field] = properties[field]
+ return result
+ def search_by_embedding(
+ self,
+ vector: list[float],
+ user_name: str,
+ top_k: int = 5,
+ scope: str | None = None,
+ status: str | None = None,
+ threshold: float | None = None,
+ search_filter: dict | None = None,
+ filter: dict | None = None,
+ knowledgebase_ids: list[str] | None = None,
+ return_fields: list[str] | None = None,
+ light_weight_mode: bool = False,
+ **kwargs,
+ ) -> list[dict]:
+ """Search nodes by vector similarity using pgvector."""
+ if light_weight_mode and not return_fields:
+ return_fields = [
+ "memory",
+ "memory_type",
+ "user_name",
+ "status",
+ "key",
+ "tags",
+ "confidence",
+ "source",
+ ]
+ params: list[Any] = []
+ conditions = ["embedding IS NOT NULL"]
+ conditions.extend(
+ self._build_search_conditions(
+ params,
+ user_name=user_name,
+ knowledgebase_ids=knowledgebase_ids,
+ scope=scope,
+ status=status,
+ search_filter=search_filter,
+ filter=filter,
+ )
+ )
where_clause = " AND ".join(conditions)
+ relation = self._tenant_relation(
+ "memories",
+ self._effective_user_names(user_name, knowledgebase_ids),
+ )
- # pgvector cosine distance: 1 - (a <=> b) gives similarity score
+ # Match PolarDB: one distance, index-ordered LIMIT, then drop rows at or past 0.9.
+ dim = self._vector_dim()
+ distance_sql = "embedding <=> (SELECT query_vector FROM q)"
conn = self._get_conn()
try:
+ self._ensure_hnsw_search_settings(conn)
with conn.cursor() as cur:
+ projected = self._projected_field_exprs(return_fields) if light_weight_mode else []
+ if light_weight_mode:
+ inner_columns = "memory, created_at, updated_at, user_name"
+ extra_inner = "".join(
+ f', {expression} AS "{field}"' for field, expression in projected
+ )
+ extra_outer = "".join(f', "{field}"' for field, _ in projected)
+ else:
+ inner_columns = "memory, properties, created_at, updated_at, user_name"
+ extra_inner = ""
+ extra_outer = ""
cur.execute(
f"""
- SELECT id, 1 - (embedding <=> %s::vector) as score
- FROM {self.schema}.memories
- WHERE {where_clause}
- ORDER BY embedding <=> %s::vector
- LIMIT %s
- """,
- (vector, *params, vector, top_k),
+ WITH q AS (
+ SELECT %s::halfvec({dim}) AS query_vector
+ ),
+ t AS MATERIALIZED (
+ SELECT COALESCE(properties->>'id', id) AS id,
+ ({distance_sql}) AS scope_distance,
+ {inner_columns}
+ {extra_inner}
+ FROM {relation}
+ WHERE {where_clause}
+ ORDER BY {distance_sql}
+ LIMIT %s
+ )
+ SELECT id,
+ ((1 - scope_distance) + 1) / 2.0 AS score,
+ {inner_columns}
+ {extra_outer}
+ FROM t
+ WHERE scope_distance < 0.9
+ """,
+ (vector, *params, top_k),
)
results = []
for row in cur.fetchall():
score = float(row[1])
if threshold is None or score >= threshold:
- results.append({"id": row[0], "score": score})
+ if light_weight_mode:
+ results.append(
+ self._build_projected_search_result(
+ row, [field for field, _ in projected]
+ )
+ )
+ else:
+ results.append(self._build_search_result(row, return_fields))
return results
finally:
self._put_conn(conn)
- def get_by_metadata(
+ def search_by_fulltext(
self,
- filters: list[dict[str, Any]],
+ query_words: list[str],
+ top_k: int = 10,
+ scope: str | None = None,
+ status: str | None = None,
+ threshold: float | None = None,
+ search_filter: dict | None = None,
+ user_name: str | None = None,
+ filter: dict | None = None,
+ knowledgebase_ids: list[str] | None = None,
+ tsvector_field: str = "properties_tsvector_zh",
+ tsquery_config: str = "jiebacfg",
+ return_fields: list[str] | None = None,
+ **kwargs,
+ ) -> list[dict]:
+ """Search with PolarDB's AND tsquery on one tenant partition."""
+ terms = [str(word).strip() for word in query_words if str(word).strip()]
+ if not terms:
+ return []
+
+ query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple")
+ if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config):
+ raise ValueError(f"Invalid PostgreSQL text search config: {query_config}")
+ quoted_terms = []
+ for term in terms:
+ if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}:
+ term = term[1:-1]
+ quoted_terms.append("'" + term.replace("'", "''") + "'")
+ params: list[Any] = [" & ".join(quoted_terms)]
+ conditions = self._build_search_conditions(
+ params,
+ user_name=user_name,
+ knowledgebase_ids=knowledgebase_ids,
+ scope=scope,
+ status=status,
+ search_filter=search_filter,
+ filter=filter,
+ )
+ where_clause = " AND ".join(conditions) if conditions else "TRUE"
+ relation = self._tenant_relation(
+ "memories",
+ self._effective_user_names(user_name, knowledgebase_ids),
+ )
+
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ WITH q AS (
+ SELECT to_tsquery('{query_config}', %s) AS query
+ )
+ SELECT COALESCE(properties->>'id', id) AS id,
+ ts_rank(search_vector, q.query) AS score,
+ memory, properties, created_at, updated_at, user_name
+ FROM {relation}
+ CROSS JOIN q
+ WHERE {where_clause}
+ AND search_vector @@ q.query
+ ORDER BY score DESC, updated_at DESC
+ LIMIT %s
+ """,
+ (*params, top_k),
+ )
+ output = []
+ for row in cur.fetchall():
+ result = self._build_search_result(row, return_fields)
+ if threshold is None or result["score"] >= threshold:
+ output.append(result)
+ return output
+ finally:
+ self._put_conn(conn)
+
+ def search_by_keywords_tfidf(
+ self,
+ query_words: list[str],
+ scope: str | None = None,
status: str | None = None,
+ search_filter: dict | None = None,
user_name: str | None = None,
filter: dict | None = None,
knowledgebase_ids: list[str] | None = None,
+ tsvector_field: str = "properties_tsvector_zh",
+ tsquery_config: str = "jiebaqry",
+ return_fields: list[str] | None = None,
+ **kwargs,
+ ) -> list[dict]:
+ """Return keyword matches with PolarDB's OR tsquery and no score."""
+ if not user_name:
+ return []
+ terms = [str(word).strip() for word in query_words if str(word).strip()]
+ if not terms:
+ return []
+
+ query_config = tsquery_config or getattr(self.config, "fulltext_config", "simple")
+ if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", query_config):
+ raise ValueError(f"Invalid PostgreSQL text search config: {query_config}")
+ quoted_terms = []
+ for term in terms:
+ if len(term) >= 2 and term[0] == term[-1] and term[0] in {"'", '"'}:
+ term = term[1:-1]
+ quoted_terms.append("'" + term.replace("'", "''") + "'")
+ params: list[Any] = [" | ".join(quoted_terms)]
+ conditions = self._build_search_conditions(
+ params,
+ user_name=user_name,
+ knowledgebase_ids=knowledgebase_ids,
+ scope=scope,
+ status=status,
+ search_filter=search_filter,
+ filter=filter,
+ )
+ where_clause = " AND ".join(conditions) if conditions else "TRUE"
+ relation = self._tenant_relation(
+ "memories",
+ self._effective_user_names(user_name, knowledgebase_ids),
+ )
+
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ WITH q AS (
+ SELECT to_tsquery('{query_config}', %s) AS query
+ )
+ SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name
+ FROM {relation}
+ CROSS JOIN q
+ WHERE {where_clause}
+ AND search_vector @@ q.query
+ """,
+ (*params,),
+ )
+ return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()]
+ finally:
+ self._put_conn(conn)
+
+ def search_by_keywords_like(
+ self,
+ query_word: str,
+ scope: str | None = None,
+ status: str | None = None,
+ search_filter: dict | None = None,
+ user_name: str | None = None,
+ filter: dict | None = None,
+ knowledgebase_ids: list[str] | None = None,
+ return_fields: list[str] | None = None,
+ **kwargs,
+ ) -> list[dict]:
+ """Fallback substring search compatible with the PolarDB adapter."""
+ if not user_name:
+ return []
+ pattern = f"%{query_word}%"
+ params: list[Any] = []
+ conditions = self._build_search_conditions(
+ params,
+ user_name=user_name,
+ knowledgebase_ids=knowledgebase_ids,
+ search_filter=search_filter,
+ filter=filter,
+ status=status,
+ scope=scope,
+ )
+ conditions.append("COALESCE(properties->>'memory', '') LIKE %s")
+ params.append(pattern)
+ relation = self._tenant_relation(
+ "memories",
+ self._effective_user_names(user_name, knowledgebase_ids),
+ )
+
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ SELECT COALESCE(properties->>'id', id) AS id, memory, properties, created_at, updated_at, user_name
+ FROM {relation}
+ WHERE {" AND ".join(conditions)}
+ """,
+ params,
+ )
+ return [self._build_keyword_result(row, return_fields) for row in cur.fetchall()]
+ finally:
+ self._put_conn(conn)
+
+ def get_by_metadata(
+ self,
+ filters: list[dict[str, Any]],
+ user_name: str | None = None,
+ filter: dict | None = None,
+ knowledgebase_ids: list | None = None,
user_name_flag: bool = True,
+ status: str | None = None,
+ **kwargs,
) -> list[str]:
"""Get node IDs matching metadata filters."""
- user_name = user_name or self.user_name
+ if not user_name:
+ raise ValueError("get_by_metadata requires user_name")
+ conditions: list[str] = []
+ params: list[Any] = []
+ user_names: list[str] = []
- conditions = []
- params = []
-
- if user_name_flag and user_name:
- conditions.append("user_name = %s")
- params.append(user_name)
+ if user_name_flag:
+ user_names = self._effective_user_names(user_name, knowledgebase_ids)
+ self._append_user_name_condition(conditions, params, user_names)
if status:
conditions.append("properties->>'status' = %s")
@@ -887,29 +2037,52 @@ def get_by_metadata(
field = f["field"]
op = f.get("op", "=")
value = f["value"]
+ text_expr, json_expr = self._field_expr(field)
if op == "=":
- conditions.append(f"properties->>'{field}' = %s")
+ conditions.append(f"{text_expr} = %s")
params.append(str(value))
elif op == "in":
- placeholders = ",".join(["%s"] * len(value))
- conditions.append(f"properties->>'{field}' IN ({placeholders})")
- params.extend([str(v) for v in value])
+ if not isinstance(value, list):
+ raise ValueError("in operator expects a list")
+ conditions.append(f"{text_expr} = ANY(%s)")
+ params.append([str(item) for item in value])
elif op in (">", ">=", "<", "<="):
- conditions.append(f"(properties->>'{field}')::numeric {op} %s")
+ conditions.append(f"NULLIF({text_expr}, '')::numeric {op} %s")
params.append(value)
elif op == "contains":
- conditions.append(f"properties->'{field}' @> %s::jsonb")
- params.append(json.dumps([value]))
+ values = value if isinstance(value, list) else [value]
+ or_parts = []
+ for item in values:
+ or_parts.append(f"{json_expr} @> %s::jsonb")
+ params.append(json.dumps([item], ensure_ascii=False))
+ conditions.append("(" + " OR ".join(or_parts) + ")")
+ elif op == "like":
+ conditions.append(f"{text_expr} LIKE %s")
+ params.append(f"%{value}%")
+ elif op == "starts_with":
+ conditions.append(f"{text_expr} LIKE %s")
+ params.append(f"{value}%")
+ elif op == "ends_with":
+ conditions.append(f"{text_expr} LIKE %s")
+ params.append(f"%{value}")
+ else:
+ raise ValueError(f"Unsupported operator: {op}")
+
+ if filter:
+ filter_sql = self._build_filter_where_clause(filter, params)
+ if filter_sql:
+ conditions.append(f"({filter_sql})")
where_clause = " AND ".join(conditions) if conditions else "TRUE"
+ relation = self._tenant_relation("memories", user_names)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(
f"""
- SELECT id FROM {self.schema}.memories
+ SELECT id FROM {relation}
WHERE {where_clause}
""",
params,
@@ -921,23 +2094,34 @@ def get_by_metadata(
def get_all_memory_items(
self,
scope: str,
+ user_name: str | None = None,
include_embedding: bool = False,
- status: str | None = None,
filter: dict | None = None,
- knowledgebase_ids: list[str] | None = None,
- **kwargs,
+ knowledgebase_ids: list | None = None,
+ status: str | None = None,
) -> list[dict]:
"""Get all memory items of a specific type."""
- user_name = kwargs.get("user_name") or self.user_name
-
- conditions = ["properties->>'memory_type' = %s", "user_name = %s"]
- params = [scope, user_name]
+ if not user_name:
+ return []
+ if scope not in {"WorkingMemory", "LongTermMemory", "UserMemory", "OuterMemory"}:
+ raise ValueError(f"Unsupported memory type scope: {scope}")
- if status:
- conditions.append("properties->>'status' = %s")
- params.append(status)
+ params: list[Any] = []
+ conditions = self._build_search_conditions(
+ params,
+ user_name=user_name,
+ knowledgebase_ids=knowledgebase_ids,
+ scope=scope,
+ status=status,
+ filter=filter,
+ default_activated=False,
+ )
- where_clause = " AND ".join(conditions)
+ where_clause = " AND ".join(conditions) if conditions else "TRUE"
+ relation = self._tenant_relation(
+ "memories",
+ self._effective_user_names(user_name, knowledgebase_ids),
+ )
conn = self._get_conn()
try:
@@ -945,10 +2129,12 @@ def get_all_memory_items(
cols = "id, memory, properties, created_at, updated_at"
if include_embedding:
cols += ", embedding"
+ cols += ", user_name"
cur.execute(
f"""
- SELECT {cols} FROM {self.schema}.memories
+ SELECT {cols} FROM {relation}
WHERE {where_clause}
+ LIMIT 100
""",
params,
)
@@ -957,29 +2143,44 @@ def get_all_memory_items(
self._put_conn(conn)
def get_structure_optimization_candidates(
- self, scope: str, include_embedding: bool = False
+ self,
+ scope: str,
+ include_embedding: bool = False,
+ user_name: str | None = None,
) -> list[dict]:
"""Find isolated nodes (no edges)."""
- user_name = self.user_name
+ if not user_name:
+ raise ValueError("get_structure_optimization_candidates requires user_name")
+ memories = self._tenant_relation("memories", [user_name])
+ edges = self._tenant_relation("edges", [user_name])
conn = self._get_conn()
try:
with conn.cursor() as cur:
cols = "m.id, m.memory, m.properties, m.created_at, m.updated_at"
+ if include_embedding:
+ cols += ", m.embedding"
+ cols += ", m.user_name"
cur.execute(
f"""
SELECT {cols}
- FROM {self.schema}.memories m
- LEFT JOIN {self.schema}.edges e1 ON m.id = e1.source_id
- LEFT JOIN {self.schema}.edges e2 ON m.id = e2.target_id
+ FROM {memories} m
+ LEFT JOIN {edges} e1
+ ON m.id = e1.source_id
+ AND e1.user_name = m.user_name
+ AND e1.edge_type = 'PARENT'
+ LEFT JOIN {edges} e2
+ ON m.id = e2.target_id
+ AND e2.user_name = m.user_name
+ AND e2.edge_type = 'PARENT'
WHERE m.properties->>'memory_type' = %s
AND m.user_name = %s
AND m.properties->>'status' = 'activated'
AND e1.id IS NULL
AND e2.id IS NULL
- """,
+ """,
(scope, user_name),
)
- return [self._parse_row(row, False) for row in cur.fetchall()]
+ return [self._parse_row(row, include_embedding) for row in cur.fetchall()]
finally:
self._put_conn(conn)
@@ -987,6 +2188,190 @@ def get_structure_optimization_candidates(
# Maintenance
# =========================================================================
+ def get_memory_count(self, memory_type: str, user_name: str | None = None) -> int:
+ """Count active memories of one type for a tenant."""
+ user_name = user_name or self.user_name
+ relation = self._tenant_relation("memories", [user_name] if user_name else [])
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ SELECT COUNT(*)
+ FROM {relation}
+ WHERE user_name = %s
+ AND properties->>'memory_type' = %s
+ AND COALESCE(properties->>'status', 'activated') = 'activated'
+ """,
+ (user_name, memory_type),
+ )
+ row = cur.fetchone()
+ return int(row[0]) if row else 0
+ finally:
+ self._put_conn(conn)
+
+ def node_not_exist(self, scope: str, user_name: str | None = None) -> int:
+ """Return 1 when no active node exists for the requested scope."""
+ return int(self.get_memory_count(scope, user_name=user_name) == 0)
+
+ def count_nodes(self, scope: str, user_name: str | None = None) -> int:
+ """Compatibility alias for memory counts."""
+ return self.get_memory_count(scope, user_name=user_name)
+
+ def get_user_names_by_memory_ids(self, memory_ids: list[str]) -> dict[str, str | None]:
+ normalized_ids = list(
+ dict.fromkeys(
+ str(memory_id).strip() for memory_id in memory_ids if str(memory_id).strip()
+ )
+ )
+ if not normalized_ids:
+ return {}
+ result: dict[str, str | None] = dict.fromkeys(normalized_ids)
+ memories = self._all_partitions_relation("memories")
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ SELECT id, user_name
+ FROM {memories}
+ WHERE id = ANY(%s)
+ """,
+ (normalized_ids,),
+ )
+ result.update({str(row[0]): row[1] for row in cur.fetchall()})
+ return result
+ finally:
+ self._put_conn(conn)
+
+ def exist_user_name(self, user_name: str) -> dict[str, bool]:
+ memories = self._physical_table("memories", user_name)
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"SELECT EXISTS(SELECT 1 FROM {memories} WHERE user_name = %s)",
+ (user_name,),
+ )
+ row = cur.fetchone()
+ return {user_name: bool(row and row[0])}
+ finally:
+ self._put_conn(conn)
+
+ def delete_node_by_mem_cube_id(
+ self,
+ mem_cube_id: str,
+ delete_record_id: str,
+ hard_delete: bool = False,
+ ) -> int:
+ """Soft-delete a tenant's memories, or purge a previously deleted batch."""
+ if not mem_cube_id:
+ raise ValueError("delete_node_by_mem_cube_id mem_cube_id is required")
+ if not delete_record_id:
+ raise ValueError("delete_node_by_mem_cube_id delete_record_id is required")
+
+ memories = self._physical_table("memories", mem_cube_id)
+ edges = self._physical_table("edges", mem_cube_id)
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ if hard_delete:
+ cur.execute(
+ f"""
+ WITH deleted AS (
+ DELETE FROM {memories}
+ WHERE user_name = %s
+ AND properties->>'delete_record_id' = %s
+ RETURNING id
+ ), deleted_edges AS (
+ DELETE FROM {edges} e
+ USING deleted d
+ WHERE e.user_name = %s
+ AND (e.source_id = d.id OR e.target_id = d.id)
+ RETURNING e.id
+ )
+ SELECT COUNT(*) FROM deleted
+ """,
+ (mem_cube_id, delete_record_id, mem_cube_id),
+ )
+ row = cur.fetchone()
+ return int(row[0]) if row else 0
+ else:
+ deleted_at = datetime.now(timezone.utc).isoformat()
+ cur.execute(
+ f"""
+ UPDATE {memories}
+ SET properties = properties || %s::jsonb,
+ deleted_at = %s::timestamptz,
+ updated_at = NOW()
+ WHERE user_name = %s
+ AND COALESCE(properties->>'status', 'activated') <> 'deleted'
+ """,
+ (
+ json.dumps(
+ {
+ "status": "deleted",
+ "delete_time": deleted_at,
+ "delete_record_id": delete_record_id,
+ }
+ ),
+ deleted_at,
+ mem_cube_id,
+ ),
+ )
+ return max(int(cur.rowcount or 0), 0)
+ finally:
+ self._put_conn(conn)
+
+ def recover_memory_by_mem_cube_id(
+ self,
+ mem_cube_id: str,
+ delete_record_id: str,
+ ) -> int:
+ """Recover one tenant soft-delete batch."""
+ if not mem_cube_id:
+ raise ValueError("recover_memory_by_mem_cube_id mem_cube_id is required")
+ if not delete_record_id:
+ raise ValueError("recover_memory_by_mem_cube_id delete_record_id is required")
+
+ memories = self._physical_table("memories", mem_cube_id)
+ conn = self._get_conn()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ f"""
+ UPDATE {memories}
+ SET properties = properties || %s::jsonb,
+ deleted_at = NULL,
+ updated_at = NOW()
+ WHERE user_name = %s
+ AND properties->>'delete_record_id' = %s
+ AND properties->>'status' = 'deleted'
+ """,
+ (
+ json.dumps(
+ {
+ "status": "activated",
+ "delete_time": "",
+ "delete_record_id": "",
+ }
+ ),
+ mem_cube_id,
+ delete_record_id,
+ ),
+ )
+ return max(int(cur.rowcount or 0), 0)
+ finally:
+ self._put_conn(conn)
+
+ def drop_database(self) -> None:
+ """Compatibility operation: clear only the configured tenant.
+
+ The standard PostgreSQL backend is shared, so dropping the physical
+ database or schema would delete other tenants.
+ """
+ self.clear(user_name=self.user_name)
+
def deduplicate_nodes(self) -> None:
"""Not implemented - handled at application level."""
@@ -1012,6 +2397,8 @@ def get_grouped_counts(
user_name = user_name or self.user_name
if not group_fields:
raise ValueError("group_fields cannot be empty")
+ if any(not self._is_safe_field_name(field) for field in group_fields):
+ raise ValueError("group_fields contains an invalid field name")
# Build SELECT and GROUP BY clauses
# Fields come from JSONB properties column
@@ -1033,10 +2420,11 @@ def get_grouped_counts(
query_params.extend(params.values())
where_sql = " AND ".join(conditions)
+ memories = self._physical_table("memories", user_name)
query = f"""
SELECT {select_fields}, COUNT(*) AS count
- FROM {self.schema}.memories
+ FROM {memories}
WHERE {where_sql}
GROUP BY {group_by}
"""
@@ -1067,13 +2455,15 @@ def merge_nodes(self, id1: str, id2: str) -> str:
def clear(self, user_name: str | None = None) -> None:
"""Clear all data for user."""
user_name = user_name or self.user_name
+ memories = self._physical_table("memories", user_name)
+ edges = self._physical_table("edges", user_name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
# Get all node IDs for user
cur.execute(
f"""
- SELECT id FROM {self.schema}.memories WHERE user_name = %s
+ SELECT id FROM {memories} WHERE user_name = %s
""",
(user_name,),
)
@@ -1083,16 +2473,17 @@ def clear(self, user_name: str | None = None) -> None:
# Delete edges
cur.execute(
f"""
- DELETE FROM {self.schema}.edges
- WHERE source_id = ANY(%s) OR target_id = ANY(%s)
+ DELETE FROM {edges}
+ WHERE user_name = %s
+ AND (source_id = ANY(%s) OR target_id = ANY(%s))
""",
- (ids, ids),
+ (user_name, ids, ids),
)
# Delete nodes
cur.execute(
f"""
- DELETE FROM {self.schema}.memories WHERE user_name = %s
+ DELETE FROM {memories} WHERE user_name = %s
""",
(user_name,),
)
@@ -1100,36 +2491,148 @@ def clear(self, user_name: str | None = None) -> None:
finally:
self._put_conn(conn)
- def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, Any]:
- """Export all data."""
- user_name = kwargs.get("user_name") or self.user_name
+ def export_graph(
+ self,
+ user_name: str | None = None,
+ include_embedding: bool = False,
+ user_id: str | None = None,
+ page: int | None = None,
+ page_size: int | None = None,
+ filter: dict | None = None,
+ memory_type: list[str] | None = None,
+ status: list[str] | None = None,
+ **kwargs,
+ ) -> dict[str, Any]:
+ """Export memories with the same filters and pagination as PolarDB."""
+
+ extracted_object_type: str | None = None
+ extracted_mem_cube_id: str | None = None
+
+ def extract_special_values(value: Any) -> Any:
+ nonlocal extracted_object_type, extracted_mem_cube_id
+ if not isinstance(value, dict):
+ return value
+ cleaned: dict[str, Any] = {}
+ for key, item in value.items():
+ if key in {"and", "or"} and isinstance(item, list):
+ children = [extract_special_values(child) for child in item]
+ children = [child for child in children if child not in (None, {}, [])]
+ if children:
+ cleaned[key] = children
+ elif key == "object_type" and isinstance(item, str):
+ extracted_object_type = extracted_object_type or item
+ elif key == "mem_cube_id" and isinstance(item, str):
+ extracted_mem_cube_id = extracted_mem_cube_id or item
+ else:
+ cleaned[key] = item
+ return cleaned or None
+
+ filter_for_sql = extract_special_values(filter)
+ conditions: list[str] = []
+ params: list[Any] = []
+ has_object_filter = bool(extracted_object_type and extracted_mem_cube_id)
+ object_type = (extracted_object_type or "").strip().lower()
+
+ if has_object_filter:
+ if object_type == "user":
+ conditions.append("user_name <> %s")
+ params.append(extracted_mem_cube_id)
+ elif object_type == "public":
+ conditions.append("user_name = %s")
+ params.append(extracted_mem_cube_id)
+ elif user_name:
+ conditions.append("user_name = %s")
+ params.append(user_name)
+
+ effective_user_id = user_id or getattr(self.config, "user_id", None)
+ if effective_user_id:
+ conditions.append("properties->>'user_id' = %s")
+ params.append(effective_user_id)
+ if memory_type:
+ conditions.append("properties->>'memory_type' = ANY(%s)")
+ params.append([str(item) for item in memory_type])
+ if status:
+ conditions.append("properties->>'status' = ANY(%s)")
+ params.append([str(item) for item in status])
+ else:
+ conditions.append("COALESCE(properties->>'status', 'activated') <> 'deleted'")
+ conditions.append("deleted_at IS NULL")
+ if filter_for_sql:
+ filter_sql = self._build_filter_where_clause(filter_for_sql, params)
+ if filter_sql:
+ conditions.append(f"({filter_sql})")
+
+ where_clause = " AND ".join(conditions) if conditions else "TRUE"
+ target_user = user_name
+ if has_object_filter and object_type == "public":
+ target_user = extracted_mem_cube_id
+ elif has_object_filter and object_type == "user":
+ target_user = None
+ memories = self._physical_table("memories", target_user)
+ use_pagination = page is not None and page_size is not None
+ if use_pagination:
+ page = max(int(page), 1)
+ page_size = max(int(page_size), 1)
+ offset = (page - 1) * page_size
+
conn = self._get_conn()
try:
with conn.cursor() as cur:
- # Get nodes
+ cur.execute(
+ f"SELECT COUNT(*) FROM {memories} WHERE {where_clause}",
+ params,
+ )
+ count_row = cur.fetchone()
+ total_nodes = int(count_row[0]) if count_row else 0
+
cols = "id, memory, properties, created_at, updated_at"
if include_embedding:
cols += ", embedding"
+ cols += ", user_name"
+ pagination_sql = ""
+ data_params = list(params)
+ if use_pagination:
+ pagination_sql = "LIMIT %s OFFSET %s"
+ data_params.extend([page_size, offset])
cur.execute(
f"""
- SELECT {cols} FROM {self.schema}.memories
- WHERE user_name = %s
- ORDER BY created_at DESC
- """,
- (user_name,),
+ SELECT {cols}
+ FROM {memories}
+ WHERE {where_clause}
+ ORDER BY created_at DESC NULLS LAST, id DESC
+ {pagination_sql}
+ """,
+ data_params,
)
nodes = [self._parse_row(row, include_embedding) for row in cur.fetchall()]
- # Get edges
node_ids = [n["id"] for n in nodes]
- if node_ids:
+ node_user_names = [
+ n.get("metadata", {}).get("user_name") or user_name for n in nodes
+ ]
+ if node_ids and all(node_user_names):
+ edges_table = self._tenant_relation(
+ "edges",
+ list(dict.fromkeys(str(name) for name in node_user_names if name)),
+ alias="e",
+ conn=conn,
+ )
cur.execute(
f"""
- SELECT source_id, target_id, edge_type
- FROM {self.schema}.edges
- WHERE source_id = ANY(%s) OR target_id = ANY(%s)
- """,
- (node_ids, node_ids),
+ WITH selected_nodes(user_name, node_id) AS (
+ SELECT * FROM unnest(%s::text[], %s::text[])
+ )
+ SELECT e.source_id, e.target_id, e.edge_type
+ FROM {edges_table}
+ JOIN selected_nodes AS source_node
+ ON source_node.user_name = e.user_name
+ AND source_node.node_id = e.source_id
+ JOIN selected_nodes AS target_node
+ ON target_node.user_name = e.user_name
+ AND target_node.node_id = e.target_id
+ ORDER BY e.id
+ """,
+ (node_user_names, node_ids),
)
edges = [
{"source": row[0], "target": row[1], "type": row[2]}
@@ -1141,7 +2644,7 @@ def export_graph(self, include_embedding: bool = False, **kwargs) -> dict[str, A
return {
"nodes": nodes,
"edges": edges,
- "total_nodes": len(nodes),
+ "total_nodes": total_nodes,
"total_edges": len(edges),
}
finally:
@@ -1164,6 +2667,7 @@ def import_graph(self, data: dict[str, Any], user_name: str | None = None) -> No
source_id=edge["source"],
target_id=edge["target"],
type=edge["type"],
+ user_name=user_name,
)
def close(self):
diff --git a/src/memos/llms/openai_new.py b/src/memos/llms/openai_new.py
index 766a17fda..50c87b78f 100644
--- a/src/memos/llms/openai_new.py
+++ b/src/memos/llms/openai_new.py
@@ -50,7 +50,7 @@ def generate(self, messages: MessageList, **kwargs) -> str:
output_reasoning = [
item for item in response.output if isinstance(item, ResponseReasoningItem)
]
- summary = output_reasoning[0].summary
+ summary = output_reasoning[0].summary if output_reasoning else None
if self.config.remove_think_prefix:
return remove_thinking_tags(output_text)
@@ -138,7 +138,7 @@ def generate(self, messages: MessageList, **kwargs) -> str:
output_reasoning = [
item for item in response.output if isinstance(item, ResponseReasoningItem)
]
- summary = output_reasoning[0].summary
+ summary = output_reasoning[0].summary if output_reasoning else None
if self.config.remove_think_prefix:
return remove_thinking_tags(output_text)
diff --git a/src/memos/mem_feedback/feedback.py b/src/memos/mem_feedback/feedback.py
index 28f495afb..18ebb0b49 100644
--- a/src/memos/mem_feedback/feedback.py
+++ b/src/memos/mem_feedback/feedback.py
@@ -657,11 +657,6 @@ def _info_comparison(self, memory: TextualMemoryItem, _info: dict, include_keys:
def _retrieve(self, query: str, info=None, top_k=20, user_name=None):
"""Retrieve memory items"""
- def check_has_edges(mem_item: TextualMemoryItem) -> tuple[TextualMemoryItem, bool]:
- """Check if a memory item has edges."""
- edges = self.searcher.graph_store.get_edges(mem_item.id, user_name=user_name)
- return (mem_item, len(edges) == 0)
-
logger.info(f"[feedback _retrieve] query: {query}, user_name: {user_name}")
text_mems = self.searcher.search(
query=query,
@@ -686,22 +681,16 @@ def check_has_edges(mem_item: TextualMemoryItem) -> tuple[TextualMemoryItem, boo
pref_mems = [item[0] for item in pref_mems if float(item[1]) > 0.01]
text_mems.extend(pref_mems)
- # Memory with edges is not modified by feedback
- retrieved_mems = []
- with ContextThreadPoolExecutor(max_workers=10) as executor:
- futures = {executor.submit(check_has_edges, item): item for item in text_mems}
- for future in concurrent.futures.as_completed(futures):
- try:
- mem_item, has_no_edges = future.result()
- if has_no_edges:
- retrieved_mems.append(mem_item)
- except Exception as e:
- logger.error(f"[0107 Feedback Core: _retrieve] Error checking edges: {e}")
+ retrieved_mems = [
+ item
+ for item in text_mems
+ if not (getattr(item.metadata, "knowledge_material_ids", None) or [])
+ ]
if len(retrieved_mems) < len(text_mems):
logger.info(
f"[0107 Feedback Core: _retrieve] {len(text_mems) - len(retrieved_mems)} "
- f"text memories are not modified by feedback due to edges."
+ f"knowledge memories are not modified by feedback."
)
return retrieved_mems
diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py
index c46331f1d..eb226c27d 100644
--- a/src/memos/mem_reader/multi_modal_struct.py
+++ b/src/memos/mem_reader/multi_modal_struct.py
@@ -769,30 +769,52 @@ def _process_one_item(
except Exception as e:
logger.error(f"[MultiModalFine] parse error: {e}")
- # save rawfile node
- if self.save_rawfile and prompt_type == "doc" and len(fine_items) > 0:
- rawfile_chunk = mem_str
- file_info = fine_items[0].metadata.sources[0].file_info
- source = self.multi_modal_parser.file_content_parser.create_source(
- message={"file": file_info},
- info=info_per_item,
- chunk_index=chunk_idx,
- chunk_total=total_chunks,
- chunk_content="",
- )
- rawfile_node = self._make_memory_item(
- value=rawfile_chunk,
- info=info_per_item,
- memory_type="RawFileMemory",
- tags=[
- "mode:fine",
- "multimodal:file",
- f"chunk:{chunk_idx + 1}/{total_chunks}",
- ],
- sources=[source],
- )
- rawfile_node.metadata.summary_ids = [mem_node.id for mem_node in fine_items]
- fine_items.append(rawfile_node)
+ # Save the original file chunk independently from prompt selection.
+ file_source = next(
+ (
+ source
+ for source in sources
+ if (
+ source.get("type")
+ if isinstance(source, dict)
+ else getattr(source, "type", None)
+ )
+ == "file"
+ ),
+ None,
+ )
+ if self.save_rawfile and file_source is not None and fine_items:
+ if isinstance(file_source, dict):
+ file_info = file_source.get("file_info") or {}
+ file_content = file_source.get("content")
+ else:
+ file_info = file_source.file_info or {}
+ file_content = file_source.content
+
+ rawfile_chunk = file_content or file_info.get("file_data", "")
+ if rawfile_chunk:
+ source = self.multi_modal_parser.file_content_parser.create_source(
+ message={"file": file_info},
+ info=info_per_item,
+ chunk_index=chunk_idx,
+ chunk_total=total_chunks,
+ chunk_content=rawfile_chunk,
+ )
+ rawfile_node = self._make_memory_item(
+ value=rawfile_chunk,
+ info=info_per_item,
+ memory_type="RawFileMemory",
+ tags=[
+ "mode:fine",
+ "multimodal:file",
+ f"chunk:{chunk_idx + 1}/{total_chunks}",
+ ],
+ sources=[source],
+ )
+ rawfile_node.metadata.summary_ids = [mem_node.id for mem_node in fine_items]
+ fine_items.append(rawfile_node)
+ else:
+ logger.warning("[RawFile] File source has no content; skipping raw-file node")
enriched_items = trigger_hook(
H.MEMORY_ITEMS_AFTER_FINE_EXTRACT,
items=fine_items,
diff --git a/src/memos/mem_reader/read_multi_modal/image_parser.py b/src/memos/mem_reader/read_multi_modal/image_parser.py
index ec19ba304..40561c430 100644
--- a/src/memos/mem_reader/read_multi_modal/image_parser.py
+++ b/src/memos/mem_reader/read_multi_modal/image_parser.py
@@ -27,6 +27,18 @@
logger = get_logger(__name__)
+def is_sendable_image_url(url: str) -> bool:
+ """Return whether a vision API can accept this image address.
+
+ Accepted forms are http(s) URLs and ``data:image/...;base64,...`` payloads.
+ Relative paths such as ``images/.jpg`` are rejected.
+ """
+ candidate = url.strip().lower()
+ if candidate.startswith(("http://", "https://")):
+ return True
+ return candidate.startswith("data:image/") and "base64," in candidate
+
+
class ImageParser(BaseMessageParser):
"""Parser for image_url content parts."""
@@ -177,6 +189,13 @@ def parse_fine(
logger.warning("[ImageParser] No image URL found in message")
return []
+ if not is_sendable_image_url(url):
+ logger.warning(
+ "[ImageParser] Skip image URL that cannot be sent to the vision model: %s",
+ url[:200],
+ )
+ return []
+
# Create source for this image
source = self.create_source(message, info)
diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py
index a954794b7..5e663c99e 100644
--- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py
+++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py
@@ -209,7 +209,7 @@ def _process_memories_with_reader(
for memory in flattened_memories
if memory.metadata.memory_type == "RawFileMemory"
]
- text_mem.add_rawfile_nodes_n_edges(
+ text_mem.add_rawfile_nodes(
raw_file_mem_group,
enhanced_mem_ids,
user_id=user_id,
@@ -328,9 +328,7 @@ def _process_memories_with_reader(
add_meta_legacy: list[dict] = []
update_content_legacy: list[dict] = []
update_meta_legacy: list[dict] = []
- for item_id, item in zip(
- enhanced_mem_ids, flattened_memories, strict=False
- ):
+ for item_id, item in zip(enhanced_mem_ids, mem_group, strict=False):
key = getattr(item.metadata, "key", None) or transform_name_to_key(
name=item.memory
)
diff --git a/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py b/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py
index 3123bfdee..e39f4bbd9 100644
--- a/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py
+++ b/src/memos/mem_scheduler/webservice_modules/rabbitmq_service.py
@@ -61,7 +61,7 @@ def __init__(self):
self.rabbitmq_conn_sleep_seconds = 1
# Thread management
- self._rabbitmq_io_loop_thread = None # For IOLoop execution
+ self._io_loop_thread = None # For IOLoop execution
self._rabbitmq_stop_flag = False # Graceful shutdown flag
# Use RLock because publishing may trigger initialization, which also grabs the lock.
self._rabbitmq_lock = threading.RLock()
diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py
index 23474cc20..a874ac77b 100644
--- a/src/memos/memories/textual/item.py
+++ b/src/memos/memories/textual/item.py
@@ -159,6 +159,16 @@ class TextualMemoryMetadata(BaseModel):
description="Internal algorithm metadata reserved for system use.",
)
+ @field_validator("internal_info", mode="before")
+ @classmethod
+ def _decode_internal_info(cls, value: Any) -> Any:
+ # Neo4j stores dictionaries as JSON strings; decode them before model validation.
+ if isinstance(value, str):
+ value = json.loads(value)
+ if not isinstance(value, dict):
+ raise ValueError("internal_info JSON must decode to an object")
+ return value
+
model_config = ConfigDict(extra="allow")
covered_history: Any | None = Field(
@@ -212,6 +222,14 @@ class TreeNodeTextualMemoryMetadata(TextualMemoryMetadata):
default_factory=list,
description="The ids of the files associated with the memory.",
)
+ knowledge_material_ids: list[str] = Field(
+ default_factory=list,
+ description="RawFileMemory ids that support this knowledge summary.",
+ )
+ knowledge_summary_ids: list[str] = Field(
+ default_factory=list,
+ description="Knowledge summary ids extracted from this raw file memory.",
+ )
@field_validator("sources", mode="before")
@classmethod
diff --git a/src/memos/memories/textual/tree.py b/src/memos/memories/textual/tree.py
index 426a2359b..c9d74d43f 100644
--- a/src/memos/memories/textual/tree.py
+++ b/src/memos/memories/textual/tree.py
@@ -399,10 +399,13 @@ def get_all(
)
return graph_output
- def delete(self, memory_ids: list[str], user_name: str | None = None) -> None:
+ def delete(self, memory_ids: str | list[str], user_name: str | None = None) -> None:
"""Hard delete: permanently remove nodes and their edges from the graph."""
if not memory_ids:
return
+ if isinstance(memory_ids, str):
+ memory_ids = [memory_ids]
+ self._unlink_knowledge_relations(memory_ids, user_name=user_name)
for mid in memory_ids:
try:
self.graph_store.delete_node(mid, user_name=user_name)
@@ -412,6 +415,7 @@ def delete(self, memory_ids: list[str], user_name: str | None = None) -> None:
def delete_by_memory_ids(self, memory_ids: list[str]) -> None:
"""Delete memories by memory_ids."""
try:
+ self._unlink_knowledge_relations(memory_ids)
self.graph_store.delete_node_by_prams(memory_ids=memory_ids)
except Exception as e:
logger.error(f"An error occurred while deleting memories by memory_ids: {e}")
@@ -432,10 +436,71 @@ def delete_by_filter(
filter: dict | None = None,
) -> None:
"""Delete memories by filter."""
+ if file_ids:
+ user_names = writable_cube_ids or [None]
+ for user_name in user_names:
+ memory_ids: set[str] = set()
+ for file_id in file_ids:
+ memory_ids.update(
+ self.graph_store.get_by_metadata(
+ [{"field": "file_ids", "op": "contains", "value": file_id}],
+ user_name=user_name,
+ )
+ )
+ self._unlink_knowledge_relations(list(memory_ids), user_name=user_name)
self.graph_store.delete_node_by_prams(
writable_cube_ids=writable_cube_ids, file_ids=file_ids, filter=filter
)
+ @staticmethod
+ def _node_metadata(node: dict[str, Any] | TextualMemoryItem) -> dict[str, Any]:
+ if isinstance(node, TextualMemoryItem):
+ return node.metadata.model_dump()
+ return node.get("metadata") or {}
+
+ def _get_node_map(
+ self, memory_ids: list[str] | set[str], user_name: str | None = None
+ ) -> dict[str, dict[str, Any] | TextualMemoryItem]:
+ if not memory_ids:
+ return {}
+ nodes = self.graph_store.get_nodes(ids=list(memory_ids), user_name=user_name) or []
+ return {
+ node.id if isinstance(node, TextualMemoryItem) else node["id"]: node for node in nodes
+ }
+
+ def _unlink_knowledge_relations(
+ self, memory_ids: list[str], user_name: str | None = None
+ ) -> None:
+ """Remove reverse knowledge ids before deleting or deactivating nodes."""
+ deleting_ids = set(memory_ids)
+ deleting_nodes = self._get_node_map(deleting_ids, user_name=user_name)
+ removals: dict[str, dict[str, set[str]]] = {}
+
+ for memory_id, node in deleting_nodes.items():
+ metadata = self._node_metadata(node)
+ for material_id in metadata.get("knowledge_material_ids") or []:
+ if material_id not in deleting_ids:
+ removals.setdefault(material_id, {}).setdefault(
+ "knowledge_summary_ids", set()
+ ).add(memory_id)
+ for summary_id in metadata.get("knowledge_summary_ids") or []:
+ if summary_id not in deleting_ids:
+ removals.setdefault(summary_id, {}).setdefault(
+ "knowledge_material_ids", set()
+ ).add(memory_id)
+
+ related_nodes = self._get_node_map(set(removals), user_name=user_name)
+ for related_id, fields in removals.items():
+ node = related_nodes.get(related_id)
+ if node is None:
+ continue
+ metadata = self._node_metadata(node)
+ updates = {
+ field: [value for value in metadata.get(field) or [] if value not in removed_ids]
+ for field, removed_ids in fields.items()
+ }
+ self.graph_store.update_node(related_id, updates, user_name=user_name)
+
def load(self, dir: str, user_name: str | None = None) -> None:
try:
memory_file = os.path.join(dir, self.config.memory_filename)
@@ -593,6 +658,109 @@ def add_rawfile_nodes_n_edges(
f"[RawFile] Time taken to add edges: {end_time - start_time} seconds for {len(from_ids)} edges"
)
+ def add_rawfile_nodes(
+ self,
+ raw_file_mem_group: list[TextualMemoryItem],
+ mem_ids: list[str],
+ user_id: str | None = None,
+ user_name: str | None = None,
+ ) -> None:
+ """Persist raw-file nodes and their knowledge relationships without graph edges."""
+ valid_summary_ids = set(mem_ids)
+ summary_to_material_ids: dict[str, list[str]] = {}
+
+ for raw_file_mem in raw_file_mem_group:
+ summary_ids = list(
+ dict.fromkeys(
+ summary_id
+ for summary_id in getattr(raw_file_mem.metadata, "summary_ids", []) or []
+ if summary_id in valid_summary_ids
+ )
+ )
+ raw_file_mem.metadata.knowledge_summary_ids = summary_ids
+ for summary_id in summary_ids:
+ summary_to_material_ids.setdefault(summary_id, []).append(raw_file_mem.id)
+
+ self.add(raw_file_mem_group, user_name=user_name)
+ summary_nodes = self._get_node_map(set(summary_to_material_ids), user_name=user_name)
+
+ merged_from_by_summary: dict[str, list[str]] = {}
+ old_summary_ids: set[str] = set()
+ for summary_id, node in summary_nodes.items():
+ info = self._node_metadata(node).get("info") or {}
+ merged_from = info.get("merged_from") or []
+ if isinstance(merged_from, str):
+ merged_from = [merged_from]
+ merged_from_by_summary[summary_id] = list(dict.fromkeys(merged_from))
+ old_summary_ids.update(merged_from)
+
+ old_summary_nodes = self._get_node_map(old_summary_ids, user_name=user_name)
+ material_relinks: dict[str, dict[str, Any]] = {}
+
+ for summary_id, direct_material_ids in summary_to_material_ids.items():
+ summary_node = summary_nodes.get(summary_id)
+ if summary_node is None:
+ logger.warning(
+ "Knowledge summary %s was not found after raw-file write", summary_id
+ )
+ continue
+
+ metadata = self._node_metadata(summary_node)
+ merged_from = merged_from_by_summary.get(summary_id, [])
+ inherited_material_ids: list[str] = []
+ for old_summary_id in merged_from:
+ old_summary = old_summary_nodes.get(old_summary_id)
+ if old_summary is not None:
+ inherited_material_ids.extend(
+ self._node_metadata(old_summary).get("knowledge_material_ids") or []
+ )
+
+ material_ids = list(
+ dict.fromkeys(
+ [
+ *(metadata.get("knowledge_material_ids") or []),
+ *direct_material_ids,
+ *inherited_material_ids,
+ ]
+ )
+ )
+ self.graph_store.update_node(
+ summary_id,
+ {"knowledge_material_ids": material_ids},
+ user_name=user_name,
+ )
+
+ for material_id in inherited_material_ids:
+ relink = material_relinks.setdefault(material_id, {"remove": set(), "add": []})
+ relink["remove"].update(merged_from)
+ relink["add"].append(summary_id)
+
+ material_nodes = self._get_node_map(set(material_relinks), user_name=user_name)
+ for material_id, relink in material_relinks.items():
+ material_node = material_nodes.get(material_id)
+ if material_node is None:
+ continue
+ current_summary_ids = (
+ self._node_metadata(material_node).get("knowledge_summary_ids") or []
+ )
+ updated_summary_ids = [
+ summary_id
+ for summary_id in current_summary_ids
+ if summary_id not in relink["remove"]
+ ]
+ updated_summary_ids.extend(relink["add"])
+ self.graph_store.update_node(
+ material_id,
+ {"knowledge_summary_ids": list(dict.fromkeys(updated_summary_ids))},
+ user_name=user_name,
+ )
+
+ logger.info(
+ "[RawFile] Added %s chunks with ID mappings for user %s",
+ len(raw_file_mem_group),
+ user_id,
+ )
+
def add_graph_edges(
self, from_ids: list[str], to_ids: list[str], types: list[str], user_name: str | None = None
) -> None:
@@ -624,6 +792,7 @@ def soft_delete(
user_name: str,
evolve_to_ids: list[str] | None = None,
) -> None:
+ self._unlink_knowledge_relations(memory_ids, user_name=user_name)
# for ruff check...
if not evolve_to_ids:
update_fields = {"status": "deleted"}
diff --git a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py
index 5c9cce78e..b44d28218 100644
--- a/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py
+++ b/src/memos/memories/textual/tree_text_memory/retrieve/searcher.py
@@ -3,8 +3,6 @@
import re
import traceback
-from concurrent.futures import as_completed
-
from memos.context.context import ContextThreadPoolExecutor
from memos.embedders.factory import OllamaEmbedder
from memos.graph_dbs.factory import Neo4jGraphDB
@@ -1282,7 +1280,7 @@ def _sort_and_trim(
@timed
def _deduplicate_rawfile_results(self, results, user_name: str | None = None):
"""
- Deduplicate rawfile related memories by edge
+ Deduplicate raw-file results using their persisted summary ids.
"""
if not results:
return results
@@ -1292,29 +1290,10 @@ def _deduplicate_rawfile_results(self, results, user_name: str | None = None):
if not rawfile_items:
return results
- with ContextThreadPoolExecutor(max_workers=min(len(rawfile_items), 10)) as executor:
- futures = [
- executor.submit(
- self.graph_store.get_edges,
- rawfile_item.id,
- type="SUMMARY",
- direction="OUTGOING",
- user_name=user_name,
- )
- for rawfile_item in rawfile_items
- ]
- for future in as_completed(futures):
- try:
- edges = future.result()
- for edge in edges:
- summary_target_id = edge.get("to")
- if summary_target_id:
- summary_ids_to_remove.add(summary_target_id)
- logger.debug(
- f"[DEDUP] Marking summary node {summary_target_id} for removal (pointed by RawFileMemory)"
- )
- except Exception as e:
- logger.warning(f"[DEDUP] Failed to get summary target ids: {e}")
+ for rawfile_item in rawfile_items:
+ summary_ids_to_remove.update(
+ getattr(rawfile_item.metadata, "knowledge_summary_ids", None) or []
+ )
filtered_results = []
for item in results:
diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py
index 27399259a..0b9829b7f 100644
--- a/src/memos/multi_mem_cube/single_cube.py
+++ b/src/memos/multi_mem_cube/single_cube.py
@@ -441,22 +441,35 @@ def extract_edge_info(edges_info: list[dict], neighbor_relativity: float):
)
return edge_mems
+ def get_neighbor(chunk_target_id: str | None, neighbor_relativity: float):
+ if not chunk_target_id:
+ return None
+ item_neighbor = self.searcher.graph_store.get_node(chunk_target_id, user_name=user_name)
+ if not item_neighbor:
+ return None
+ item_neighbor_mem = TextualMemoryItem(**item_neighbor)
+ item_neighbor_mem.metadata.relativity = neighbor_relativity
+ self.logger.info("Add neighbor chunk by id: %s", chunk_target_id)
+ return item_neighbor_mem
+
final_items = []
if neighbor_discovery:
for item in search_results:
if item.metadata.memory_type == "RawFileMemory":
neighbor_relativity = item.metadata.relativity * 0.8
- preceding_info = self.searcher.graph_store.get_edges(
- item.id, type="PRECEDING", direction="OUTGOING", user_name=user_name
+ preceding = get_neighbor(
+ getattr(item.metadata, "preceding_id", None), neighbor_relativity
)
- final_items.extend(extract_edge_info(preceding_info, neighbor_relativity))
+ if preceding is not None:
+ final_items.append(preceding)
final_items.append(item)
- following_info = self.searcher.graph_store.get_edges(
- item.id, type="FOLLOWING", direction="OUTGOING", user_name=user_name
+ following = get_neighbor(
+ getattr(item.metadata, "following_id", None), neighbor_relativity
)
- final_items.extend(extract_edge_info(following_info, neighbor_relativity))
+ if following is not None:
+ final_items.append(following)
else:
final_items.append(item)
@@ -748,7 +761,7 @@ def _process_text_mem(
for memory in flattened_local
if memory.metadata.memory_type == "RawFileMemory"
]
- self.naive_mem_cube.text_mem.add_rawfile_nodes_n_edges(
+ self.naive_mem_cube.text_mem.add_rawfile_nodes(
raw_file_mem_group,
mem_ids_local,
user_id=add_req.user_id,
diff --git a/tests/api/test_playground_cube_recall.py b/tests/api/test_playground_cube_recall.py
new file mode 100644
index 000000000..4b1d8fd40
--- /dev/null
+++ b/tests/api/test_playground_cube_recall.py
@@ -0,0 +1,24 @@
+"""Playground recall must keep memories from every searched cube."""
+
+from memos.api.handlers.chat_handler import ChatHandler
+
+
+def test_playground_recall_keeps_memories_from_later_cubes():
+ search_data = {
+ "text_mem": [
+ {"cube_id": "knowledge-base", "memories": []},
+ {
+ "cube_id": "personal",
+ "memories": [{"id": "mem-1", "memory": "use event_id as the idempotency key"}],
+ },
+ ]
+ }
+
+ memories = ChatHandler._memories_from_cube_results(search_data)
+
+ assert [item["id"] for item in memories] == ["mem-1"]
+
+
+def test_playground_recall_returns_empty_when_search_data_missing():
+ assert ChatHandler._memories_from_cube_results(None) == []
+ assert ChatHandler._memories_from_cube_results({}) == []
diff --git a/tests/api/test_rate_limit_bearer.py b/tests/api/test_rate_limit_bearer.py
new file mode 100644
index 000000000..710346cf0
--- /dev/null
+++ b/tests/api/test_rate_limit_bearer.py
@@ -0,0 +1,78 @@
+"""Regression tests for issue #2428.
+
+``_get_client_key`` in ``src/memos/api/middleware/rate_limit.py`` checked
+whether ``request.headers['Authorization']`` started with ``krlk_`` — but
+real clients send ``Authorization: Bearer krlk_`` per RFC 6750, so
+the prefix check never matched and every request fell through to IP-based
+limiting. Behind a shared egress IP all keyed clients shared one bucket
+and the per-key rate-limit was dead code.
+
+The fix strips the ``Bearer `` scheme before testing the key prefix, so
+``Bearer krlk_`` correctly maps to ``ratelimit:key:``.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from memos.api.middleware.rate_limit import _get_client_key
+
+
+def _fake_request(authorization: str | None, *, client_host: str = "10.0.0.1"):
+ headers: dict[str, str] = {}
+ if authorization is not None:
+ headers["Authorization"] = authorization
+ return SimpleNamespace(
+ headers=headers,
+ client=SimpleNamespace(host=client_host),
+ )
+
+
+class TestGetClientKey:
+ def test_bearer_krlk_key_uses_per_key_bucket(self):
+ body = "krlk_" + "a" * 64
+ request = _fake_request("Bearer " + body)
+ key = _get_client_key(request)
+ assert key == "ratelimit:key:" + body[:20]
+ assert "10.0.0.1" not in key
+
+ def test_bearer_krlk_key_first_twenty_chars_of_actual_key(self):
+ body = "krlk_" + "f" * 64
+ request = _fake_request("Bearer " + body)
+ key = _get_client_key(request)
+ assert key == "ratelimit:key:" + body[:20]
+
+ def test_lowercase_bearer_scheme(self):
+ # RFC 7235: the scheme name is case-insensitive. We normalise to
+ # lowercase before the prefix check.
+ body = "krlk_" + "b" * 64
+ request = _fake_request("bearer " + body)
+ key = _get_client_key(request)
+ assert key == "ratelimit:key:" + body[:20]
+
+ def test_bearer_with_non_key_token_falls_back_to_ip(self):
+ request = _fake_request("Bearer some-jwt-token")
+ key = _get_client_key(request)
+ assert key.startswith("ratelimit:ip:")
+ assert "10.0.0.1" in key
+
+ def test_missing_authorization_header_falls_back_to_ip(self):
+ request = _fake_request(None)
+ key = _get_client_key(request)
+ assert key.startswith("ratelimit:ip:")
+ assert "10.0.0.1" in key
+
+ def test_x_forwarded_for_header_preferred_over_ip(self):
+ request = _fake_request(None, client_host="203.0.113.1")
+ request.headers["X-Forwarded-For"] = "198.51.100.1, 10.0.0.1"
+ key = _get_client_key(request)
+ assert key == "ratelimit:ip:198.51.100.1"
+
+ def test_krlk_prefix_without_bearer_scheme_still_matches(self):
+ # Backwards compatibility: clients that sent a bare krlk_ header
+ # (before the Bearer scheme was added) must still match.
+ body = "krlk_" + "c" * 64
+ request = _fake_request(body)
+ key = _get_client_key(request)
+ assert key == "ratelimit:key:" + body[:20]
diff --git a/tests/api/test_validate_key_format.py b/tests/api/test_validate_key_format.py
new file mode 100644
index 000000000..757d2519a
--- /dev/null
+++ b/tests/api/test_validate_key_format.py
@@ -0,0 +1,99 @@
+"""Regression tests for issue #2427.
+
+``validate_key_format`` exists in two copies — ``src/memos/api/utils/api_keys.py``
+(the canonical home) and ``src/memos/api/middleware/auth.py`` (the
+middleware gating every authenticated request). Both check the hex part
+with ``int(hex_part, 16)``, which accepts non-canonical forms a generated
+key can never have:
+
+- leading ``+`` / ``-`` (sign)
+- ``0x`` / ``0X`` prefix
+- ``_`` separators (PEP 515)
+- surrounding whitespace
+
+So the documented format ``krlk_<64-hex-chars>`` is not actually enforced:
+keys with embedded underscores, signs, or ``0x`` prefixes pass the format
+gate and are hashed/looked up as-is, bypassing the canonical form that
+``generate_api_key()`` produces.
+
+These tests pin the new contract on both validator copies:
+
+- the hex part must match ``[0-9a-f]{64}`` exactly
+- the only valid form is the canonical lowercase hex string
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from memos.api.middleware.auth import validate_key_format as validate_in_auth
+from memos.api.utils.api_keys import generate_api_key, validate_key_format
+
+
+CANONICAL = "krlk_" + "0123456789abcdef" * 4 # 64 hex chars, lowercase
+
+
+class TestValidateKeyFormatUtils:
+ def test_canonical_key_passes(self):
+ assert validate_key_format(CANONICAL) is True
+
+ def test_generated_key_always_passes(self):
+ # The validator must accept every key that generate_api_key()
+ # produces, since that is the documented format.
+ for _ in range(20):
+ assert validate_key_format(generate_api_key().key) is True
+
+ @pytest.mark.parametrize(
+ "bad_key",
+ [
+ # Leading sign
+ "krlk_" + "+" + "a" * 63,
+ "krlk_" + "-" + "a" * 63,
+ # 0x prefix
+ "krlk_" + "0x" + "a" * 62,
+ "krlk_" + "0X" + "a" * 62,
+ # Underscore separator (PEP 515)
+ "krlk_" + "a" * 32 + "_" + "b" * 31,
+ # Uppercase hex (the generator emits lowercase only)
+ "krlk_" + "A" * 64,
+ # Surrounding whitespace
+ " krlk_" + "a" * 64,
+ "krlk_" + "a" * 64 + " ",
+ # Wrong total length
+ "krlk_" + "a" * 63,
+ "krlk_" + "a" * 65,
+ # Empty hex part
+ "krlk_",
+ # Missing prefix
+ "0123456789abcdef" * 4,
+ "Bearer " + CANONICAL,
+ # Non-string
+ "",
+ ],
+ )
+ def test_non_canonical_keys_rejected(self, bad_key):
+ assert validate_key_format(bad_key) is False, (
+ f"validate_key_format({bad_key!r}) should return False"
+ )
+
+
+class TestAuthMiddlewareValidator:
+ """The middleware copy must enforce the same contract as the canonical
+ one — issue #2427 documents both copies accepting non-canonical hex."""
+
+ def test_canonical_key_passes(self):
+ assert validate_in_auth(CANONICAL) is True
+
+ def test_signed_hex_rejected(self):
+ # The most surprising bit: ``int("+a..", 16)`` succeeds, so a signed
+ # hex passes the middleware's validator today.
+ assert validate_in_auth("krlk_" + "+" + "a" * 63) is False
+
+ def test_underscore_rejected(self):
+ assert validate_in_auth("krlk_" + "a" * 32 + "_" + "b" * 31) is False
+
+ def test_uppercase_rejected(self):
+ assert validate_in_auth("krlk_" + "A" * 64) is False
+
+ def test_0x_prefix_rejected(self):
+ assert validate_in_auth("krlk_" + "0x" + "a" * 62) is False
diff --git a/tests/graph_dbs/test_postgres_400_compat.py b/tests/graph_dbs/test_postgres_400_compat.py
new file mode 100644
index 000000000..bd23fd8ff
--- /dev/null
+++ b/tests/graph_dbs/test_postgres_400_compat.py
@@ -0,0 +1,391 @@
+import json
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from memos.graph_dbs.postgres import PostgresGraphDB
+
+
+@pytest.fixture
+def postgres_db():
+ db = PostgresGraphDB.__new__(PostgresGraphDB)
+ db.schema = "memos"
+ db.db_name = "memos"
+ db.user_name = "tenant-a"
+ db._pool_closed = False
+ db.config = SimpleNamespace(
+ embedding_dimension=3,
+ fulltext_config="simple",
+ vector_index_type="hnsw",
+ hnsw_m=16,
+ hnsw_ef_construction=64,
+ ivfflat_lists=100,
+ shard_count=400,
+ )
+
+ cursor = MagicMock()
+ connection = MagicMock()
+ connection.cursor.return_value.__enter__.return_value = cursor
+ db._get_conn = MagicMock(return_value=connection)
+ db._put_conn = MagicMock()
+ db._partition_remainders = {
+ "tenant-a": 17,
+ "tenant-b": 17,
+ "kb-1": 17,
+ }
+ return db, cursor
+
+
+def _executed_sql(cursor: MagicMock) -> str:
+ return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list)
+
+
+def test_schema_does_not_create_profile_storage(postgres_db):
+ db, cursor = postgres_db
+
+ db._init_schema()
+
+ sql = _executed_sql(cursor)
+ assert "profile_templates" not in sql
+ assert "profile_instances" not in sql
+
+
+def test_schema_creates_400_md5_memory_shards(postgres_db):
+ db, cursor = postgres_db
+
+ db._init_schema()
+
+ sql = _executed_sql(cursor)
+ assert "PARTITION BY HASH" not in sql
+ assert '"Memory"' in sql
+ assert "embedding halfvec(" in sql
+ assert "memos_graph_0" in sql
+ assert "memos_graph_399" in sql
+ assert "MODULUS" not in sql
+ assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 401
+ assert "PRIMARY KEY (user_name, id)" in sql
+
+
+def test_schema_rejects_legacy_nonpartitioned_parent(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchall.return_value = [("memories", "r")]
+
+ with pytest.raises(RuntimeError, match="configure a new MEMOS_SCHEMA"):
+ db._init_schema()
+
+
+def test_schema_skips_partition_ddl_when_layout_is_complete(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchone.return_value = (400,)
+
+ db._init_schema()
+
+ sql = _executed_sql(cursor)
+ assert sql.count("CREATE SCHEMA IF NOT EXISTS") == 1
+ assert "memos_graph_0" not in sql
+
+
+def test_node_upsert_targets_composite_partition_key(postgres_db):
+ db, cursor = postgres_db
+
+ db.add_node(
+ "memory-1",
+ "memory",
+ {"embedding": [1.0, 0.0, 0.0]},
+ user_name="tenant-a",
+ )
+
+ assert "ON CONFLICT (user_name, id)" in _executed_sql(cursor)
+
+
+def test_batch_insert_uses_one_statement_and_polardb_properties(postgres_db):
+ db, cursor = postgres_db
+
+ db.add_nodes_batch(
+ [
+ {"id": "m1", "memory": "one", "metadata": {"embedding": [1.0, 0.0, 0.0]}},
+ {"id": "m2", "memory": "two", "metadata": {"embedding": [0.0, 1.0, 0.0]}},
+ ],
+ user_name="tenant-a",
+ )
+
+ assert cursor.execute.call_count == 1
+ query, params = cursor.execute.call_args.args
+ assert query.count("::halfvec(") == 2
+ assert "ON CONFLICT (user_name, id)" in query
+ first_properties = json.loads(params[2])
+ second_properties = json.loads(params[9])
+ assert first_properties["id"] == "m1"
+ assert first_properties["memory"] == "one"
+ assert second_properties["id"] == "m2"
+ assert second_properties["memory"] == "two"
+ assert "embedding" not in first_properties
+
+
+def test_node_upsert_persists_tenant_in_metadata(postgres_db):
+ db, cursor = postgres_db
+
+ db.add_node(
+ "memory-1",
+ "memory",
+ {"embedding": [1.0, 0.0, 0.0]},
+ user_name="tenant-a",
+ )
+
+ _, params = cursor.execute.call_args.args
+ properties = json.loads(params[2])
+ assert properties["user_name"] == "tenant-a"
+ assert properties["delete_time"] == ""
+ assert properties["delete_record_id"] == ""
+
+
+def test_batch_node_metadata_matches_polardb_shape(postgres_db):
+ db, _ = postgres_db
+ now = datetime.now(timezone.utc)
+
+ node = db._parse_row(
+ (
+ "memory-1",
+ "service HTTP comparison memory",
+ {"status": "activated"},
+ now,
+ now,
+ )
+ )
+
+ assert node["metadata"]["id"] == "memory-1"
+ assert node["metadata"]["memory"] == "service HTTP comparison memory"
+
+
+def test_parse_row_restores_tenant_and_deserializes_sources(postgres_db):
+ db, _ = postgres_db
+ now = datetime.now(timezone.utc)
+ source = {"type": "chat", "role": "user", "content": "hello"}
+
+ node = db._parse_row(
+ (
+ "memory-1",
+ "memory",
+ {"sources": [json.dumps(source)]},
+ now,
+ now,
+ "tenant-a",
+ )
+ )
+
+ assert node["metadata"]["user_name"] == "tenant-a"
+ assert node["metadata"]["sources"] == [source]
+
+
+def test_new_polardb_business_methods_are_available():
+ required_methods = {
+ "delete_node_by_cube_memory_groups",
+ "get_user_names_by_cube_memory_groups",
+ "get_children_with_embeddings",
+ "get_neighbors_by_tag",
+ }
+
+ missing = sorted(name for name in required_methods if not hasattr(PostgresGraphDB, name))
+ assert missing == []
+
+
+def test_get_subgraph_matches_polardb_return_shape(postgres_db):
+ db, cursor = postgres_db
+ core = {
+ "id": "core",
+ "memory": "core memory",
+ "metadata": {"status": "activated"},
+ }
+ neighbor = {
+ "id": "neighbor",
+ "memory": "neighbor memory",
+ "metadata": {"status": "activated"},
+ }
+ db.get_node = MagicMock(return_value=core)
+ db.get_nodes = MagicMock(return_value=[neighbor])
+ cursor.fetchall.side_effect = [
+ [("core",), ("neighbor",)],
+ [("core", "neighbor", "RELATE_TO")],
+ ]
+
+ result = db.get_subgraph(
+ "core",
+ depth=2,
+ center_status="activated",
+ user_name="tenant-a",
+ )
+
+ assert result == {
+ "core_node": core,
+ "neighbors": [neighbor],
+ "edges": [{"source": "core", "target": "neighbor", "type": "RELATE_TO"}],
+ }
+ assert "user_name = %s" in _executed_sql(cursor)
+ db.get_node.assert_called_once_with("core", user_name="tenant-a")
+ db.get_nodes.assert_called_once_with(["neighbor"], user_name="tenant-a")
+
+
+def test_cube_group_delete_pairs_ids_with_tenants(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchone.return_value = (2,)
+
+ deleted = db.delete_node_by_cube_memory_groups(
+ [
+ {"cube_id": "tenant-a", "memory_ids": ["m1"]},
+ {"cube_id": "tenant-b", "memory_ids": ["m2"]},
+ ]
+ )
+
+ query, params = cursor.execute.call_args.args
+ assert deleted == 2
+ assert "unnest" in query.lower()
+ assert "requested.user_name" in query
+ assert params == (["tenant-a", "tenant-b"], ["m1", "m2"])
+
+
+def test_memory_owner_lookup_preserves_missing_ids(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchall.return_value = [("m1", "tenant-a")]
+
+ owners = db.get_user_names_by_memory_ids(["m1", "missing"])
+
+ assert owners == {"m1": "tenant-a", "missing": None}
+
+
+def test_export_graph_supports_latest_filters_and_pagination(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchone.return_value = (3,)
+ cursor.fetchall.side_effect = [
+ [
+ (
+ "m1",
+ "memory",
+ {"memory_type": "LongTermMemory", "status": "activated"},
+ now,
+ now,
+ )
+ ],
+ [("m1", "m2", "RELATE_TO")],
+ ]
+
+ result = db.export_graph(
+ user_name="tenant-a",
+ page=2,
+ page_size=1,
+ filter={"agent_id": "agent-1"},
+ memory_type=["LongTermMemory"],
+ status=["activated"],
+ )
+
+ sql = _executed_sql(cursor)
+ assert result["total_nodes"] == 3
+ assert result["nodes"][0]["id"] == "m1"
+ assert "properties->'info'->>'agent_id'" in sql
+ assert "LIMIT %s OFFSET %s" in sql
+ assert "properties->>'memory_type' = ANY(%s)" in sql
+ assert "JOIN selected_nodes AS source_node" in sql
+ assert "source_node.user_name = e.user_name" in sql
+
+
+def test_keyword_like_matches_polardb_filters_and_parameter_name(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchall.return_value = [
+ (
+ "memory-1",
+ "needle memory",
+ {"source": "conversation"},
+ now,
+ now,
+ "tenant-a",
+ )
+ ]
+
+ result = db.search_by_keywords_like(
+ query_word="needle",
+ scope="LongTermMemory",
+ status="activated",
+ search_filter={"source": "conversation"},
+ user_name="tenant-a",
+ knowledgebase_ids=["kb-1"],
+ return_fields=["memory", "source"],
+ )
+
+ query, params = cursor.execute.call_args.args
+ assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}]
+ assert "user_name = ANY(%s)" in query
+ assert ["tenant-a", "kb-1"] in params
+ assert "properties->>'source'" in query
+ assert "properties->>'memory', '') LIKE %s" in query
+
+
+def test_keyword_tfidf_matches_polardb_result_shape(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchall.return_value = [
+ (
+ "memory-1",
+ "needle memory",
+ {"source": "conversation"},
+ now,
+ now,
+ "tenant-a",
+ )
+ ]
+
+ result = db.search_by_keywords_tfidf(
+ ["needle", "memory"],
+ scope="LongTermMemory",
+ user_name="tenant-a",
+ return_fields=["memory", "source"],
+ )
+
+ assert result == [{"id": "memory-1", "memory": "needle memory", "source": "conversation"}]
+ query, params = cursor.execute.call_args.args
+ assert params[0] == "'needle' | 'memory'"
+ assert "to_tsquery('jiebaqry'" in query
+ assert "ILIKE" not in query
+
+
+def test_keyword_searches_require_explicit_user_name(postgres_db):
+ db, cursor = postgres_db
+
+ assert db.search_by_keywords_like("needle") == []
+ assert db.search_by_keywords_tfidf(["needle"]) == []
+ cursor.execute.assert_not_called()
+
+
+def test_get_all_memory_items_matches_polardb_defaults(postgres_db):
+ db, cursor = postgres_db
+
+ assert db.get_all_memory_items("LongTermMemory") == []
+ cursor.execute.assert_not_called()
+
+ with pytest.raises(ValueError, match="Unsupported memory type scope"):
+ db.get_all_memory_items("Context", user_name="tenant-a")
+
+ cursor.fetchall.return_value = []
+ db.get_all_memory_items("LongTermMemory", user_name="tenant-a")
+ query, _ = cursor.execute.call_args.args
+ assert "properties->>'status'" not in query
+
+
+def test_metadata_contains_list_uses_json_array_overlap(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchall.return_value = [("m1",)]
+
+ result = db.get_by_metadata(
+ [{"field": "tags", "op": "contains", "value": ["blue", "green"]}],
+ user_name="tenant-a",
+ )
+
+ query, params = cursor.execute.call_args.args
+ assert result == ["m1"]
+ assert "properties->'tags' @> %s::jsonb" in query
+ assert " OR " in query
+ assert json.dumps(["blue"], ensure_ascii=False) in params
+ assert json.dumps(["green"], ensure_ascii=False) in params
diff --git a/tests/graph_dbs/test_postgres_pgvector.py b/tests/graph_dbs/test_postgres_pgvector.py
new file mode 100644
index 000000000..d404cf230
--- /dev/null
+++ b/tests/graph_dbs/test_postgres_pgvector.py
@@ -0,0 +1,461 @@
+import hashlib
+import json
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from memos.configs.graph_db import PostgresGraphDBConfig
+from memos.graph_dbs.postgres import PostgresGraphDB
+
+
+@pytest.fixture
+def postgres_db():
+ db = PostgresGraphDB.__new__(PostgresGraphDB)
+ db.schema = "memos"
+ db.db_name = "memos"
+ db.user_name = "tenant-a"
+ db._pool_closed = False
+ db.config = SimpleNamespace(
+ embedding_dimension=3,
+ fulltext_config="simple",
+ vector_index_type="hnsw",
+ hnsw_m=16,
+ hnsw_ef_construction=64,
+ ivfflat_lists=100,
+ shard_count=400,
+ )
+ db._partition_remainders = {
+ "tenant-a": 17,
+ "tenant-b": 17,
+ "kb-1": 17,
+ }
+
+ cursor = MagicMock()
+ connection = MagicMock()
+ connection.cursor.return_value.__enter__.return_value = cursor
+ db._get_conn = MagicMock(return_value=connection)
+ db._put_conn = MagicMock()
+ return db, cursor
+
+
+def _executed_sql(cursor: MagicMock) -> str:
+ return "\n".join(str(call.args[0]) for call in cursor.execute.call_args_list)
+
+
+def test_postgres_pool_keeps_eight_warm_connections():
+ config = PostgresGraphDBConfig(
+ host="postgres",
+ user="postgres",
+ password="secret",
+ db_name="memos",
+ user_name="tenant-a",
+ )
+ assert config.minconn == 8
+ assert config.maxconn == 20
+
+
+def test_get_conn_does_not_ping_before_the_query():
+ db = PostgresGraphDB.__new__(PostgresGraphDB)
+ db._pool_closed = False
+ cursor = MagicMock()
+ connection = MagicMock()
+ connection.closed = 0
+ connection.cursor.return_value.__enter__.return_value = cursor
+ db.pool = MagicMock()
+ db.pool.getconn.return_value = connection
+
+ assert db._get_conn() is connection
+ cursor.execute.assert_not_called()
+
+
+def test_postgres_config_defaults_to_current_embedding_dimension():
+ config = PostgresGraphDBConfig(
+ host="postgres",
+ user="memos",
+ password="secret",
+ db_name="memos",
+ user_name="tenant-a",
+ )
+
+ assert config.embedding_dimension == 1024
+ assert config.shard_count == 400
+
+
+def test_init_schema_creates_tenant_edges_fulltext_and_hnsw(postgres_db):
+ db, cursor = postgres_db
+
+ db._init_schema()
+
+ sql = _executed_sql(cursor)
+ assert "user_name TEXT NOT NULL" in sql
+ assert "UNIQUE(user_name, source_id, target_id, edge_type)" in sql
+ assert "search_vector tsvector" in sql
+ assert "properties::text" in sql
+ assert "USING GIN(search_vector)" in sql
+ assert "halfvec_cosine_ops" in sql
+ assert "idx_memory_embedding_halfvec" in sql
+ assert "embedding halfvec(" in sql
+ assert '"Memory"' in sql
+ assert "USING ivfflat" not in sql
+
+
+def test_edge_operations_are_tenant_scoped_and_support_any_direction(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchone.return_value = (1,)
+ cursor.fetchall.return_value = [("source", "target", "RELATE_TO")]
+
+ db.add_edge("source", "target", "RELATE_TO", user_name="tenant-a")
+ assert db.edge_exists(
+ "source",
+ "target",
+ "RELATE_TO",
+ direction="ANY",
+ user_name="tenant-a",
+ )
+ edges = db.get_edges("source", type="ANY", direction="ANY", user_name="tenant-a")
+
+ sql = _executed_sql(cursor)
+ assert "user_name" in sql
+ assert edges == [{"from": "source", "to": "target", "type": "RELATE_TO"}]
+ assert any(
+ "tenant-a" in call.args[1] for call in cursor.execute.call_args_list if len(call.args) > 1
+ )
+
+ db.edge_exists("source", "target", type="ANY", user_name="tenant-a")
+ query, params = cursor.execute.call_args.args
+ assert "edge_type = %s" not in query
+ assert params == ["tenant-a", "source", "target"]
+
+
+def test_vector_search_applies_filters_kb_ids_and_return_fields(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchall.return_value = [
+ (
+ "node-1",
+ 0.95,
+ "remember this",
+ {"status": "activated", "source": "conversation"},
+ now,
+ now,
+ "tenant-a",
+ )
+ ]
+
+ results = db.search_by_embedding(
+ vector=[0.1, 0.2, 0.3],
+ top_k=5,
+ scope="LongTermMemory",
+ status="activated",
+ user_name="tenant-a",
+ knowledgebase_ids=["kb-1"],
+ filter={"and": [{"info.agent_id": "agent-1"}, {"tags": {"contains": "blue"}}]},
+ return_fields=["memory", "source", "created_at"],
+ )
+
+ query, params = cursor.execute.call_args.args
+ assert "user_name = ANY(%s)" in query
+ assert "COALESCE(properties->>'id', id)" in query
+ assert "properties->'info'->>'agent_id'" in query
+ assert "properties->'tags' @> %s::jsonb" in query
+ assert ["tenant-a", "kb-1"] in params
+ assert results[0]["memory"] == "remember this"
+ assert results[0]["source"] == "conversation"
+ assert results[0]["created_at"] == now.isoformat()
+
+
+def test_lightweight_vector_search_returns_memory_without_extra_lookup(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchall.return_value = [
+ (
+ "node-1",
+ 0.95,
+ "remember this",
+ now,
+ now,
+ "tenant-a",
+ '"LongTermMemory"',
+ '"activated"',
+ None,
+ None,
+ None,
+ None,
+ )
+ ]
+
+ results = db.search_by_embedding(
+ vector=[0.1, 0.2, 0.3],
+ top_k=5,
+ scope="LongTermMemory",
+ user_name="tenant-a",
+ light_weight_mode=True,
+ )
+
+ assert results[0]["memory"] == "remember this"
+ assert results[0]["memory_type"] == "LongTermMemory"
+ assert PostgresGraphDB.supports_lightweight_vector_search is True
+ assert PostgresGraphDB.prefer_lightweight_vector_search is True
+
+
+def test_search_result_does_not_add_postgres_only_property_aliases(postgres_db):
+ db, _ = postgres_db
+ now = datetime.now(timezone.utc)
+ row = (
+ "node-1",
+ 0.95,
+ "remember this",
+ {"status": "activated"},
+ now,
+ now,
+ "tenant-a",
+ )
+
+ result = db._build_search_result(row, ["metadata", "properties"])
+
+ assert result == {"id": "node-1", "score": 0.95}
+
+
+def test_search_result_keeps_polardb_serialized_source_shape(postgres_db):
+ db, _ = postgres_db
+ now = datetime.now(timezone.utc)
+ source = json.dumps({"type": "chat", "role": "user", "content": "hello"})
+ row = (
+ "node-1",
+ 0.95,
+ "remember this",
+ {"sources": [source]},
+ now,
+ now,
+ "tenant-a",
+ )
+
+ result = db._build_search_result(row, ["sources"])
+
+ assert result["sources"] == [source]
+
+
+def test_fulltext_search_uses_polardb_and_tsquery(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchall.return_value = [
+ (
+ "node-1",
+ 0.8,
+ "中文记忆内容",
+ {"status": "activated"},
+ now,
+ now,
+ "tenant-a",
+ )
+ ]
+
+ results = db.search_by_fulltext(
+ query_words=["中文", "记忆"],
+ top_k=10,
+ scope="LongTermMemory",
+ user_name="tenant-a",
+ filter={"info.agent_id": "agent-1"},
+ return_fields=["memory"],
+ )
+
+ query, params = cursor.execute.call_args.args
+ assert "to_tsquery('jiebacfg'" in query
+ assert "search_vector @@" in query
+ assert "ILIKE" not in query
+ assert "properties->'info'->>'agent_id'" in query
+ assert params[0] == "'中文' & '记忆'"
+ assert results == [{"id": "node-1", "score": 0.8, "memory": "中文记忆内容"}]
+
+
+def test_get_node_without_tenant_searches_all_partitions(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchone.return_value = (
+ "node-1",
+ "remember this",
+ {"status": "activated", "user_name": "tenant-b"},
+ now,
+ now,
+ )
+
+ result = db.get_node("node-1")
+
+ query, params = cursor.execute.call_args.args
+ assert "WHERE id = %s" in query
+ assert "user_name = %s" not in query
+ assert '"memos_graph_0"."Memory"' in query
+ assert '"memos_graph_399"."Memory"' in query
+ assert "FROM memos.memories" not in query
+ assert params == ("node-1",)
+ assert result["id"] == "node-1"
+
+
+def test_get_node_with_tenant_remains_tenant_scoped(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchone.return_value = None
+
+ assert db.get_node("node-1", user_name="tenant-b") is None
+
+ query, params = cursor.execute.call_args.args
+ assert "WHERE id = %s AND user_name = %s" in query
+ assert params == ("node-1", "tenant-b")
+
+
+def test_get_nodes_without_tenant_searches_all_partitions(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchall.return_value = []
+
+ assert db.get_nodes(["node-1", "node-2"]) == []
+
+ query, params = cursor.execute.call_args.args
+ assert "WHERE id = ANY(%s)" in query
+ assert "user_name = %s" not in query
+ assert '"memos_graph_0"."Memory"' in query
+ assert '"memos_graph_399"."Memory"' in query
+ assert "FROM memos.memories" not in query
+ assert params == (["node-1", "node-2"],)
+
+
+def test_cube_soft_delete_recover_and_counts_are_available(postgres_db):
+ db, cursor = postgres_db
+ cursor.rowcount = 3
+ cursor.fetchone.side_effect = [(3,), (0,)]
+
+ assert db.delete_node_by_mem_cube_id("tenant-a", "delete-1") == 3
+ assert db.recover_memory_by_mem_cube_id("tenant-a", "delete-1") == 3
+ assert db.get_memory_count("LongTermMemory", user_name="tenant-a") == 3
+ assert db.node_not_exist("UserMemory", user_name="tenant-a") == 1
+
+ sql = _executed_sql(cursor)
+ assert "delete_record_id" in sql
+ assert "status" in sql
+ assert "user_name = %s" in sql
+
+
+def test_drop_database_is_scoped_to_current_tenant(postgres_db):
+ db, _cursor = postgres_db
+ db.clear = MagicMock()
+
+ db.drop_database()
+
+ db.clear.assert_called_once_with(user_name="tenant-a")
+
+
+def test_structure_candidates_accept_tenant_and_optional_embedding(postgres_db):
+ db, cursor = postgres_db
+ now = datetime.now(timezone.utc)
+ cursor.fetchall.return_value = [
+ (
+ "node-1",
+ "remember this",
+ {"status": "activated", "memory_type": "LongTermMemory"},
+ now,
+ now,
+ [0.1, 0.2, 0.3],
+ )
+ ]
+
+ results = db.get_structure_optimization_candidates(
+ "LongTermMemory",
+ include_embedding=True,
+ user_name="tenant-b",
+ )
+
+ query, params = cursor.execute.call_args.args
+ assert "m.embedding" in query
+ assert params == ("LongTermMemory", "tenant-b")
+ assert results[0]["metadata"]["embedding"] == [0.1, 0.2, 0.3]
+
+
+def test_structure_candidates_only_exclude_parent_edges(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchall.return_value = []
+
+ db.get_structure_optimization_candidates(
+ "LongTermMemory",
+ user_name="tenant-a",
+ )
+
+ query, _ = cursor.execute.call_args.args
+ assert "e1.edge_type = 'PARENT'" in query
+ assert "e2.edge_type = 'PARENT'" in query
+
+
+def test_time_filter_uses_timestamp_comparison(postgres_db):
+ db, cursor = postgres_db
+ cursor.fetchall.return_value = []
+
+ db.search_by_embedding(
+ [0.1, 0.2, 0.3],
+ user_name="tenant-a",
+ top_k=5,
+ filter={"created_at": {"gt": "2026-01-01T00:00:00"}},
+ )
+
+ query, _params = cursor.execute.call_args.args
+ assert "::timestamp >" in query
+ assert "timestamptz" not in query
+
+
+def test_single_tenant_vector_search_reads_one_hash_partition(postgres_db):
+ db, cursor = postgres_db
+ db._partition_remainders = {"tenant-a": 17}
+ cursor.fetchall.return_value = []
+
+ db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5)
+
+ query, params = cursor.execute.call_args.args
+ assert 'FROM "memos_graph_17"."Memory"' in query
+ assert "FROM memos.memories" not in query
+ assert query.count("%s::halfvec(3)") == 1
+ assert "AS MATERIALIZED" in query
+ assert "ORDER BY embedding <=> (SELECT query_vector FROM q)" in query
+ materialized_and_score, outer_filter = query.split("FROM t", 1)
+ materialized_body = materialized_and_score.split("AS MATERIALIZED", 1)[1]
+ assert "< 0.9" not in materialized_body
+ assert "scope_distance < 0.9" in outer_filter
+ assert "((1 - scope_distance) + 1) / 2.0" in materialized_and_score
+ assert "embedding::halfvec" not in query
+ assert "user_name = %s" in query
+ assert "user_name = ANY(%s)" not in query
+ assert params[0] == [0.1, 0.2, 0.3]
+ assert params[1] == "tenant-a"
+ assert params[-1] == 5
+ assert any("hnsw.iterative_scan" in str(call.args[0]) for call in cursor.execute.call_args_list)
+
+
+def test_cross_shard_vector_search_reads_only_those_partitions(postgres_db):
+ db, cursor = postgres_db
+ db._partition_remainders = {"tenant-a": 17, "kb-1": 19}
+ cursor.fetchall.return_value = []
+
+ db.search_by_embedding(
+ [0.1, 0.2, 0.3],
+ user_name="tenant-a",
+ knowledgebase_ids=["kb-1"],
+ top_k=5,
+ )
+
+ query, _params = cursor.execute.call_args.args
+ assert '"memos_graph_17"."Memory"' in query
+ assert '"memos_graph_19"."Memory"' in query
+ assert "UNION ALL" in query
+ assert "FROM memos.memories" not in query
+
+
+def test_partition_remainder_uses_polardb_md5(postgres_db):
+ db, cursor = postgres_db
+ db._partition_remainders = {}
+ cursor.fetchall.return_value = []
+ shard = int(hashlib.md5(b"tenant-a").hexdigest(), 16) % 400
+
+ db.search_by_embedding([0.1, 0.2, 0.3], user_name="tenant-a", top_k=5)
+
+ query = cursor.execute.call_args.args[0]
+ assert f'"memos_graph_{shard}"."Memory"' in query
+ assert "satisfies_hash_partition" not in _executed_sql(cursor)
+ assert db._partition_remainder("tenant-a") == shard
diff --git a/tests/graph_dbs/test_search_return_fields.py b/tests/graph_dbs/test_search_return_fields.py
index 1ed0f6173..a23374e5a 100644
--- a/tests/graph_dbs/test_search_return_fields.py
+++ b/tests/graph_dbs/test_search_return_fields.py
@@ -264,6 +264,73 @@ def fake_connection():
def test_polardb_declares_lightweight_vector_search_support(self, polardb_instance):
assert polardb_instance.supports_lightweight_vector_search is True
+ @pytest.mark.parametrize(
+ "stored_user_id,expected_user_id",
+ [
+ ("1234567890123456789", "1234567890123456789"),
+ (1234567890123456789, "1234567890123456789"),
+ ('"1234567890123456789"', "1234567890123456789"),
+ ("00123", "00123"),
+ (None, None),
+ ("null", None),
+ ],
+ )
+ def test_lightweight_numeric_user_id_can_be_recalled(
+ self, polardb_instance, stored_user_id, expected_user_id
+ ):
+ from memos.memories.textual.tree_text_memory.retrieve.recall import GraphMemoryRetriever
+
+ polardb_instance.db_name = "test_db"
+ polardb_instance.config = {"user_name": "default_user"}
+ polardb_instance._build_user_name_and_kb_ids_conditions_sql = MagicMock(return_value=[])
+ polardb_instance._build_filter_conditions_sql = MagicMock(return_value=[])
+ cursor = MagicMock()
+ cursor.fetchall.return_value = [
+ (
+ 1,
+ None,
+ None,
+ '"00000000-0000-4000-8000-000000000001"',
+ 0.2,
+ '"synthetic knowledge"',
+ '"LongTermMemory"',
+ stored_user_id,
+ '["synthetic"]',
+ '{"test":true}',
+ "0.9",
+ "true",
+ 0.8,
+ )
+ ]
+ connection = MagicMock()
+ connection.cursor.return_value.__enter__.return_value = cursor
+
+ @contextmanager
+ def fake_connection():
+ yield connection
+
+ polardb_instance._get_connection = fake_connection
+ hits = polardb_instance.search_by_embedding(
+ vector=[0.1] * 1024,
+ user_name="knowledge-cube",
+ light_weight_mode=True,
+ return_fields=[
+ "memory",
+ "memory_type",
+ "user_id",
+ "tags",
+ "info",
+ "confidence",
+ "is_fast",
+ ],
+ )
+ memory = GraphMemoryRetriever._memory_item_from_vector_hit(hits[0])
+ assert memory.metadata.user_id == expected_user_id
+ assert memory.metadata.tags == ["synthetic"]
+ assert memory.metadata.info == {"test": True}
+ assert memory.metadata.confidence == 0.9
+ assert memory.metadata.is_fast is True
+
class TestFieldNameValidation:
"""Tests for _validate_return_fields injection prevention."""
diff --git a/tests/llms/test_openai_responses_llm.py b/tests/llms/test_openai_responses_llm.py
new file mode 100644
index 000000000..d6915b122
--- /dev/null
+++ b/tests/llms/test_openai_responses_llm.py
@@ -0,0 +1,104 @@
+"""Tests for OpenAIResponsesLLM and AzureResponsesLLM generate reasoning handling."""
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary
+
+from memos.configs.llm import AzureLLMConfig, OpenAIResponsesLLMConfig
+from memos.llms.openai_new import AzureResponsesLLM, OpenAIResponsesLLM
+
+
+MESSAGES = [{"role": "user", "content": "Hello"}]
+
+
+def _make_openai_llm(
+ *, enable_thinking: bool = False, remove_think_prefix: bool = False
+) -> OpenAIResponsesLLM:
+ llm = OpenAIResponsesLLM.__new__(OpenAIResponsesLLM)
+ llm.config = OpenAIResponsesLLMConfig(
+ api_key="test-key",
+ model_name_or_path="gpt-4o",
+ enable_thinking=enable_thinking,
+ remove_think_prefix=remove_think_prefix,
+ )
+ llm.client = MagicMock()
+ return llm
+
+
+def _make_azure_llm(
+ *, enable_thinking: bool = False, remove_think_prefix: bool = False
+) -> AzureResponsesLLM:
+ llm = AzureResponsesLLM.__new__(AzureResponsesLLM)
+ llm.config = AzureLLMConfig(
+ api_key="test-key",
+ model_name_or_path="gpt-4o",
+ enable_thinking=enable_thinking,
+ remove_think_prefix=remove_think_prefix,
+ )
+ llm.client = MagicMock()
+ return llm
+
+
+@pytest.mark.parametrize(
+ "make_llm",
+ [_make_openai_llm, _make_azure_llm],
+ ids=["openai", "azure"],
+)
+class TestResponsesLLMGenerate:
+ def test_generate_without_reasoning_item_returns_output_text(self, make_llm):
+ llm = make_llm(enable_thinking=False)
+ llm.client.responses.create.return_value = SimpleNamespace(
+ output=[SimpleNamespace(type="message")],
+ output_text="plain answer",
+ )
+
+ result = llm.generate(MESSAGES)
+
+ assert result == "plain answer"
+
+ def test_generate_with_empty_reasoning_summary_returns_output_text(self, make_llm):
+ llm = make_llm(enable_thinking=True)
+ reasoning_item = ResponseReasoningItem(id="rs_1", summary=[], type="reasoning")
+ llm.client.responses.create.return_value = SimpleNamespace(
+ output=[reasoning_item],
+ output_text="plain answer",
+ )
+
+ result = llm.generate(MESSAGES)
+
+ assert result == "plain answer"
+
+ def test_generate_with_reasoning_summary_prepends_think_tag(self, make_llm):
+ llm = make_llm(enable_thinking=True, remove_think_prefix=False)
+ reasoning_item = ResponseReasoningItem(
+ id="rs_1",
+ summary=[Summary(text="step by step", type="summary_text")],
+ type="reasoning",
+ )
+ llm.client.responses.create.return_value = SimpleNamespace(
+ output=[reasoning_item],
+ output_text="final answer",
+ )
+
+ result = llm.generate(MESSAGES)
+
+ assert result == "step by stepfinal answer"
+
+ def test_generate_with_remove_think_prefix_strips_thinking(self, make_llm):
+ llm = make_llm(enable_thinking=True, remove_think_prefix=True)
+ reasoning_item = ResponseReasoningItem(
+ id="rs_1",
+ summary=[Summary(text="step by step", type="summary_text")],
+ type="reasoning",
+ )
+ llm.client.responses.create.return_value = SimpleNamespace(
+ output=[reasoning_item],
+ output_text="inline thoughtfinal answer",
+ )
+
+ result = llm.generate(MESSAGES)
+
+ assert result == "final answer"
diff --git a/tests/mem_cube/test_single_cube_knowledge_neighbors.py b/tests/mem_cube/test_single_cube_knowledge_neighbors.py
new file mode 100644
index 000000000..5cb433d77
--- /dev/null
+++ b/tests/mem_cube/test_single_cube_knowledge_neighbors.py
@@ -0,0 +1,50 @@
+import uuid
+
+from unittest.mock import Mock
+
+from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
+
+
+def make_raw(item_id: str, relativity: float, **extra) -> TextualMemoryItem:
+ return TextualMemoryItem(
+ id=item_id,
+ memory=item_id,
+ metadata=TreeNodeTextualMemoryMetadata(
+ memory_type="RawFileMemory",
+ relativity=relativity,
+ **extra,
+ ),
+ )
+
+
+def test_postformat_discovers_rawfile_neighbors_by_id_without_edges():
+ import memos.api.handlers # noqa: F401 - initialize package before known circular import
+
+ from memos.multi_mem_cube.single_cube import SingleCubeView
+
+ previous = make_raw(str(uuid.uuid4()), 0.1)
+ following = make_raw(str(uuid.uuid4()), 0.1)
+ current = make_raw(str(uuid.uuid4()), 1.0, preceding_id=previous.id, following_id=following.id)
+ searcher = Mock()
+ neighbors = {
+ previous.id: previous.model_dump(),
+ following.id: following.model_dump(),
+ }
+ searcher.graph_store.get_node.side_effect = lambda node_id, **_: neighbors.get(node_id)
+ view = SingleCubeView(
+ cube_id="cube-1",
+ naive_mem_cube=Mock(),
+ mem_reader=Mock(),
+ mem_scheduler=Mock(),
+ logger=Mock(),
+ searcher=searcher,
+ )
+
+ result = view._postformat_memories(
+ [current], "cube-1", include_embedding=False, neighbor_discovery=True
+ )
+
+ assert [item["id"] for item in result] == [previous.id, current.id, following.id]
+ assert result[0]["metadata"]["relativity"] == 0.8
+ assert result[2]["metadata"]["relativity"] == 0.8
+ searcher.graph_store.get_edges.assert_not_called()
diff --git a/tests/mem_feedback/test_feedback.py b/tests/mem_feedback/test_feedback.py
index bbf30ae94..3f86ac482 100644
--- a/tests/mem_feedback/test_feedback.py
+++ b/tests/mem_feedback/test_feedback.py
@@ -1,3 +1,5 @@
+import uuid
+
from unittest.mock import Mock
from memos.mem_feedback.feedback import MemFeedback
@@ -139,3 +141,21 @@ def test_update_judgement_prompt_omits_memory_text_from_response_schema():
assert '"reason"' in output_section
assert '"text"' not in output_section
assert '"old_memory"' not in output_section
+
+
+def test_retrieve_locks_only_knowledge_summaries_without_edge_queries():
+ feedback = MemFeedback.__new__(MemFeedback)
+ feedback.pref_feedback = False
+ feedback.searcher = Mock()
+ normal = TextualMemoryItem(id=str(uuid.uuid4()), memory="normal")
+ knowledge = TextualMemoryItem(
+ id=str(uuid.uuid4()),
+ memory="knowledge",
+ metadata={"knowledge_material_ids": [str(uuid.uuid4())]},
+ )
+ feedback.searcher.search.return_value = [(normal, 0.9), (knowledge, 0.8)]
+
+ result = feedback._retrieve("query", user_name="cube-1")
+
+ assert result == [normal]
+ feedback.searcher.graph_store.get_edges.assert_not_called()
diff --git a/tests/mem_reader/test_image_parser.py b/tests/mem_reader/test_image_parser.py
new file mode 100644
index 000000000..374ac17e9
--- /dev/null
+++ b/tests/mem_reader/test_image_parser.py
@@ -0,0 +1,76 @@
+"""Tests for vision image URL validation in ImageParser.parse_fine."""
+
+import json
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from memos.mem_reader.read_multi_modal.image_parser import ImageParser
+
+
+def _parser() -> tuple[ImageParser, MagicMock]:
+ llm = MagicMock()
+ llm.generate.return_value = json.dumps(
+ {
+ "memory list": [
+ {
+ "key": "visible label",
+ "memory_type": "LongTermMemory",
+ "value": "A product label is visible.",
+ "tags": ["image"],
+ }
+ ],
+ "summary": "A labeled product.",
+ }
+ )
+ embedder = MagicMock()
+ embedder.embed.return_value = [[0.1, 0.2]]
+ return ImageParser(embedder, llm), llm
+
+
+def _image_message(url: str) -> dict:
+ return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}}
+
+
+INFO = {"user_id": "user-1", "session_id": "session-1"}
+
+
+@pytest.mark.parametrize(
+ "url",
+ [
+ "images/c502e59b213998cd695d7f57a6e33de73f5493f2d3fbab6c4e9635341aa7e7b7.jpg",
+ "image/photo.png",
+ "/tmp/photo.jpg",
+ "file:///tmp/photo.jpg",
+ "data:images/jpeg;base64,aaaa",
+ "data:image/jpeg,not-base64",
+ ],
+)
+def test_parse_fine_skips_non_sendable_image_url(url: str) -> None:
+ parser, llm = _parser()
+
+ result = parser.parse_fine(_image_message(url), INFO)
+
+ assert result == []
+ llm.generate.assert_not_called()
+
+
+@pytest.mark.parametrize(
+ "url",
+ [
+ "https://example.com/device.jpg",
+ "http://example.com/images/device.jpg",
+ "data:image/jpeg;base64,aaaa",
+ "DATA:IMAGE/PNG;base64,bbbb",
+ ],
+)
+def test_parse_fine_sends_http_and_base64_image_urls(url: str) -> None:
+ parser, llm = _parser()
+
+ result = parser.parse_fine(_image_message(url), INFO)
+
+ assert len(result) == 1
+ llm.generate.assert_called_once()
+ sent_url = llm.generate.call_args.args[0][0]["content"][1]["image_url"]["url"]
+ assert sent_url == url
diff --git a/tests/mem_reader/test_multi_modal_rawfile.py b/tests/mem_reader/test_multi_modal_rawfile.py
new file mode 100644
index 000000000..f583b7173
--- /dev/null
+++ b/tests/mem_reader/test_multi_modal_rawfile.py
@@ -0,0 +1,89 @@
+from unittest.mock import MagicMock, patch
+
+from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader
+from memos.memories.textual.item import (
+ SourceMessage,
+ TextualMemoryItem,
+ TreeNodeTextualMemoryMetadata,
+)
+
+
+def test_mixed_file_source_creates_rawfile_without_changing_prompt_type():
+ reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader)
+ reader.save_rawfile = True
+ reader.memory_version_switch = "off"
+ reader.embedder = MagicMock()
+ reader.embedder.embed.return_value = [[0.1]]
+ reader.multi_modal_parser = MagicMock()
+
+ file_info = {
+ "file_id": "file-1",
+ "filename": "knowledge.txt",
+ "file_data": "Complete file content",
+ }
+ file_source = SourceMessage(
+ type="file",
+ role=None,
+ content="Current file chunk",
+ file_info=file_info,
+ )
+ fast_item = TextualMemoryItem(
+ memory="Please parse this file\nCurrent file chunk\nRecord it completely",
+ metadata=TreeNodeTextualMemoryMetadata(
+ user_id="user-1",
+ session_id="session-1",
+ memory_type="LongTermMemory",
+ sources=[
+ SourceMessage(type="chat", role="user", content="Please parse this file"),
+ file_source,
+ SourceMessage(type="chat", role="user", content="Record it completely"),
+ ],
+ file_ids=[file_info["file_id"]],
+ ),
+ )
+ reader.multi_modal_parser.file_content_parser.create_source.return_value = SourceMessage(
+ type="file",
+ content=file_source.content,
+ doc_path=file_info["filename"],
+ )
+
+ llm_response = {
+ "memory list": [
+ {
+ "key": "Knowledge summary",
+ "memory_type": "LongTermMemory",
+ "value": "Summarized knowledge",
+ "tags": ["knowledge"],
+ }
+ ],
+ "summary": "Summary background",
+ }
+
+ with (
+ patch.object(reader, "_get_llm_response", return_value=llm_response) as get_llm_response,
+ patch.object(
+ reader,
+ "_get_maybe_merged_memory",
+ side_effect=lambda extracted_memory_dict, **kwargs: extracted_memory_dict,
+ ),
+ patch(
+ "memos.mem_reader.multi_modal_struct.trigger_hook",
+ side_effect=lambda _hook, **kwargs: kwargs["items"],
+ ),
+ ):
+ result = reader._process_string_fine(
+ [fast_item],
+ {"user_id": "user-1", "session_id": "session-1"},
+ )
+
+ summaries = [item for item in result if item.metadata.memory_type != "RawFileMemory"]
+ rawfiles = [item for item in result if item.metadata.memory_type == "RawFileMemory"]
+
+ assert get_llm_response.call_args.args[3] == "chat"
+ assert len(summaries) == 1
+ assert len(rawfiles) == 1
+ assert rawfiles[0].memory == file_source.content
+ assert rawfiles[0].metadata.summary_ids == [summaries[0].id]
+ assert reader.multi_modal_parser.file_content_parser.create_source.call_args.kwargs[
+ "message"
+ ] == {"file": file_info}
diff --git a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py
index c54b49de6..8cdb873ce 100644
--- a/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py
+++ b/tests/mem_scheduler/test_mem_read_handler_fast_cleanup.py
@@ -4,11 +4,18 @@
from unittest.mock import MagicMock
-def _make_text_item(item_id: str, memory: str):
+def _make_text_item(
+ item_id: str,
+ memory: str,
+ *,
+ memory_type: str = "LongTermMemory",
+ key: str | None = None,
+ info: dict | None = None,
+):
metadata = SimpleNamespace(
- memory_type="LongTermMemory",
- info={},
- key=None,
+ memory_type=memory_type,
+ info=info if info is not None else {},
+ key=key,
status="activated",
confidence=0.9,
tags=[],
@@ -17,8 +24,10 @@ def _make_text_item(item_id: str, memory: str):
class _FakeTextMem:
- def __init__(self, items):
+ def __init__(self, items, added_ids: list[str] | None = None):
self._items = {item.id: item for item in items}
+ self._added_ids = added_ids
+ self.add_calls: list = []
self.delete_calls: list = []
self.soft_delete_calls: list = []
self.memory_manager = SimpleNamespace(
@@ -29,6 +38,13 @@ def __init__(self, items):
def get(self, mem_id, user_name=None):
return self._items[mem_id]
+ def add(self, memories, user_name=None):
+ mem_list = list(memories)
+ self.add_calls.append((mem_list, user_name))
+ if self._added_ids is not None:
+ return list(self._added_ids)
+ return [item.id for item in mem_list]
+
def delete(self, ids, user_name=None):
self.delete_calls.append((list(ids), user_name))
@@ -98,3 +114,124 @@ def test_fine_transfer_error_does_not_soft_delete_fast_memory_with_versions():
assert text_mem.delete_calls == []
assert text_mem.soft_delete_calls == []
+
+
+def test_playground_legacy_events_align_enhanced_ids_with_filtered_mem_group(monkeypatch):
+ import memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler as handler_module
+
+ from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import (
+ MemReadMessageHandler,
+ )
+
+ monkeypatch.setattr(handler_module, "is_playground_api", lambda: True)
+
+ raw_chunk = _make_text_item("raw_1", "raw chunk")
+ raw_file_mem = _make_text_item(
+ "raw_file_1",
+ "raw file content",
+ memory_type="RawFileMemory",
+ key="raw-file-key",
+ )
+ added_mem = _make_text_item(
+ "mem_add_1",
+ "user prefers dark mode",
+ memory_type="LongTermMemory",
+ key="theme-pref",
+ )
+ updated_mem = _make_text_item(
+ "mem_upd_1",
+ "user lives in Shanghai",
+ memory_type="UserMemory",
+ key="city-pref",
+ info={"merged_from": ["old_mem_1"]},
+ )
+
+ text_mem = _FakeTextMem(
+ [raw_chunk],
+ added_ids=["enhanced_id_add_1", "enhanced_id_upd_1"],
+ )
+ mem_cube = SimpleNamespace(text_mem=text_mem)
+
+ mem_reader = MagicMock()
+ mem_reader.fine_transfer_simple_mem.return_value = [[raw_file_mem, added_mem, updated_mem]]
+ mem_reader.memory_version_switch = "off"
+ mem_reader.graph_db = None
+ mem_reader.save_rawfile = False
+
+ submitted_events: list = []
+ services = SimpleNamespace(
+ create_event_log=lambda **kwargs: SimpleNamespace(**kwargs),
+ submit_web_logs=lambda events, **kwargs: submitted_events.extend(events),
+ map_memcube_name=lambda mem_cube_id: "UserMemCube",
+ submit_messages=lambda messages: None,
+ )
+ scheduler_context = SimpleNamespace(
+ get_mem_cube=lambda: mem_cube,
+ get_mem_reader=lambda: mem_reader,
+ services=services,
+ )
+
+ handler = MemReadMessageHandler.__new__(MemReadMessageHandler)
+ handler.scheduler_context = scheduler_context
+
+ handler._process_memories_with_reader(
+ mem_ids=["raw_1"],
+ user_id="user_1",
+ mem_cube_id="cube_1",
+ text_mem=text_mem,
+ user_name="cube_1",
+ task_id="task_1",
+ info={"trigger_source": "Messages"},
+ )
+
+ assert len(text_mem.add_calls) == 1
+ assert text_mem.add_calls[0] == ([added_mem, updated_mem], "cube_1")
+
+ assert len(submitted_events) == 2
+ add_event, update_event = submitted_events
+
+ assert add_event.label == "addMemory"
+ assert add_event.task_id == "task_1"
+ assert add_event.memory_len == 1
+ assert add_event.memcube_log_content == [
+ {
+ "content": "theme-pref: user prefers dark mode",
+ "ref_id": "enhanced_id_add_1",
+ }
+ ]
+ assert add_event.metadata == [
+ {
+ "ref_id": "enhanced_id_add_1",
+ "id": "enhanced_id_add_1",
+ "key": "theme-pref",
+ "memory": "user prefers dark mode",
+ "memory_type": "LongTermMemory",
+ "status": "activated",
+ "confidence": 0.9,
+ "tags": [],
+ "updated_at": None,
+ }
+ ]
+
+ assert update_event.label == "updateMemory"
+ assert update_event.task_id == "task_1"
+ assert update_event.memory_len == 1
+ assert update_event.memcube_log_content == [
+ {
+ "content": "city-pref: user lives in Shanghai",
+ "ref_id": "enhanced_id_upd_1",
+ }
+ ]
+ assert update_event.metadata == [
+ {
+ "ref_id": "enhanced_id_upd_1",
+ "id": "enhanced_id_upd_1",
+ "key": "city-pref",
+ "memory": "user lives in Shanghai",
+ "memory_type": "UserMemory",
+ "status": "activated",
+ "confidence": 0.9,
+ "tags": [],
+ "updated_at": None,
+ }
+ ]
diff --git a/tests/mem_scheduler/test_rabbitmq_shutdown.py b/tests/mem_scheduler/test_rabbitmq_shutdown.py
new file mode 100644
index 000000000..5cb594c43
--- /dev/null
+++ b/tests/mem_scheduler/test_rabbitmq_shutdown.py
@@ -0,0 +1,110 @@
+"""Native scheduler cleanup only; no models, broker or business data."""
+
+import threading
+import time
+
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from memos.api import lifecycle
+from memos.configs.mem_scheduler import GeneralSchedulerConfig
+from memos.mem_scheduler.optimized_scheduler import OptimizedScheduler
+from memos.mem_scheduler.webservice_modules.rabbitmq_service import RabbitMQSchedulerModule
+
+
+def test_close_without_rabbitmq_initialization_is_safe_and_repeatable():
+ scheduler = RabbitMQSchedulerModule()
+ scheduler.rabbitmq_close()
+ scheduler.rabbitmq_close()
+ assert scheduler._io_loop_thread is None
+ assert scheduler._rabbitmq_stop_flag is True
+
+
+def test_early_initialization_skip_still_closes(monkeypatch):
+ monkeypatch.setenv("CI", "true")
+ monkeypatch.delenv("MEMOS_ENABLE_RABBITMQ", raising=False)
+ scheduler = RabbitMQSchedulerModule()
+ scheduler.initialize_rabbitmq()
+ assert scheduler._rabbitmq_initializing is False
+ scheduler.rabbitmq_close()
+ assert scheduler._io_loop_thread is None
+
+
+def test_real_io_thread_and_connection_cleanup_still_execute():
+ scheduler = RabbitMQSchedulerModule()
+ stopped = threading.Event()
+ calls = []
+ scheduler.rabbitmq_channel = SimpleNamespace(
+ is_open=True, close=lambda: calls.append("channel")
+ )
+ scheduler.rabbitmq_connection = SimpleNamespace(
+ is_open=True,
+ close=lambda: calls.append("connection"),
+ ioloop=SimpleNamespace(stop=lambda: (calls.append("ioloop"), stopped.set())),
+ )
+ worker = threading.Thread(target=stopped.wait, daemon=True)
+ scheduler._io_loop_thread = worker
+ worker.start()
+ try:
+ scheduler.rabbitmq_close()
+ assert not worker.is_alive()
+ assert calls == ["channel", "connection", "ioloop"]
+ assert scheduler._rabbitmq_stop_flag is True
+ finally:
+ stopped.set()
+ worker.join(timeout=1)
+
+
+def test_actual_optimized_scheduler_lifecycle_joins_threads_and_shuts_executor(monkeypatch):
+ scheduler = OptimizedScheduler(GeneralSchedulerConfig(use_redis_queue=False))
+ scheduler._running = True
+
+ def native_running_loop():
+ while scheduler._running:
+ time.sleep(0.005)
+
+ consumer = threading.Thread(target=native_running_loop, daemon=True)
+ monitor = threading.Thread(target=native_running_loop, daemon=True)
+ scheduler._consumer_thread = consumer
+ scheduler._monitor_thread = monitor
+ consumer.start()
+ monitor.start()
+ task = scheduler.dispatcher.dispatcher_executor.submit(lambda: 42)
+ assert task.result(timeout=2) == 42
+ shutdown = Mock(wraps=scheduler.dispatcher.shutdown)
+ monkeypatch.setattr(scheduler.dispatcher, "shutdown", shutdown)
+ error = Mock()
+ monkeypatch.setattr(lifecycle.logger, "exception", error)
+ try:
+ lifecycle.shutdown_components({"mem_scheduler": scheduler})
+ assert not consumer.is_alive() and not monitor.is_alive()
+ assert scheduler._running is False
+ assert task.done() and scheduler.dispatcher.dispatcher_executor._shutdown is True
+ deadline = time.monotonic() + 2
+ while (
+ any(t.is_alive() for t in scheduler.dispatcher.dispatcher_executor._threads)
+ and time.monotonic() < deadline
+ ):
+ time.sleep(0.005)
+ assert not any(t.is_alive() for t in scheduler.dispatcher.dispatcher_executor._threads)
+ shutdown.assert_called_once()
+ assert scheduler._rabbitmq_stop_flag is True
+ error.assert_not_called()
+ finally:
+ scheduler._running = False
+ consumer.join(timeout=1)
+ monitor.join(timeout=1)
+ scheduler.dispatcher.shutdown()
+
+
+def test_unexpected_join_error_is_not_hidden():
+ scheduler = RabbitMQSchedulerModule()
+
+ def fail_join(**kwargs):
+ raise RuntimeError("fixture join failed")
+
+ scheduler._io_loop_thread = SimpleNamespace(is_alive=lambda: True, join=fail_join)
+ with pytest.raises(RuntimeError, match="fixture join failed"):
+ scheduler.rabbitmq_close()
diff --git a/tests/memories/textual/test_item.py b/tests/memories/textual/test_item.py
new file mode 100644
index 000000000..71c206742
--- /dev/null
+++ b/tests/memories/textual/test_item.py
@@ -0,0 +1,91 @@
+"""Regression tests for JSON-encoded internal metadata read from graph storage."""
+
+import copy
+import json
+import uuid
+
+import pytest
+
+from pydantic import ValidationError
+
+from memos.memories.textual.item import (
+ PreferenceTextualMemoryMetadata,
+ SearchedTreeNodeTextualMemoryMetadata,
+ TextualMemoryItem,
+ TextualMemoryMetadata,
+ TreeNodeTextualMemoryMetadata,
+)
+
+
+@pytest.mark.parametrize(
+ "metadata_cls",
+ [
+ TextualMemoryMetadata,
+ TreeNodeTextualMemoryMetadata,
+ SearchedTreeNodeTextualMemoryMetadata,
+ PreferenceTextualMemoryMetadata,
+ ],
+)
+def test_internal_info_decodes_json_object(metadata_cls):
+ value = {"ingest_batch_id": "test-batch", "nested": {"chunk_index": 0, "values": [True, None]}}
+
+ metadata = metadata_cls(internal_info=json.dumps(value))
+
+ assert metadata.internal_info == value
+ assert metadata.model_dump()["internal_info"] == value
+
+
+@pytest.mark.parametrize("value", [None, {}, {"ingest_batch_id": "test-batch", "nested": [1, 2]}])
+def test_internal_info_preserves_existing_values(value):
+ original = copy.deepcopy(value)
+
+ metadata = TextualMemoryMetadata(internal_info=value)
+
+ assert metadata.internal_info == original
+ assert value == original
+
+
+@pytest.mark.parametrize(
+ "value",
+ ["{", "not-json", "[]", "null", "1", "true", '"text"', json.dumps('{"batch": "test"}')],
+)
+def test_internal_info_rejects_malformed_json_and_encoded_non_objects(value):
+ with pytest.raises(ValidationError):
+ TextualMemoryMetadata(internal_info=value)
+
+
+@pytest.mark.parametrize("value", [[], [["batch", "test"]], 1, False])
+def test_internal_info_preserves_rejection_of_invalid_types(value):
+ with pytest.raises(ValidationError):
+ TextualMemoryMetadata(internal_info=value)
+
+
+def test_search_item_from_dict_decodes_internal_info_without_changing_other_fields():
+ value = {"ingest_batch_id": "test-batch", "chunk_index": 0, "chunk_total": 2}
+ record = {
+ "id": str(uuid.uuid4()),
+ "memory": "Synthetic memory for a graph storage round trip.",
+ "metadata": {
+ "memory_type": "LongTermMemory",
+ "internal_info": json.dumps(value),
+ "relativity": 0.75,
+ "sources": [{"type": "chat", "role": "user", "content": "Synthetic source."}],
+ "session_id": "test-session",
+ },
+ }
+ original = copy.deepcopy(record)
+
+ item = TextualMemoryItem.from_dict(record)
+
+ assert item.metadata.internal_info == value
+ assert item.id == record["id"]
+ assert item.memory == record["memory"]
+ assert item.metadata.sources[0].content == "Synthetic source."
+ assert item.metadata.session_id == "test-session"
+ assert item.metadata.relativity == 0.75
+ assert record == original
+
+ # Graph storage serializes dictionaries as JSON strings again on the next write.
+ stored = item.to_dict()
+ stored["metadata"]["internal_info"] = json.dumps(stored["metadata"]["internal_info"])
+ assert TextualMemoryItem.from_dict(stored).to_dict() == item.to_dict()
diff --git a/tests/memories/textual/test_tree_knowledge_relations.py b/tests/memories/textual/test_tree_knowledge_relations.py
new file mode 100644
index 000000000..79bd80743
--- /dev/null
+++ b/tests/memories/textual/test_tree_knowledge_relations.py
@@ -0,0 +1,182 @@
+import uuid
+
+from unittest.mock import Mock
+
+from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata
+from memos.memories.textual.tree import TreeTextMemory
+
+
+class FakeGraphStore:
+ def __init__(self, items: list[TextualMemoryItem]) -> None:
+ self.nodes = {item.id: item.model_dump() for item in items}
+ self.edge_calls: list[tuple] = []
+ self.deleted_by_params: list[dict] = []
+
+ def get_nodes(self, ids, user_name=None):
+ return [self.nodes[node_id] for node_id in ids if node_id in self.nodes]
+
+ def update_node(self, id, fields, user_name=None):
+ self.nodes[id]["metadata"].update(fields)
+
+ def delete_node(self, id, user_name=None):
+ self.nodes.pop(id, None)
+
+ def delete_node_by_prams(self, **kwargs):
+ self.deleted_by_params.append(kwargs)
+ for node_id in kwargs.get("memory_ids") or []:
+ self.nodes.pop(node_id, None)
+
+ def get_by_metadata(self, filters, user_name=None, status=None):
+ matched = []
+ for node_id, node in self.nodes.items():
+ metadata = node["metadata"]
+ if all(
+ (
+ item["value"] in (metadata.get(item["field"]) or [])
+ if item.get("op") == "contains"
+ else metadata.get(item["field"]) == item["value"]
+ )
+ for item in filters
+ ):
+ matched.append(node_id)
+ return matched
+
+ def add_edge(self, *args, **kwargs):
+ self.edge_calls.append((args, kwargs))
+
+ def get_edges(self, *args, **kwargs):
+ self.edge_calls.append((args, kwargs))
+ return []
+
+
+def _test_id(value: str) -> str:
+ try:
+ return str(uuid.UUID(value))
+ except ValueError:
+ return str(uuid.uuid5(uuid.NAMESPACE_DNS, value))
+
+
+def make_item(
+ item_id: str,
+ memory_type: str,
+ *,
+ knowledge_material_ids: list[str] | None = None,
+ knowledge_summary_ids: list[str] | None = None,
+ info: dict | None = None,
+ file_ids: list[str] | None = None,
+ summary_ids: list[str] | None = None,
+) -> TextualMemoryItem:
+ metadata = TreeNodeTextualMemoryMetadata(
+ memory_type=memory_type,
+ knowledge_material_ids=[_test_id(value) for value in knowledge_material_ids or []],
+ knowledge_summary_ids=[_test_id(value) for value in knowledge_summary_ids or []],
+ info=(
+ {
+ **info,
+ "merged_from": [_test_id(value) for value in info.get("merged_from", [])],
+ }
+ if info and "merged_from" in info
+ else info
+ ),
+ file_ids=file_ids or [],
+ )
+ if summary_ids is not None:
+ metadata.summary_ids = [_test_id(value) for value in summary_ids]
+ return TextualMemoryItem(id=_test_id(item_id), memory=item_id, metadata=metadata)
+
+
+def make_tree(items: list[TextualMemoryItem]) -> tuple[TreeTextMemory, FakeGraphStore]:
+ tree = TreeTextMemory.__new__(TreeTextMemory)
+ graph_store = FakeGraphStore(items)
+ tree.graph_store = graph_store
+
+ def add(memories, user_name=None, **kwargs):
+ for memory in memories:
+ graph_store.nodes[memory.id] = memory.model_dump()
+ return [memory.id for memory in memories]
+
+ tree.add = Mock(side_effect=add)
+ return tree, graph_store
+
+
+def test_add_rawfile_nodes_persists_bidirectional_ids_without_edges():
+ summary = make_item("summary-1", "LongTermMemory")
+ material = make_item("material-1", "RawFileMemory", summary_ids=[summary.id])
+ tree, graph_store = make_tree([summary])
+
+ tree.add_rawfile_nodes([material], [summary.id], user_name="cube-1")
+
+ assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == [summary.id]
+ assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == [material.id]
+ assert graph_store.edge_calls == []
+
+
+def test_add_rawfile_nodes_reconnects_materials_when_summary_is_merged():
+ old_material = make_item("material-old", "RawFileMemory", knowledge_summary_ids=["summary-old"])
+ old_summary = make_item(
+ "summary-old", "LongTermMemory", knowledge_material_ids=[old_material.id]
+ )
+ new_summary = make_item("summary-new", "LongTermMemory", info={"merged_from": [old_summary.id]})
+ new_material = make_item("material-new", "RawFileMemory", summary_ids=[new_summary.id])
+ tree, graph_store = make_tree([old_material, old_summary, new_summary])
+
+ tree.add_rawfile_nodes([new_material], [new_summary.id], user_name="cube-1")
+
+ assert graph_store.nodes[new_summary.id]["metadata"]["knowledge_material_ids"] == [
+ new_material.id,
+ old_material.id,
+ ]
+ assert graph_store.nodes[old_material.id]["metadata"]["knowledge_summary_ids"] == [
+ new_summary.id
+ ]
+
+
+def test_delete_summary_unlinks_material_without_deleting_it():
+ material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"])
+ summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id])
+ tree, graph_store = make_tree([material, summary])
+
+ tree.delete_by_memory_ids([summary.id])
+
+ assert summary.id not in graph_store.nodes
+ assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == []
+
+
+def test_delete_material_unlinks_summary_without_deleting_it():
+ material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"])
+ summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id])
+ tree, graph_store = make_tree([material, summary])
+
+ tree.delete_by_memory_ids([material.id])
+
+ assert material.id not in graph_store.nodes
+ assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == []
+
+
+def test_file_delete_unlinks_knowledge_relations_before_bulk_delete():
+ material = make_item(
+ "material-1",
+ "RawFileMemory",
+ knowledge_summary_ids=["summary-1"],
+ file_ids=["file-1"],
+ )
+ summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id])
+ tree, graph_store = make_tree([material, summary])
+
+ tree.delete_by_filter(writable_cube_ids=["cube-1"], file_ids=["file-1"])
+
+ assert graph_store.nodes[summary.id]["metadata"]["knowledge_material_ids"] == []
+ assert graph_store.deleted_by_params == [
+ {"writable_cube_ids": ["cube-1"], "file_ids": ["file-1"], "filter": None}
+ ]
+
+
+def test_soft_delete_unlinks_summary_before_marking_it_deleted():
+ material = make_item("material-1", "RawFileMemory", knowledge_summary_ids=["summary-1"])
+ summary = make_item("summary-1", "LongTermMemory", knowledge_material_ids=[material.id])
+ tree, graph_store = make_tree([material, summary])
+
+ tree.soft_delete([summary.id], user_name="cube-1")
+
+ assert graph_store.nodes[material.id]["metadata"]["knowledge_summary_ids"] == []
+ assert graph_store.nodes[summary.id]["metadata"]["status"] == "deleted"
diff --git a/tests/memories/textual/test_tree_searcher.py b/tests/memories/textual/test_tree_searcher.py
index b79958ca1..fe40cbb7d 100644
--- a/tests/memories/textual/test_tree_searcher.py
+++ b/tests/memories/textual/test_tree_searcher.py
@@ -1,3 +1,5 @@
+import uuid
+
from unittest.mock import MagicMock
import pytest
@@ -143,3 +145,26 @@ def test_searcher_respects_memory_type(mock_searcher):
)
# WorkingMemory triggers only once path A
assert mock_searcher.graph_retriever.retrieve.call_args[1]["memory_scope"] == "WorkingMemory"
+
+
+def test_deduplicate_rawfile_results_uses_knowledge_summary_ids(mock_searcher):
+ summary_id = str(uuid.uuid4())
+ material_id = str(uuid.uuid4())
+ summary = TextualMemoryItem(
+ id=summary_id,
+ memory="summary",
+ metadata=TreeNodeTextualMemoryMetadata(memory_type="LongTermMemory"),
+ )
+ raw_file = TextualMemoryItem(
+ id=material_id,
+ memory="material",
+ metadata=TreeNodeTextualMemoryMetadata(
+ memory_type="RawFileMemory",
+ knowledge_summary_ids=[summary.id],
+ ),
+ )
+
+ result = mock_searcher._deduplicate_rawfile_results([summary, raw_file], user_name="cube-1")
+
+ assert result == [raw_file]
+ mock_searcher.graph_store.get_edges.assert_not_called()
diff --git a/uv.lock b/uv.lock
index 385ef4382..d081fa9d5 100644
--- a/uv.lock
+++ b/uv.lock
@@ -132,11 +132,11 @@ wheels = [
[[package]]
name = "certifi"
-version = "2026.2.25"
+version = "2026.10.19"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
]
[[package]]
@@ -2069,9 +2069,10 @@ wheels = [
[[package]]
name = "memoryos"
-version = "2.0.14"
+version = "2.0.33"
source = { editable = "." }
dependencies = [
+ { name = "cachetools" },
{ name = "concurrent-log-handler" },
{ name = "fastapi", extra = ["all"] },
{ name = "fastmcp" },
@@ -2090,7 +2091,6 @@ dependencies = [
[package.optional-dependencies]
all = [
{ name = "alibabacloud-oss-v2" },
- { name = "cachetools" },
{ name = "chonkie" },
{ name = "datasketch" },
{ name = "jieba" },
@@ -2142,7 +2142,7 @@ tree-mem = [
requires-dist = [
{ name = "alibabacloud-oss-v2", marker = "extra == 'all'", specifier = ">=1.2.2,<1.2.3" },
{ name = "alibabacloud-oss-v2", marker = "extra == 'skill-mem'", specifier = ">=1.2.2,<1.2.3" },
- { name = "cachetools", marker = "extra == 'all'", specifier = ">=6.0.0" },
+ { name = "cachetools", specifier = ">=6.0.0" },
{ name = "chonkie", marker = "extra == 'all'", specifier = ">=1.0.7,<2.0.0" },
{ name = "chonkie", marker = "extra == 'mem-reader'", specifier = ">=1.0.7,<2.0.0" },
{ name = "concurrent-log-handler", specifier = ">=0.9.28,<1.0.0" },