From 7fcb14025d9d4b1e0a148dba37d621af1f6126ec Mon Sep 17 00:00:00 2001 From: qzs Date: Fri, 4 Sep 2026 23:00:57 +0800 Subject: [PATCH 01/22] refactor(hooks): replace hook events with hook context --- src/memos/api/handlers/search_handler.py | 119 +++++-- src/memos/mem_reader/simple_struct.py | 101 ++++-- .../task_schedule_modules/base_handler.py | 26 +- .../handlers/mem_read_handler.py | 112 +++++-- .../operation_observer.py | 111 +++++++ src/memos/multi_mem_cube/single_cube.py | 63 +++- src/memos/plugins/README.md | 26 ++ src/memos/plugins/__init__.py | 13 +- src/memos/plugins/base.py | 2 +- src/memos/plugins/hook_context.py | 47 +++ src/memos/plugins/hook_defs.py | 122 ++++++- src/memos/plugins/hooks.py | 96 +++++- tests/api/test_search_pipeline_hooks.py | 183 +++++++++- tests/mem_reader/test_hook_pipeline.py | 121 +++++++ .../mem_scheduler/test_operation_observer.py | 150 +++++++++ .../multi_mem_cube/test_single_cube_hooks.py | 142 ++++++++ tests/plugins/test_hook_context.py | 312 ++++++++++++++++++ 17 files changed, 1639 insertions(+), 107 deletions(-) create mode 100644 src/memos/mem_scheduler/task_schedule_modules/operation_observer.py create mode 100644 src/memos/plugins/hook_context.py create mode 100644 tests/mem_reader/test_hook_pipeline.py create mode 100644 tests/mem_scheduler/test_operation_observer.py create mode 100644 tests/multi_mem_cube/test_single_cube_hooks.py create mode 100644 tests/plugins/test_hook_context.py diff --git a/src/memos/api/handlers/search_handler.py b/src/memos/api/handlers/search_handler.py index 363e0b253..c95a89270 100644 --- a/src/memos/api/handlers/search_handler.py +++ b/src/memos/api/handlers/search_handler.py @@ -15,6 +15,7 @@ from memos.api.handlers.base_handler import BaseHandler, HandlerDependencies from memos.api.handlers.formatters_handler import rerank_knowledge_mem from memos.api.product_models import APISearchRequest, SearchResponse +from memos.context.context import get_current_trace_id from memos.dream.contextualization import CONTEXT_MEMORY_TYPE from memos.log import get_logger, summarize_search_request, summarize_search_results from memos.memories.textual.tree_text_memory.retrieve.retrieve_utils import ( @@ -23,6 +24,7 @@ from memos.multi_mem_cube.composite_cube import CompositeCubeView from memos.multi_mem_cube.single_cube import SingleCubeView from memos.multi_mem_cube.views import MemCubeView +from memos.plugins.hook_context import HookContext, build_hook_context from memos.plugins.hook_defs import H from memos.plugins.hooks import hookable, trigger_hook @@ -65,8 +67,16 @@ def __init__(self, dependencies: HandlerDependencies): "naive_mem_cube", "mem_scheduler", "searcher", "deepsearch_agent" ) - @hookable("search") - def handle_search_memories(self, search_req: APISearchRequest) -> SearchResponse: + @hookable( + "search", + context_builder=lambda handler, request: handler._build_search_hook_context(request), + ) + def handle_search_memories( + self, + search_req: APISearchRequest, + *, + hook_context: HookContext | None = None, + ) -> SearchResponse: """ Main handler for search memories endpoint. @@ -75,6 +85,8 @@ def handle_search_memories(self, search_req: APISearchRequest) -> SearchResponse Args: search_req: Search request containing query and parameters + hook_context: Internal correlation context shared by all Hooks for + this search operation. A context is created when omitted. Returns: SearchResponse with formatted results @@ -86,6 +98,8 @@ def handle_search_memories(self, search_req: APISearchRequest) -> SearchResponse # Use deepcopy to avoid modifying the original request object search_req_local = copy.deepcopy(search_req) + if hook_context is None: + hook_context = self._build_search_hook_context(search_req_local) # Expand top_k for deduplication (5x to ensure enough candidates) if search_req_local.dedup in ("sim", "mmr"): @@ -94,46 +108,78 @@ def handle_search_memories(self, search_req: APISearchRequest) -> SearchResponse # Search and deduplicate cube_view = self._build_cube_view(search_req_local) results = cube_view.search_memories(search_req_local) + hooked_results = trigger_hook( H.SEARCH_MEMORY_RESULTS, + hook_context=hook_context, handler=self, search_req=search_req_local, results=results, ) if hooked_results is not None: results = hooked_results - if not search_req_local.relativity: - search_req_local.relativity = 0 - self.logger.info(f"[SearchHandler] Relativity filter: {search_req_local.relativity}") - results = self._apply_relativity_threshold(results, search_req_local.relativity) - - if search_req_local.dedup == "sim": - results = self._dedup_text_memories(results, search_req.top_k) - self._strip_embeddings(results) - elif search_req_local.dedup == "mmr": - pref_top_k = getattr(search_req_local, "pref_top_k", 6) - results = self._mmr_dedup_text_memories(results, search_req.top_k, pref_top_k) - self._strip_embeddings(results) - - text_mem = results["text_mem"] - results["text_mem"] = rerank_knowledge_mem( - self.reranker, - query=search_req.query, - text_mem=text_mem, - top_k=search_req_local.top_k, - file_mem_proportion=0.5, - ) - hooked_results = trigger_hook( - H.SEARCH_RESULTS_AFTER_RERANK, - handler=self, - search_req=search_req_local, - results=results, - ) - if hooked_results is not None: - results = hooked_results + try: + if not search_req_local.relativity: + search_req_local.relativity = 0 + self.logger.info(f"[SearchHandler] Relativity filter: {search_req_local.relativity}") + results = self._apply_relativity_threshold(results, search_req_local.relativity) + hooked_results = trigger_hook( + H.SEARCH_RESULTS_AFTER_THRESHOLD, + hook_context=hook_context, + handler=self, + search_req=search_req_local, + results=results, + ) + if hooked_results is not None: + results = hooked_results + + if search_req_local.dedup == "sim": + results = self._dedup_text_memories(results, search_req.top_k) + self._strip_embeddings(results) + elif search_req_local.dedup == "mmr": + pref_top_k = getattr(search_req_local, "pref_top_k", 6) + results = self._mmr_dedup_text_memories(results, search_req.top_k, pref_top_k) + self._strip_embeddings(results) + hooked_results = trigger_hook( + H.SEARCH_RESULTS_AFTER_DEDUP, + hook_context=hook_context, + handler=self, + search_req=search_req_local, + results=results, + ) + if hooked_results is not None: + results = hooked_results + + text_mem = results["text_mem"] + results["text_mem"] = rerank_knowledge_mem( + self.reranker, + query=search_req.query, + text_mem=text_mem, + top_k=search_req_local.top_k, + file_mem_proportion=0.5, + ) + hooked_results = trigger_hook( + H.SEARCH_RESULTS_AFTER_RERANK, + hook_context=hook_context, + handler=self, + search_req=search_req_local, + results=results, + ) + if hooked_results is not None: + results = hooked_results + except Exception as error: + trigger_hook( + H.SEARCH_POST_PROCESS_FAILED, + hook_context=hook_context, + handler=self, + search_req=search_req_local, + error=error, + ) + raise hooked_results = trigger_hook( H.SEARCH_CONTEXT_RENDER, + hook_context=hook_context, handler=self, search_req=search_req_local, results=results, @@ -1036,6 +1082,17 @@ def _resolve_cube_ids(self, search_req: APISearchRequest) -> list[str]: return [search_req.user_id] + def _build_search_hook_context(self, search_req: APISearchRequest) -> HookContext: + mode = search_req.mode.value if hasattr(search_req.mode, "value") else str(search_req.mode) + return build_hook_context( + source="api.search", + trace_id=get_current_trace_id(), + user_id=search_req.user_id, + session_id=search_req.session_id or "default_session", + cube_ids=self._resolve_cube_ids(search_req), + attributes={"mode": mode}, + ) + def _build_cube_view(self, search_req: APISearchRequest, searcher=None) -> MemCubeView: cube_ids = self._resolve_cube_ids(search_req) searcher_to_use = searcher if searcher is not None else self.searcher diff --git a/src/memos/mem_reader/simple_struct.py b/src/memos/mem_reader/simple_struct.py index 2804bac23..14521cf47 100644 --- a/src/memos/mem_reader/simple_struct.py +++ b/src/memos/mem_reader/simple_struct.py @@ -13,10 +13,13 @@ from memos.chunkers import ChunkerFactory from memos.configs.llm import LLMConfigFactory from memos.configs.mem_reader import SimpleStructMemReaderConfig -from memos.context.context import ContextThreadPoolExecutor +from memos.context.context import ContextThreadPoolExecutor, get_current_trace_id from memos.embedders.factory import EmbedderFactory from memos.llms.factory import LLMFactory from memos.mem_reader.base import BaseMemReader +from memos.plugins.hook_context import HookContext, build_hook_context +from memos.plugins.hook_defs import H +from memos.plugins.hooks import trigger_hook if TYPE_CHECKING: @@ -476,6 +479,33 @@ def _process_transfer_chat_data( return chat_read_nodes + def _build_extract_hook_data( + self, + scene_data: SceneDataInput, + type: str, + info: dict[str, Any], + mode: str, + user_name: str | None, + kwargs: dict[str, Any], + ) -> tuple[dict[str, Any], HookContext]: + hook_input = { + "mem_reader": self, + "scene_data": scene_data, + "type": type, + "info": info, + "mode": mode, + "user_name": user_name, + "kwargs": kwargs, + } + return hook_input, build_hook_context( + trace_id=get_current_trace_id(), + user_id=info.get("user_id") if isinstance(info, dict) else None, + session_id=info.get("session_id") if isinstance(info, dict) else None, + cube_ids=[user_name] if user_name else None, + source="mem_reader.extract", + attributes={"mode": mode, "type": type}, + ) + def get_memory( self, scene_data: SceneDataInput, @@ -509,27 +539,56 @@ def get_memory( Raises: ValueError: If scene_data is empty or if info dictionary is missing required fields """ - if not scene_data: - raise ValueError("scene_data is empty") - - # Validate info dictionary format - if not isinstance(info, dict): - raise ValueError("info must be a dictionary") - - required_fields = {"user_id", "session_id"} - missing_fields = required_fields - set(info.keys()) - if missing_fields: - raise ValueError(f"info dictionary is missing required fields: {missing_fields}") - - if not all(isinstance(info[field], str) for field in required_fields): - raise ValueError("user_id and session_id must be strings") - - # Backward compatibility, after coercing scene_data, we only tackle - # with standard scene_data type: MessagesType - standard_scene_data = coerce_scene_data(scene_data, type) - return self._read_memory( - standard_scene_data, type, info, mode, user_name=user_name, **kwargs + # Build Hook business arguments and correlation context. + hook_input, hook_context = self._build_extract_hook_data( + scene_data, + type, + info, + mode, + user_name, + kwargs, + ) + + try: + if not scene_data: + raise ValueError("scene_data is empty") + + # Validate info dictionary format + if not isinstance(info, dict): + raise ValueError("info must be a dictionary") + + required_fields = {"user_id", "session_id"} + missing_fields = required_fields - set(info.keys()) + if missing_fields: + raise ValueError(f"info dictionary is missing required fields: {missing_fields}") + + if not all(isinstance(info[field], str) for field in required_fields): + raise ValueError("user_id and session_id must be strings") + + # Backward compatibility, after coercing scene_data, we only tackle + # with standard scene_data type: MessagesType + standard_scene_data = coerce_scene_data(scene_data, type) + result = self._read_memory( + standard_scene_data, type, info, mode, user_name=user_name, **kwargs + ) + except Exception as error: + trigger_hook( + H.MEM_READER_EXTRACT_FAILED, + hook_context=hook_context, + **hook_input, + error=error, + ) + raise + + hook_result = trigger_hook( + H.MEM_READER_EXTRACT_AFTER, + hook_context=hook_context, + **hook_input, + result=result, ) + if hook_result is not None: + result = hook_result + return result def rewrite_memories( self, messages: list[dict], memory_list: list[TextualMemoryItem], user_only: bool = True diff --git a/src/memos/mem_scheduler/task_schedule_modules/base_handler.py b/src/memos/mem_scheduler/task_schedule_modules/base_handler.py index 603b038e1..438c30a0b 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/base_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/base_handler.py @@ -1,9 +1,12 @@ from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from memos.log import get_logger +from memos.mem_scheduler.task_schedule_modules.operation_observer import ( + SchedulerOperationObserver, +) from memos.mem_scheduler.utils.misc_utils import group_messages_by_user_and_mem_cube @@ -34,6 +37,27 @@ def validate_and_log_messages(self, messages: list[ScheduleMessageItem], label: def handle_exception(self, e: Exception, message: str = "Error processing messages") -> None: logger.error(f"{message}: {e}", exc_info=True) + def observe_operation( + self, + operation_name: str, + *, + operation: str, + target: str, + operation_context: Any, + operation_input: dict[str, Any], + capture_failure: bool = False, + ) -> SchedulerOperationObserver: + """Expose a scheduler operation through the standard plugin Hook contract.""" + source = f"scheduler.{self.expected_task_label}.{operation_name}" + return SchedulerOperationObserver( + source=source, + operation=operation, + target=target, + operation_context=operation_context, + operation_input=operation_input, + capture_failure=capture_failure, + ) + def process_grouped_messages( self, messages: list[ScheduleMessageItem], diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index 90ace633c..a61744785 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -6,7 +6,7 @@ import traceback from datetime import datetime, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from memos.context.context import ContextThreadPoolExecutor from memos.log import get_logger @@ -96,6 +96,7 @@ def process_message(self, message: ScheduleMessageItem): info=info, chat_history=chat_history, user_context=user_context, + operation_context=message, ) logger.info( @@ -119,6 +120,7 @@ def _process_memories_with_reader( info: dict | None = None, chat_history: list | None = None, user_context: UserContext | None = None, + operation_context: Any | None = None, ) -> None: logger.info( "[DIAGNOSTIC] mem_read_handler._process_memories_with_reader called. mem_ids: %s, user_id: %s, mem_cube_id: %s, task_id: %s", @@ -127,6 +129,12 @@ def _process_memories_with_reader( mem_cube_id, task_id, ) + operation_context = operation_context or { + "task_id": task_id, + "label": MEM_READ_TASK_LABEL, + "user_id": user_id, + "cube_id": mem_cube_id, + } kb_log_content: list[dict] = [] try: mem_reader = self.scheduler_context.get_mem_reader() @@ -169,15 +177,26 @@ def _process_memories_with_reader( is_upload_skill = info.pop("is_upload_skill", False) try: - processed_memories = mem_reader.fine_transfer_simple_mem( - memory_items, - type="chat", - custom_tags=custom_tags, - user_name=user_name, - chat_history=chat_history, - user_context=user_context, - is_upload_skill=is_upload_skill, - ) + with self.observe_operation( + "fine_transfer_simple_mem", + operation="fine", + target="textual_memory", + operation_context=operation_context, + operation_input={"memories": memory_items, "type": "chat"}, + capture_failure=True, + ) as observation: + observation.result( + mem_reader.fine_transfer_simple_mem( + memory_items, + type="chat", + custom_tags=custom_tags, + user_name=user_name, + chat_history=chat_history, + user_context=user_context, + is_upload_skill=is_upload_skill, + ) + ) + processed_memories = observation.value except Exception as e: logger.warning("%s: Fail to transfer mem: %s", e, memory_items) processed_memories = [] @@ -195,7 +214,16 @@ def _process_memories_with_reader( for memory in flattened_memories if memory.metadata.memory_type != "RawFileMemory" ] - enhanced_mem_ids = text_mem.add(mem_group, user_name=user_name) + with self.observe_operation( + "add_enhanced_memories", + operation="add", + target="textual_memory", + operation_context=operation_context, + operation_input={"memories": mem_group, "user_name": user_name}, + capture_failure=True, + ) as observation: + observation.result(text_mem.add(mem_group, user_name=user_name)) + enhanced_mem_ids = observation.value logger.info( "Added %s enhanced memories: %s", len(enhanced_mem_ids), @@ -244,11 +272,21 @@ def _process_memories_with_reader( ) for old_id in old_ids: try: - mem_reader.graph_db.update_node( - str(old_id), - {"status": "archived"}, - user_name=user_name, - ) + with self.observe_operation( + "archive_merged_memories", + operation="archive", + target="textual_memory", + operation_context=operation_context, + operation_input={"memory_ids": [str(old_id)]}, + capture_failure=True, + ) as observation: + observation.result( + mem_reader.graph_db.update_node( + str(old_id), + {"status": "archived"}, + user_name=user_name, + ) + ) logger.info( "[Scheduler] Archived merged_from memory: %s", old_id, @@ -405,7 +443,15 @@ def _process_memories_with_reader( if delete_ids: try: if getattr(mem_reader, "memory_version_switch", "off") != "on": - text_mem.delete(delete_ids, user_name=user_name) + with self.observe_operation( + "remove_memories", + operation="delete", + target="textual_memory", + operation_context=operation_context, + operation_input={"memory_ids": delete_ids}, + capture_failure=True, + ) as observation: + observation.result(text_mem.delete(delete_ids, user_name=user_name)) logger.info( "Delete raw/working mem_ids: %s for user_name: %s", delete_ids, @@ -418,15 +464,29 @@ def _process_memories_with_reader( for memory_list in processed_memories: flattened_memories.extend(memory_list) allowed_types = ["UserMemory", "LongTermMemory"] - text_mem.soft_delete( - delete_ids, - user_name, - [ - mem.id - for mem in flattened_memories - if mem.metadata.memory_type in allowed_types - ], - ) + preserved_memory_ids = [ + mem.id + for mem in flattened_memories + if mem.metadata.memory_type in allowed_types + ] + with self.observe_operation( + "remove_memories", + operation="soft_delete", + target="textual_memory", + operation_context=operation_context, + operation_input={ + "memory_ids": delete_ids, + "preserved_memory_ids": preserved_memory_ids, + }, + capture_failure=True, + ) as observation: + observation.result( + text_mem.soft_delete( + delete_ids, + user_name, + preserved_memory_ids, + ) + ) logger.info( "Soft delete raw/working mem_ids: %s for user_name: %s", delete_ids, diff --git a/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py b/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py new file mode 100644 index 000000000..7a151d4ab --- /dev/null +++ b/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from collections.abc import Mapping +from contextlib import AbstractContextManager +from typing import Any + +from memos.plugins.hook_context import build_hook_context +from memos.plugins.hook_defs import H +from memos.plugins.hooks import trigger_hook + + +_MISSING = object() + + +class SchedulerOperationObserver(AbstractContextManager["SchedulerOperationObserver"]): + """Observe one Handler operation without changing its exception semantics. + + Usage: + with SchedulerOperationObserver(...) as observer: + result = perform_operation() + observer.result(result) + """ + + def __init__( + self, + *, + source: str, + operation: str, + target: str, + operation_context: Any, + operation_input: Mapping[str, Any], + capture_failure: bool = False, + ) -> None: + self._hook_kwargs = { + "operation": operation, + "target": target, + "operation_input": dict(operation_input), + } + cube_id = self._context_value(operation_context, "cube_id") or self._context_value( + operation_context, "mem_cube_id" + ) + self._hook_context = build_hook_context( + trace_id=self._context_value(operation_context, "trace_id"), + user_id=self._context_value(operation_context, "user_id"), + session_id=self._context_value(operation_context, "session_id"), + task_id=self._context_value(operation_context, "task_id"), + cube_ids=[str(cube_id)] if cube_id else None, + source=source, + attributes=self._attributes(operation_context), + ) + self._capture_failure = capture_failure + self._result: Any = _MISSING + self._failed = False + + def __enter__(self) -> SchedulerOperationObserver: + return self + + def __exit__(self, exc_type, exc_value, traceback) -> bool: + if exc_value is not None: + if self._capture_failure: + self.failed(exc_value) + return False + + if not self._failed and self._result is not _MISSING: + hook_result = trigger_hook( + H.SCHEDULER_MEMORY_OPERATION_AFTER, + hook_context=self._hook_context, + **self._hook_kwargs, + result=self._result, + ) + if hook_result is not None: + self._result = hook_result + return False + + def result(self, value: Any) -> Any: + """Record and return the operation result for concise call sites.""" + self._result = value + return value + + @property + def value(self) -> Any: + """Return the result after successful Hook pipeline processing.""" + return None if self._result is _MISSING else self._result + + def failed(self, error: BaseException) -> None: + """Emit the failure Hook once at an existing exception boundary.""" + if self._failed: + return + self._failed = True + trigger_hook( + H.SCHEDULER_MEMORY_OPERATION_FAILED, + hook_context=self._hook_context, + **self._hook_kwargs, + error=error, + ) + + @staticmethod + def _context_value(operation_context: Any, name: str) -> Any: + if isinstance(operation_context, Mapping): + return operation_context.get(name) + return getattr(operation_context, name, None) + + @classmethod + def _attributes(cls, operation_context: Any) -> dict[str, Any]: + attributes = { + "scheduler_item_id": cls._context_value(operation_context, "scheduler_item_id") + or cls._context_value(operation_context, "item_id"), + "redis_message_id": cls._context_value(operation_context, "redis_message_id"), + "label": cls._context_value(operation_context, "label"), + } + return {key: value for key, value in attributes.items() if value is not None} diff --git a/src/memos/multi_mem_cube/single_cube.py b/src/memos/multi_mem_cube/single_cube.py index 27399259a..8d78adbff 100644 --- a/src/memos/multi_mem_cube/single_cube.py +++ b/src/memos/multi_mem_cube/single_cube.py @@ -12,6 +12,7 @@ format_memory_item, post_process_textual_mem, ) +from memos.context.context import get_current_trace_id from memos.log import get_logger, summarize_search_request, summarize_search_results from memos.mem_reader.utils import parse_keep_filter_response from memos.mem_scheduler.schemas.message_schemas import ScheduleMessageItem @@ -22,6 +23,9 @@ ) from memos.memories.textual.item import TextualMemoryItem from memos.multi_mem_cube.views import MemCubeView +from memos.plugins.hook_context import HookContext, build_hook_context +from memos.plugins.hook_defs import H +from memos.plugins.hooks import trigger_hook from memos.search import resolve_filter_for_cube, search_text_memories from memos.templates.mem_reader_prompts import PROMPT_MAPPING from memos.types.general_types import ( @@ -658,6 +662,32 @@ def add_before_search( self.logger.error(f"[add_before_search] LLM execution error: {e}") return memory_list + def _build_text_memory_add_hook_data( + self, + *, + add_req: APIADDRequest, + user_context: UserContext, + text_memory: Any, + memories: list[TextualMemoryItem], + session_id: str, + sync_mode: str, + extract_mode: str, + ) -> tuple[dict[str, Any], HookContext]: + hook_input = { + "text_memory": text_memory, + "memories": memories, + "kwargs": {"user_name": user_context.mem_cube_id}, + } + return hook_input, build_hook_context( + trace_id=get_current_trace_id(), + user_id=add_req.user_id, + session_id=session_id, + task_id=add_req.task_id, + cube_ids=[self.cube_id], + source="cube_view.text_mem.add", + attributes={"sync_mode": sync_mode, "extract_mode": extract_mode}, + ) + @timed def _process_text_mem( self, @@ -731,10 +761,37 @@ def _process_text_mem( # Stage 3: write_db with timed_stage("add", "write_db", cube_id=self.cube_id) as ts_db: - mem_ids_local: list[str] = self.naive_mem_cube.text_mem.add( - mem_group, - user_name=user_context.mem_cube_id, + text_memory = self.naive_mem_cube.text_mem + hook_input, hook_context = self._build_text_memory_add_hook_data( + add_req=add_req, + user_context=user_context, + text_memory=text_memory, + memories=mem_group, + session_id=target_session_id, + sync_mode=sync_mode, + extract_mode=extract_mode, + ) + + try: + mem_ids_local: list[str] = text_memory.add( + mem_group, user_name=user_context.mem_cube_id + ) + except Exception as error: + trigger_hook( + H.TEXT_MEMORY_ADD_FAILED, + hook_context=hook_context, + **hook_input, + error=error, + ) + raise + hook_result = trigger_hook( + H.TEXT_MEMORY_ADD_AFTER, + hook_context=hook_context, + **hook_input, + result=mem_ids_local, ) + if hook_result is not None: + mem_ids_local = hook_result self.logger.info( f"Added {len(mem_ids_local)} memories for user {add_req.user_id} " diff --git a/src/memos/plugins/README.md b/src/memos/plugins/README.md index c9b6a0c29..859fe76d7 100644 --- a/src/memos/plugins/README.md +++ b/src/memos/plugins/README.md @@ -269,8 +269,16 @@ H.ADD_AFTER H.SEARCH_BEFORE H.SEARCH_AFTER H.SEARCH_MEMORY_RESULTS +H.SEARCH_RESULTS_AFTER_THRESHOLD +H.SEARCH_RESULTS_AFTER_DEDUP +H.SEARCH_RESULTS_AFTER_RERANK +H.SEARCH_CONTEXT_RENDER +H.SEARCH_POST_PROCESS_FAILED H.MEM_READER_PRE_EXTRACT H.MEMORY_ITEMS_AFTER_FINE_EXTRACT +H.MEM_READER_EXTRACT_AFTER +H.TEXT_MEMORY_ADD_AFTER +H.SCHEDULER_MEMORY_OPERATION_AFTER H.DREAM_EXECUTE ``` @@ -278,6 +286,24 @@ Hook callbacks receive keyword arguments. Some Hooks define a `pipe_key`; when a callback returns a non-`None` value, that value replaces the named argument for later callbacks and for the caller. +`hook_context` is additive correlation metadata, not a separate event model. +Context-aware callbacks can declare it explicitly or accept `**kwargs`. Legacy +callbacks with an exact signature remain compatible: the runtime omits only an +unsupported `hook_context` argument and still invokes the callback once. New +callbacks should accept `**kwargs` so later additive metadata remains backward +compatible. + +All Search Hooks triggered for one request—from `SEARCH_BEFORE` through +`SEARCH_AFTER`, including result and post-processing Hooks—receive the same +`HookContext` instance and `operation_id`. + +The MemReader extraction, Textual Memory add, and successful Scheduler +operation Hooks pipe their `result` value. Their matching failure Hooks are +notifications and therefore do not define a `pipe_key`. + +`SEARCH_POST_PROCESS_FAILED` reports a result post-processing error that +continues to propagate to the Search API caller. + For example, `add.after` can replace `result`: ```python diff --git a/src/memos/plugins/__init__.py b/src/memos/plugins/__init__.py index 0a0f8cde3..213c8ed9f 100644 --- a/src/memos/plugins/__init__.py +++ b/src/memos/plugins/__init__.py @@ -1,15 +1,26 @@ from memos.plugins.base import MemOSPlugin +from memos.plugins.hook_context import ( + HookContext, + build_hook_context, +) from memos.plugins.hook_defs import H, HookSpec, all_hook_specs, define_hook, get_hook_spec -from memos.plugins.hooks import hookable, register_hook, register_hooks, trigger_hook +from memos.plugins.hooks import ( + hookable, + register_hook, + register_hooks, + trigger_hook, +) from memos.plugins.manager import PluginManager, plugin_manager __all__ = [ "H", + "HookContext", "HookSpec", "MemOSPlugin", "PluginManager", "all_hook_specs", + "build_hook_context", "define_hook", "get_hook_spec", "hookable", diff --git a/src/memos/plugins/base.py b/src/memos/plugins/base.py index 24d356bbf..ad494a6f1 100644 --- a/src/memos/plugins/base.py +++ b/src/memos/plugins/base.py @@ -15,7 +15,7 @@ class MemOSPlugin: """MemOS plugin base class. - Provides three unified registration methods. Plugin developers need only + Provides unified registration methods. Plugin developers need only inherit from this class and register capabilities via self.register_* in init_app. diff --git a/src/memos/plugins/hook_context.py b/src/memos/plugins/hook_context.py new file mode 100644 index 000000000..7b6f3637c --- /dev/null +++ b/src/memos/plugins/hook_context.py @@ -0,0 +1,47 @@ +"""Execution context shared by related plugin Hook calls.""" + +from __future__ import annotations + +# Keep this import available for runtime inspection of the public annotations. +from collections.abc import Mapping # noqa: TC003 +from dataclasses import dataclass, field +from typing import Any +from uuid import uuid4 + + +@dataclass(frozen=True, slots=True) +class HookContext: + """Correlation metadata for one Hook-observable operation.""" + + trace_id: str | None = None + user_id: str | None = None + session_id: str | None = None + task_id: str | None = None + cube_ids: tuple[str, ...] = () + operation_id: str | None = None + source: str | None = None + attributes: Mapping[str, Any] = field(default_factory=dict) + + +def build_hook_context( + *, + source: str, + trace_id: str | None = None, + user_id: str | None = None, + session_id: str | None = None, + task_id: str | None = None, + cube_ids: list[str] | tuple[str, ...] | None = None, + operation_id: str | None = None, + attributes: Mapping[str, Any] | None = None, +) -> HookContext: + """Build correlation metadata at a business-operation boundary.""" + return HookContext( + trace_id=trace_id, + user_id=user_id, + session_id=session_id, + task_id=task_id, + cube_ids=tuple(cube_ids or ()), + operation_id=operation_id or uuid4().hex, + source=source, + attributes=dict(attributes or {}), + ) diff --git a/src/memos/plugins/hook_defs.py b/src/memos/plugins/hook_defs.py index f955da0af..87375e942 100644 --- a/src/memos/plugins/hook_defs.py +++ b/src/memos/plugins/hook_defs.py @@ -74,8 +74,11 @@ class H: # Search extension point before core threshold/dedup/rerank processing. SEARCH_MEMORY_RESULTS = "search.memory_results" + SEARCH_RESULTS_AFTER_THRESHOLD = "search.results.after_threshold" + SEARCH_RESULTS_AFTER_DEDUP = "search.results.after_dedup" SEARCH_RESULTS_AFTER_RERANK = "search.results.after_rerank" SEARCH_CONTEXT_RENDER = "search.context.render" + SEARCH_POST_PROCESS_FAILED = "search.post_process.failed" # Custom Hook (manually triggered via trigger_hook) ADD_MEMORIES_POST_PROCESS = "add.memories.post_process" @@ -92,9 +95,21 @@ class H: # dream — single-provider business hook DREAM_EXECUTE = "dream.execute" + # textual memory — write boundary + TEXT_MEMORY_ADD_AFTER = "text_memory.add.after" + TEXT_MEMORY_ADD_FAILED = "text_memory.add.failed" + + # scheduler — memory operation boundaries inside task handlers + SCHEDULER_MEMORY_OPERATION_AFTER = "scheduler.memory.operation.after" + SCHEDULER_MEMORY_OPERATION_FAILED = "scheduler.memory.operation.failed" + + # mem_reader — generic extension point before LLM extraction + MEM_READER_EXTRACT_AFTER = "mem_reader.extract.after" + MEM_READER_EXTRACT_FAILED = "mem_reader.extract.failed" + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# CE custom Hook declarations (@hookable-generated ones need not be declared here) +# CE Hook declarations # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ define_hook( @@ -111,30 +126,67 @@ class H: pipe_key="prompt", ) +define_hook( + H.SEARCH_BEFORE, + description="Before search executes; can modify request", + params=["hook_context", "request"], + pipe_key="request", +) + +define_hook( + H.SEARCH_AFTER, + description="After search executes; can modify result", + params=["hook_context", "request", "result"], + pipe_key="result", +) + define_hook( H.SEARCH_MEMORY_RESULTS, description=( "Allow plugins to merge additional search result buckets before core " "threshold, deduplication, and reranking." ), - params=["handler", "search_req", "results"], + params=["hook_context", "handler", "search_req", "results"], + pipe_key="results", +) + +define_hook( + H.SEARCH_RESULTS_AFTER_THRESHOLD, + description="Allow plugins to observe or update search results after threshold filtering.", + params=["hook_context", "handler", "search_req", "results"], + pipe_key="results", +) + +define_hook( + H.SEARCH_RESULTS_AFTER_DEDUP, + description="Allow plugins to observe or update search results after deduplication.", + params=["hook_context", "handler", "search_req", "results"], pipe_key="results", ) define_hook( H.SEARCH_RESULTS_AFTER_RERANK, description="Allow plugins to update search results after core rerank and before rendering.", - params=["handler", "search_req", "results"], + params=["hook_context", "handler", "search_req", "results"], pipe_key="results", ) define_hook( H.SEARCH_CONTEXT_RENDER, description="Render final search context after retrieval, rerank, and result-level plugins.", - params=["handler", "search_req", "results"], + params=["hook_context", "handler", "search_req", "results"], pipe_key="results", ) +define_hook( + H.SEARCH_POST_PROCESS_FAILED, + description=( + "Observe a failure during core search result processing: " + "threshold filtering, deduplication, or reranking" + ), + params=["hook_context", "handler", "search_req", "error"], +) + define_hook( H.MEMORY_ITEMS_AFTER_FINE_EXTRACT, description="Post-process memory items after mem_reader fine extraction completes", @@ -183,3 +235,65 @@ class H: "scheduler_context", ], ) + +# Operation-boundary Hooks. Their business data remains explicit in the +# normal Hook keyword-argument contract; HookContext only carries correlation +# metadata. +define_hook( + H.TEXT_MEMORY_ADD_AFTER, + description="Observe or replace CubeView Textual Memory output after add returns", + params=["hook_context", "text_memory", "memories", "kwargs", "result"], + pipe_key="result", +) + +define_hook( + H.TEXT_MEMORY_ADD_FAILED, + description="Observe CubeView Textual Memory input and output after add fails", + params=["hook_context", "text_memory", "memories", "kwargs", "error"], +) + +define_hook( + H.SCHEDULER_MEMORY_OPERATION_AFTER, + description="Observe or replace a successful Scheduler memory operation result", + params=["hook_context", "operation", "target", "operation_input", "result"], + pipe_key="result", +) + +define_hook( + H.SCHEDULER_MEMORY_OPERATION_FAILED, + description="Observe a failed Scheduler memory operation and its error", + params=["hook_context", "operation", "target", "operation_input", "error"], +) + +define_hook( + H.MEM_READER_EXTRACT_AFTER, + description="Observe or replace MemReader output after extraction completes", + params=[ + "hook_context", + "mem_reader", + "scene_data", + "type", + "info", + "mode", + "user_name", + "kwargs", + "result", + ], + pipe_key="result", +) + +define_hook( + H.MEM_READER_EXTRACT_FAILED, + description="Observe MemReader extraction input and propagated error after processing fails", + params=[ + "hook_context", + "mem_reader", + "scene_data", + "type", + "info", + "mode", + "user_name", + "kwargs", + "error", + ], +) diff --git a/src/memos/plugins/hooks.py b/src/memos/plugins/hooks.py index d5b64ce82..4128da3b7 100644 --- a/src/memos/plugins/hooks.py +++ b/src/memos/plugins/hooks.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import inspect import logging from collections import defaultdict @@ -19,6 +20,43 @@ _hooks: dict[str, list[Callable]] = defaultdict(list) +def _accepts_hook_context(callback: Callable) -> bool: + """Return whether a callback accepts the additive ``hook_context`` keyword.""" + try: + parameters = inspect.signature(callback).parameters.values() + except (TypeError, ValueError): + return False + + return any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + or ( + parameter.name == "hook_context" + and parameter.kind + in { + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + } + ) + for parameter in parameters + ) + + +def _invoke_hook_callback(callback: Callable, kwargs: dict[str, Any]) -> Any: + """Invoke one callback while preserving legacy exact-signature Hooks. + + ``hook_context`` is additive framework metadata. Older callbacks that use + an exact keyword-only signature continue to receive the original business + arguments; context-aware callbacks and callbacks with ``**kwargs`` receive + the complete contract. + """ + if "hook_context" not in kwargs or _accepts_hook_context(callback): + return callback(**kwargs) + + legacy_kwargs = dict(kwargs) + legacy_kwargs.pop("hook_context") + return callback(**legacy_kwargs) + + def register_hook(name: str, callback: Callable) -> None: """Register a hook callback. Undeclared hook names will log a warning.""" from memos.plugins.hook_defs import get_hook_spec @@ -61,7 +99,7 @@ def trigger_hook(name: str, **kwargs: Any) -> Any: for cb in _hooks.get(name, []): try: - rv = cb(**kwargs) + rv = _invoke_hook_callback(cb, kwargs) if pipe_key is not None and rv is not None: kwargs[pipe_key] = rv except Exception: @@ -90,7 +128,7 @@ def trigger_single_hook(name: str, **kwargs: Any) -> Any: cb = callbacks[0] try: - return cb(**kwargs) + return _invoke_hook_callback(cb, kwargs) except Exception: logger.exception( "Single hook %s callback %s failed", @@ -100,25 +138,37 @@ def trigger_single_hook(name: str, **kwargs: Any) -> Any: raise -def hookable(name: str): +def hookable( + name: str, + *, + context_builder: Callable[[Any, Any], Any] | None = None, +): """Decorator: automatically triggers name.before / name.after hook before and after the method. Auto-declares before/after Hooks (idempotent); no need to manually define_hook in hook_defs.py. Supports piped return values: before can modify request, after can modify result. + When ``context_builder`` is provided, one context is built before the first + Hook and passed to both Hooks and the decorated method as ``hook_context``. Compatible with both sync and async methods. """ from memos.plugins.hook_defs import define_hook + before_params = ["request"] + after_params = ["request", "result"] + if context_builder is not None: + before_params.insert(0, "hook_context") + after_params.insert(0, "hook_context") + define_hook( f"{name}.before", description=f"Before {name} executes; can modify request", - params=["request"], + params=before_params, pipe_key="request", ) define_hook( f"{name}.after", description=f"After {name} executes; can modify result", - params=["request", "result"], + params=after_params, pipe_key="result", ) @@ -127,10 +177,22 @@ def decorator(func): @wraps(func) async def async_wrapper(self, request, *args, **kwargs): - rv = trigger_hook(f"{name}.before", request=request) + hook_context = kwargs.get("hook_context") + if hook_context is None and context_builder is not None: + hook_context = context_builder(self, request) + before_kwargs = {"request": request} + call_kwargs = dict(kwargs) + if context_builder is not None: + before_kwargs["hook_context"] = hook_context + call_kwargs["hook_context"] = hook_context + + rv = trigger_hook(f"{name}.before", **before_kwargs) request = rv if rv is not None else request - result = await func(self, request, *args, **kwargs) - rv = trigger_hook(f"{name}.after", request=request, result=result) + result = await func(self, request, *args, **call_kwargs) + after_kwargs = {"request": request, "result": result} + if context_builder is not None: + after_kwargs["hook_context"] = hook_context + rv = trigger_hook(f"{name}.after", **after_kwargs) result = rv if rv is not None else result return result @@ -138,10 +200,22 @@ async def async_wrapper(self, request, *args, **kwargs): @wraps(func) def sync_wrapper(self, request, *args, **kwargs): - rv = trigger_hook(f"{name}.before", request=request) + hook_context = kwargs.get("hook_context") + if hook_context is None and context_builder is not None: + hook_context = context_builder(self, request) + before_kwargs = {"request": request} + call_kwargs = dict(kwargs) + if context_builder is not None: + before_kwargs["hook_context"] = hook_context + call_kwargs["hook_context"] = hook_context + + rv = trigger_hook(f"{name}.before", **before_kwargs) request = rv if rv is not None else request - result = func(self, request, *args, **kwargs) - rv = trigger_hook(f"{name}.after", request=request, result=result) + result = func(self, request, *args, **call_kwargs) + after_kwargs = {"request": request, "result": result} + if context_builder is not None: + after_kwargs["hook_context"] = hook_context + rv = trigger_hook(f"{name}.after", **after_kwargs) result = rv if rv is not None else result return result diff --git a/tests/api/test_search_pipeline_hooks.py b/tests/api/test_search_pipeline_hooks.py index befff6b80..329a03350 100644 --- a/tests/api/test_search_pipeline_hooks.py +++ b/tests/api/test_search_pipeline_hooks.py @@ -1,6 +1,20 @@ +import pytest + +from memos.api.handlers.base_handler import HandlerDependencies from memos.api.handlers.formatters_handler import rerank_knowledge_mem +from memos.api.handlers.search_handler import SearchHandler from memos.api.product_models import APISearchRequest +from memos.multi_mem_cube.single_cube import SingleCubeView +from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H, get_hook_spec +from memos.plugins.hooks import _hooks, register_hook + + +@pytest.fixture(autouse=True) +def _reset_registered_hooks(): + _hooks.clear() + yield + _hooks.clear() def _memory(memory_id: str, memory: str, memory_type: str = "LongTermMemory") -> dict: @@ -27,6 +41,39 @@ def _file_memory(memory_id: str, memory: str, source_content: str) -> dict: } +def _empty_search_results() -> dict: + return { + "text_mem": [], + "act_mem": [], + "para_mem": [], + "pref_mem": [], + "pref_note": "", + "tool_mem": [], + "skill_mem": [], + } + + +def _search_handler(results: dict | None = None) -> tuple[SearchHandler, SingleCubeView]: + handler = SearchHandler( + HandlerDependencies( + naive_mem_cube=object(), + mem_scheduler=object(), + searcher=object(), + deepsearch_agent=object(), + ) + ) + cube_view = SingleCubeView.__new__(SingleCubeView) + search_results = results if results is not None else _empty_search_results() + cube_view.search_memories = lambda _request: search_results + # Preserve the existing _build_cube_view(search_req) call contract. + handler._build_cube_view = lambda _search_req: cube_view + return handler, cube_view + + +def _raise(error: Exception): + raise error + + def test_search_request_passes_context_format_through_to_plugins(): req = APISearchRequest( user_id="user", @@ -38,16 +85,136 @@ def test_search_request_passes_context_format_through_to_plugins(): def test_search_pipeline_hook_specs_are_registered(): - after_rerank = get_hook_spec(H.SEARCH_RESULTS_AFTER_RERANK) - render = get_hook_spec(H.SEARCH_CONTEXT_RENDER) + for hook_name in (H.SEARCH_RESULTS_AFTER_RERANK, H.SEARCH_CONTEXT_RENDER): + spec = get_hook_spec(hook_name) + assert spec is not None + assert spec.pipe_key == "results" + assert spec.params == ["hook_context", "handler", "search_req", "results"] + + +@pytest.mark.parametrize( + ("hook_name", "params", "pipe_key"), + [ + (H.SEARCH_BEFORE, ["hook_context", "request"], "request"), + (H.SEARCH_AFTER, ["hook_context", "request", "result"], "result"), + ( + H.SEARCH_MEMORY_RESULTS, + ["hook_context", "handler", "search_req", "results"], + "results", + ), + ( + H.SEARCH_RESULTS_AFTER_THRESHOLD, + ["hook_context", "handler", "search_req", "results"], + "results", + ), + ( + H.SEARCH_RESULTS_AFTER_DEDUP, + ["hook_context", "handler", "search_req", "results"], + "results", + ), + ( + H.SEARCH_POST_PROCESS_FAILED, + ["hook_context", "handler", "search_req", "error"], + None, + ), + ], +) +def test_search_hook_specs_match_trigger_contracts(hook_name, params, pipe_key): + spec = get_hook_spec(hook_name) + + assert spec is not None + assert spec.params == params + assert spec.pipe_key == pipe_key + + +def test_search_success_hooks_run_in_order_and_share_one_context(monkeypatch): + events: list[tuple[str, dict]] = [] + handler, cube_view = _search_handler() + search_req = APISearchRequest( + user_id="user", + query="query", + session_id="session-1", + readable_cube_ids=["cube-1", "cube-2", "cube-1"], + ) + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", + lambda: "trace-1", + ) + + def record(hook_name): + def callback(**kwargs): + events.append((hook_name, kwargs)) + + return callback + + ordered_hooks = [ + H.SEARCH_BEFORE, + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + H.SEARCH_CONTEXT_RENDER, + H.SEARCH_AFTER, + ] + for hook_name in ordered_hooks: + register_hook(hook_name, record(hook_name)) + + handler.handle_search_memories(search_req) + + assert [name for name, _kwargs in events] == ordered_hooks + contexts = [kwargs["hook_context"] for _name, kwargs in events] + assert all(context is contexts[0] for context in contexts) + context = contexts[0] + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "user" + assert context.session_id == "session-1" + assert context.cube_ids == ("cube-1", "cube-2") + assert context.source == "api.search" + assert context.attributes == {"mode": "fast"} + assert context.operation_id is not None + assert not hasattr(cube_view, "search_hook_context") + - assert after_rerank is not None - assert after_rerank.pipe_key == "results" - assert after_rerank.params == ["handler", "search_req", "results"] +def test_search_context_render_hook_can_replace_results(): + handler, _cube_view = _search_handler() + search_req = APISearchRequest(user_id="user", query="query") + + def render(*, results, **_kwargs): + return {**results, "rendered_by_plugin": True} + + register_hook(H.SEARCH_CONTEXT_RENDER, render) + + response = handler.handle_search_memories(search_req) + + assert response.data["rendered_by_plugin"] is True + + +def test_search_post_processing_failure_triggers_only_post_process_hook(monkeypatch): + error = RuntimeError("post processing failed") + post_process_failures = [] + handler, _cube_view = _search_handler() + handler._apply_relativity_threshold = lambda _results, _relativity: _raise(error) + search_req = APISearchRequest(user_id="user", query="query") + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", + lambda: "trace-1", + ) + + register_hook( + H.SEARCH_POST_PROCESS_FAILED, + lambda **kwargs: post_process_failures.append(kwargs), + ) + with pytest.raises(RuntimeError, match="post processing failed"): + handler.handle_search_memories(search_req) - assert render is not None - assert render.pipe_key == "results" - assert render.params == ["handler", "search_req", "results"] + assert len(post_process_failures) == 1 + assert post_process_failures[0]["error"] is error + context = post_process_failures[0]["hook_context"] + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "user" + assert context.operation_id is not None def test_rerank_knowledge_mem_preserves_conversation_sources_by_default(): diff --git a/tests/mem_reader/test_hook_pipeline.py b/tests/mem_reader/test_hook_pipeline.py new file mode 100644 index 000000000..986fcffda --- /dev/null +++ b/tests/mem_reader/test_hook_pipeline.py @@ -0,0 +1,121 @@ +"""MemReader success/failure Hook contracts and result piping.""" + +from __future__ import annotations + +from types import MethodType +from uuid import UUID + +import pytest + +from memos.mem_reader.simple_struct import SimpleStructMemReader +from memos.plugins.hook_context import HookContext +from memos.plugins.hook_defs import H +from memos.plugins.hooks import _hooks, register_hook + + +@pytest.fixture(autouse=True) +def _reset_registered_hooks(): + _hooks.clear() + yield + _hooks.clear() + + +def test_extract_after_hook_can_replace_reader_result(): + reader = object.__new__(SimpleStructMemReader) + original_result = [["original"]] + plugin_result = [["plugin"]] + + def fake_read_memory(self, *args, **kwargs): + return original_result + + reader._read_memory = MethodType(fake_read_memory, reader) + register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **_kwargs: plugin_result) + + result = reader.get_memory( + [{"role": "user", "content": "remember this"}], + type="chat", + info={"user_id": "alice", "session_id": "session-1"}, + ) + + assert result == plugin_result + + +def test_extract_after_hook_receives_context_and_business_values(monkeypatch): + reader = object.__new__(SimpleStructMemReader) + result = [["memory"]] + scene_data = [{"role": "user", "content": "remember this"}] + info = {"user_id": "alice", "session_id": "session-1"} + captured = {} + + def fake_read_memory(self, *args, **kwargs): + return result + + reader._read_memory = MethodType(fake_read_memory, reader) + monkeypatch.setattr("memos.mem_reader.simple_struct.get_current_trace_id", lambda: "trace-1") + register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **kwargs: captured.update(kwargs)) + + actual = reader.get_memory( + scene_data, + type="chat", + info=info, + mode="fine", + user_name="cube-1", + chat_history=[], + ) + + assert actual is result + context = captured.pop("hook_context") + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "alice" + assert context.session_id == "session-1" + assert context.cube_ids == ("cube-1",) + assert context.source == "mem_reader.extract" + assert context.attributes == {"mode": "fine", "type": "chat"} + assert context.operation_id is not None + assert UUID(context.operation_id).hex == context.operation_id + assert captured == { + "mem_reader": reader, + "scene_data": scene_data, + "type": "chat", + "info": info, + "mode": "fine", + "user_name": "cube-1", + "kwargs": {"chat_history": []}, + "result": result, + } + + +def test_extract_failure_hook_preserves_original_error_and_skips_after(monkeypatch): + reader = object.__new__(SimpleStructMemReader) + error = ValueError("reader failed") + failed_events = [] + after_events = [] + + def fake_read_memory(self, *args, **kwargs): + raise error + + def on_failed(**kwargs): + failed_events.append(kwargs) + raise RuntimeError("plugin failed") + + reader._read_memory = MethodType(fake_read_memory, reader) + monkeypatch.setattr("memos.mem_reader.simple_struct.get_current_trace_id", lambda: "trace-1") + register_hook(H.MEM_READER_EXTRACT_FAILED, on_failed) + register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **kwargs: after_events.append(kwargs)) + + with pytest.raises(ValueError, match="reader failed") as exc_info: + reader.get_memory( + [{"role": "user", "content": "remember this"}], + type="chat", + info={"user_id": "alice", "session_id": "session-1"}, + mode="fast", + user_name="cube-1", + ) + + assert exc_info.value is error + assert len(failed_events) == 1 + assert failed_events[0]["error"] is error + assert failed_events[0]["hook_context"].source == "mem_reader.extract" + assert failed_events[0]["hook_context"].operation_id is not None + assert after_events == [] diff --git a/tests/mem_scheduler/test_operation_observer.py b/tests/mem_scheduler/test_operation_observer.py new file mode 100644 index 000000000..b8513e5d5 --- /dev/null +++ b/tests/mem_scheduler/test_operation_observer.py @@ -0,0 +1,150 @@ +"""Scheduler operation Hook contracts at existing exception boundaries.""" + +from __future__ import annotations + +from uuid import UUID + +import pytest + +from memos.mem_scheduler.task_schedule_modules.operation_observer import ( + SchedulerOperationObserver, +) +from memos.plugins.hook_context import HookContext +from memos.plugins.hook_defs import H +from memos.plugins.hooks import _hooks, register_hook + + +@pytest.fixture(autouse=True) +def _reset_registered_hooks(): + _hooks.clear() + yield + _hooks.clear() + + +def test_scheduler_observer_emits_context_and_business_values_as_hook_kwargs(): + captured = {} + register_hook( + H.SCHEDULER_MEMORY_OPERATION_AFTER, + lambda **kwargs: captured.update(kwargs), + ) + + with SchedulerOperationObserver( + source="scheduler.mem_read.add_enhanced_memories", + operation="add", + target="textual_memory", + operation_context={"trace_id": "trace-1", "task_id": "task-1"}, + operation_input={"memory_ids": ["memory-1"]}, + ) as observer: + observer.result(["stored-1"]) + + assert isinstance(captured["hook_context"], HookContext) + assert captured["hook_context"].trace_id == "trace-1" + assert captured["operation"] == "add" + assert captured["target"] == "textual_memory" + assert captured["operation_input"] == {"memory_ids": ["memory-1"]} + assert captured["result"] == ["stored-1"] + assert "event" not in captured + + +def test_scheduler_observer_failure_hook_preserves_original_error_and_skips_after(): + error = ValueError("scheduler operation failed") + failed_events = [] + after_events = [] + + def on_failed(**kwargs): + failed_events.append(kwargs) + raise RuntimeError("plugin failed") + + register_hook(H.SCHEDULER_MEMORY_OPERATION_FAILED, on_failed) + register_hook( + H.SCHEDULER_MEMORY_OPERATION_AFTER, + lambda **kwargs: after_events.append(kwargs), + ) + observer = SchedulerOperationObserver( + source="scheduler.mem_read.fine_transfer_simple_mem", + operation="fine", + target="textual_memory", + operation_context={ + "trace_id": "trace-1", + "user_id": "alice", + "session_id": "session-1", + "task_id": "task-1", + "mem_cube_id": "cube-1", + "item_id": "item-1", + "label": "mem_read", + }, + operation_input={"memory_ids": ["memory-1"]}, + capture_failure=True, + ) + + with pytest.raises(ValueError, match="scheduler operation failed") as exc_info, observer: + raise error + + assert exc_info.value is error + assert len(failed_events) == 1 + event = failed_events[0] + context = event["hook_context"] + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "alice" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube-1",) + assert context.source == "scheduler.mem_read.fine_transfer_simple_mem" + assert context.attributes == {"scheduler_item_id": "item-1", "label": "mem_read"} + assert context.operation_id is not None + assert UUID(context.operation_id).hex == context.operation_id + assert event["operation"] == "fine" + assert event["target"] == "textual_memory" + assert event["operation_input"] == {"memory_ids": ["memory-1"]} + assert event["error"] is error + assert after_events == [] + assert observer.value is None + + +def test_scheduler_observer_emits_explicit_failure_only_once(): + failed_events = [] + after_events = [] + error = ValueError("scheduler operation failed") + + register_hook( + H.SCHEDULER_MEMORY_OPERATION_FAILED, + lambda **kwargs: failed_events.append(kwargs), + ) + register_hook( + H.SCHEDULER_MEMORY_OPERATION_AFTER, + lambda **kwargs: after_events.append(kwargs), + ) + + with SchedulerOperationObserver( + source="scheduler.mem_read.remove_memories", + operation="delete", + target="textual_memory", + operation_context={"trace_id": "trace-1"}, + operation_input={"memory_ids": ["memory-1"]}, + ) as observer: + observer.failed(error) + observer.failed(error) + observer.result(None) + + assert len(failed_events) == 1 + assert failed_events[0]["error"] is error + assert after_events == [] + + +def test_scheduler_observer_exposes_piped_result_after_successful_exit(): + register_hook( + H.SCHEDULER_MEMORY_OPERATION_AFTER, + lambda **_kwargs: ["plugin-result"], + ) + + with SchedulerOperationObserver( + source="scheduler.mem_read.add_enhanced_memories", + operation="add", + target="textual_memory", + operation_context={"trace_id": "trace-1"}, + operation_input={}, + ) as observer: + observer.result(["original-result"]) + + assert observer.value == ["plugin-result"] diff --git a/tests/multi_mem_cube/test_single_cube_hooks.py b/tests/multi_mem_cube/test_single_cube_hooks.py new file mode 100644 index 000000000..b384157ba --- /dev/null +++ b/tests/multi_mem_cube/test_single_cube_hooks.py @@ -0,0 +1,142 @@ +"""Textual Memory Hook behavior at the SingleCubeView write boundary.""" + +from __future__ import annotations + +import logging +import uuid + +from importlib import import_module +from unittest.mock import MagicMock + +import pytest + +from memos.plugins.hook_context import HookContext +from memos.plugins.hook_defs import H +from memos.plugins.hooks import _hooks, register_hook + + +@pytest.fixture(autouse=True) +def _reset_registered_hooks(): + _hooks.clear() + yield + _hooks.clear() + + +def _make_add_request(**overrides): + from memos.api.product_models import APIADDRequest + + values = { + "user_id": "test_user", + "messages": [ + {"role": "user", "content": "remember this"}, + {"role": "assistant", "content": "ok"}, + ], + **overrides, + } + return APIADDRequest(**values) + + +@pytest.fixture +def single_cube_view(): + # Initialize handlers before importing SingleCubeView to avoid the existing + # handlers/single_cube import cycle when this file runs in isolation. + import_module("memos.api.handlers") + from memos.memories.textual.item import ( + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, + ) + from memos.multi_mem_cube.single_cube import SingleCubeView + + memory = TextualMemoryItem( + id=str(uuid.uuid4()), + memory="hello world", + metadata=TreeNodeTextualMemoryMetadata( + user_id="u1", + session_id="s1", + memory_type="WorkingMemory", + sources=[], + info={}, + ), + ) + mem_reader = MagicMock() + mem_reader.get_memory.return_value = [[memory]] + mem_reader.save_rawfile = False + text_memory = MagicMock() + text_memory.add.return_value = [memory.id] + text_memory.mode = "async" + mem_cube = MagicMock() + mem_cube.text_mem = text_memory + + view = SingleCubeView( + cube_id="cube_test", + naive_mem_cube=mem_cube, + mem_reader=mem_reader, + mem_scheduler=MagicMock(), + logger=logging.getLogger("test.single_cube.hooks"), + searcher=None, + feedback_server=None, + ) + return view, memory + + +def test_text_memory_after_hook_can_replace_memory_ids(single_cube_view): + view, _memory = single_cube_view + register_hook(H.TEXT_MEMORY_ADD_AFTER, lambda **_kwargs: ["plugin-memory-id"]) + + results = view.add_memories(_make_add_request(async_mode="async")) + + assert results[0]["memory_id"] == "plugin-memory-id" + submitted_message = view.mem_scheduler.submit_messages.call_args.kwargs["messages"][0] + assert submitted_message.mem_cube_id == "cube_test" + assert submitted_message.content == '["plugin-memory-id"]' + + +def test_text_memory_failure_hook_preserves_original_error_and_skips_after( + single_cube_view, + monkeypatch, +): + view, memory = single_cube_view + error = ValueError("text memory add failed") + failed_calls = [] + after_calls = [] + view.naive_mem_cube.text_mem.add.side_effect = error + monkeypatch.setattr( + "memos.multi_mem_cube.single_cube.get_current_trace_id", + lambda: "trace-1", + ) + + def on_failed(**kwargs): + failed_calls.append(kwargs) + raise RuntimeError("plugin failed") + + register_hook(H.TEXT_MEMORY_ADD_FAILED, on_failed) + register_hook(H.TEXT_MEMORY_ADD_AFTER, lambda **kwargs: after_calls.append(kwargs)) + + with pytest.raises(ValueError, match="text memory add failed") as exc_info: + view.add_memories( + _make_add_request( + async_mode="async", + session_id="session-1", + task_id="task-1", + ) + ) + + assert exc_info.value is error + assert after_calls == [] + assert len(failed_calls) == 1 + call = failed_calls[0] + context = call["hook_context"] + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "test_user" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube_test",) + assert context.source == "cube_view.text_mem.add" + assert context.attributes == {"sync_mode": "async", "extract_mode": "fast"} + assert context.operation_id is not None + assert call["text_memory"] is view.naive_mem_cube.text_mem + assert call["memories"] == [memory] + assert call["kwargs"] == {"user_name": "cube_test"} + assert call["error"] is error + view.mem_scheduler.submit_messages.assert_not_called() diff --git a/tests/plugins/test_hook_context.py b/tests/plugins/test_hook_context.py new file mode 100644 index 000000000..c1fdad0bd --- /dev/null +++ b/tests/plugins/test_hook_context.py @@ -0,0 +1,312 @@ +"""HookContext values, callback compatibility, and @hookable propagation.""" + +from __future__ import annotations + +import asyncio + +from collections.abc import Mapping +from typing import Any, get_type_hints +from uuid import UUID + +import pytest + +from memos.plugins.hook_context import HookContext, build_hook_context +from memos.plugins.hook_defs import H, define_hook, get_hook_spec +from memos.plugins.hooks import ( + _hooks, + hookable, + register_hook, + trigger_hook, + trigger_single_hook, +) + + +@pytest.fixture(autouse=True) +def _reset_registered_hooks(): + _hooks.clear() + yield + _hooks.clear() + + +def test_hook_context_type_hints_are_runtime_resolvable(): + hints = get_type_hints(HookContext) + + assert hints["attributes"] == Mapping[str, Any] + assert hints["operation_id"] == str | None + + +def test_build_hook_context_populates_correlation_metadata(): + cube_ids = ["cube-1", "cube-2"] + attributes = {"mode": "fine"} + + context = build_hook_context( + source="mem_reader.extract", + trace_id="trace-1", + user_id="user-1", + session_id="session-1", + task_id="task-1", + cube_ids=cube_ids, + operation_id="operation-1", + attributes=attributes, + ) + cube_ids.append("cube-3") + attributes["mode"] = "mutated" + + assert context == HookContext( + trace_id="trace-1", + user_id="user-1", + session_id="session-1", + task_id="task-1", + cube_ids=("cube-1", "cube-2"), + operation_id="operation-1", + source="mem_reader.extract", + attributes={"mode": "fine"}, + ) + + +def test_build_hook_context_generates_unique_operation_ids(): + first = build_hook_context(source="text_memory.add") + second = build_hook_context(source="text_memory.add") + + assert first.operation_id is not None + assert second.operation_id is not None + assert UUID(first.operation_id).hex == first.operation_id + assert UUID(second.operation_id).hex == second.operation_id + assert first.operation_id != second.operation_id + + +def test_context_aware_hook_specs_match_trigger_contracts(): + expected = { + H.TEXT_MEMORY_ADD_AFTER: ( + ["hook_context", "text_memory", "memories", "kwargs", "result"], + "result", + ), + H.TEXT_MEMORY_ADD_FAILED: ( + ["hook_context", "text_memory", "memories", "kwargs", "error"], + None, + ), + H.SCHEDULER_MEMORY_OPERATION_AFTER: ( + ["hook_context", "operation", "target", "operation_input", "result"], + "result", + ), + H.SCHEDULER_MEMORY_OPERATION_FAILED: ( + ["hook_context", "operation", "target", "operation_input", "error"], + None, + ), + H.MEM_READER_EXTRACT_AFTER: ( + [ + "hook_context", + "mem_reader", + "scene_data", + "type", + "info", + "mode", + "user_name", + "kwargs", + "result", + ], + "result", + ), + H.MEM_READER_EXTRACT_FAILED: ( + [ + "hook_context", + "mem_reader", + "scene_data", + "type", + "info", + "mode", + "user_name", + "kwargs", + "error", + ], + None, + ), + } + + for hook_name, (params, pipe_key) in expected.items(): + spec = get_hook_spec(hook_name) + assert spec is not None + assert spec.params == params + assert spec.pipe_key == pipe_key + + +def test_hook_context_is_omitted_only_for_legacy_exact_signature_callbacks(): + hook_name = "test.hook-context.compat" + define_hook( + hook_name, + description="hook context compatibility test", + params=["hook_context", "result"], + pipe_key="result", + ) + context = object() + calls = [] + + def legacy_callback(*, result): + calls.append(("legacy", result)) + return result + 1 + + def context_callback(*, hook_context, result): + calls.append(("context", hook_context, result)) + return result + 1 + + def kwargs_callback(*, result, **kwargs): + calls.append(("kwargs", kwargs["hook_context"], result)) + return result + 1 + + register_hook(hook_name, legacy_callback) + register_hook(hook_name, context_callback) + register_hook(hook_name, kwargs_callback) + + result = trigger_hook(hook_name, hook_context=context, result=1) + + assert result == 4 + assert calls == [ + ("legacy", 1), + ("context", context, 2), + ("kwargs", context, 3), + ] + + +def test_hook_callback_type_error_is_not_retried(): + hook_name = "test.hook-context.type-error" + define_hook( + hook_name, + description="hook callback type error test", + params=["hook_context", "value"], + ) + calls = [] + + def callback(*, hook_context, value): + calls.append((hook_context, value)) + raise TypeError("raised inside callback") + + register_hook(hook_name, callback) + + trigger_hook(hook_name, hook_context="context", value="value") + + assert calls == [("context", "value")] + + +def test_trigger_single_hook_supports_legacy_exact_signature(): + hook_name = "test.single.hook-context" + define_hook( + hook_name, + description="single hook context compatibility test", + params=["hook_context", "value"], + ) + + def handler(*, value): + return value + 1 + + register_hook(hook_name, handler) + + result = trigger_single_hook(hook_name, hook_context=object(), value=1) + + assert result == 2 + + +def test_hookable_shares_context_with_sync_operation_and_legacy_callback(): + context = object() + calls = [] + + class Handler: + @hookable( + "context_sync", + context_builder=lambda _self, _request: context, + ) + def run(self, request, *, hook_context=None): + calls.append(("operation", hook_context, request)) + return f"processed:{request}" + + def legacy_before(*, request): + calls.append(("legacy_before", request)) + + def context_before(*, hook_context, request): + calls.append(("context_before", hook_context, request)) + + def context_after(*, hook_context, request, result): + calls.append(("context_after", hook_context, request, result)) + + register_hook("context_sync.before", legacy_before) + register_hook("context_sync.before", context_before) + register_hook("context_sync.after", context_after) + + result = Handler().run("request") + + assert result == "processed:request" + assert calls == [ + ("legacy_before", "request"), + ("context_before", context, "request"), + ("operation", context, "request"), + ("context_after", context, "request", "processed:request"), + ] + before_spec = get_hook_spec("context_sync.before") + after_spec = get_hook_spec("context_sync.after") + assert before_spec is not None + assert after_spec is not None + assert before_spec.params == ["hook_context", "request"] + assert after_spec.params == [ + "hook_context", + "request", + "result", + ] + + +def test_hookable_reuses_explicit_context_without_calling_builder(): + supplied_context = object() + builder_calls = [] + observed_contexts = [] + + def build_context(_self, request): + builder_calls.append(request) + return object() + + class Handler: + @hookable("explicit_context", context_builder=build_context) + def run(self, request, *, hook_context=None): + observed_contexts.append(hook_context) + return request + + def observe_context(*, hook_context, **_kwargs): + observed_contexts.append(hook_context) + + register_hook("explicit_context.before", observe_context) + register_hook("explicit_context.after", observe_context) + + result = Handler().run("request", hook_context=supplied_context) + + assert result == "request" + assert builder_calls == [] + assert observed_contexts == [supplied_context] * 3 + + +def test_hookable_shares_context_with_async_operation(): + context = object() + calls = [] + + class Handler: + @hookable( + "context_async", + context_builder=lambda _self, _request: context, + ) + async def run(self, request, *, hook_context=None): + calls.append(("operation", hook_context, request)) + return "async_result" + + def on_before(*, hook_context, request): + calls.append(("before", hook_context, request)) + + def on_after(*, hook_context, request, result): + calls.append(("after", hook_context, request, result)) + + register_hook("context_async.before", on_before) + register_hook("context_async.after", on_after) + + result = asyncio.run(Handler().run("request")) + + assert result == "async_result" + assert calls == [ + ("before", context, "request"), + ("operation", context, "request"), + ("after", context, "request", "async_result"), + ] From e80b7ced6d20f7d5042d76c481ee2b9191c220ec Mon Sep 17 00:00:00 2001 From: qzs Date: Sat, 26 Sep 2026 00:56:46 +0800 Subject: [PATCH 02/22] feat: integrate SmartComment tracing --- apps/memtrace-memos-integration/.gitignore | 35 + apps/memtrace-memos-integration/README.md | 265 ++++++ apps/memtrace-memos-integration/__init__.py | 8 + .../memos_smartcomment/CONVENTIONS.md | 382 +++++++++ .../memos_smartcomment/__init__.py | 19 + .../memos_smartcomment/adapter.py | 250 ++++++ .../memos_smartcomment/config.py | 64 ++ .../memos_smartcomment/events.py | 65 ++ .../memos_smartcomment/handlers/__init__.py | 25 + .../handlers/add_handler.py | 790 ++++++++++++++++++ .../memos_smartcomment/handlers/base.py | 197 +++++ .../handlers/search_handler.py | 537 ++++++++++++ .../memos_smartcomment/plugin.py | 96 +++ .../memos_smartcomment/recorder.py | 261 ++++++ .../memos_smartcomment/serialization.py | 153 ++++ .../memtrace-memos-integration/pyproject.toml | 32 + src/memos/mem_reader/simple_struct.py | 12 +- .../handlers/mem_read_handler.py | 21 +- tests/mem_reader/test_fine_transfer_order.py | 44 + tests/mem_scheduler/test_mem_read_grouping.py | 139 +++ 20 files changed, 3388 insertions(+), 7 deletions(-) create mode 100644 apps/memtrace-memos-integration/.gitignore create mode 100644 apps/memtrace-memos-integration/README.md create mode 100644 apps/memtrace-memos-integration/__init__.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/__init__.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/adapter.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/config.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/events.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/plugin.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/recorder.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/serialization.py create mode 100644 apps/memtrace-memos-integration/pyproject.toml create mode 100644 tests/mem_reader/test_fine_transfer_order.py create mode 100644 tests/mem_scheduler/test_mem_read_grouping.py diff --git a/apps/memtrace-memos-integration/.gitignore b/apps/memtrace-memos-integration/.gitignore new file mode 100644 index 000000000..ad1838dc3 --- /dev/null +++ b/apps/memtrace-memos-integration/.gitignore @@ -0,0 +1,35 @@ +# Local runtime state and credentials +/.memos/ +/.env +/.env.* +/logs/ +/tmp/ +*.log + +# Python environments and generated packages +/.venv/ +/venv/ +/build/ +/dist/ +*.egg-info/ +__pycache__/ +*.py[cod] + +# Tests and tool output +tests/ +test_*.py +*_test.py +conftest.py +.pytest_cache/ +.ruff_cache/ +.tests/ +.coverage +.coverage.* +htmlcov/ +.tox/ +.nox/ + +# Editor and operating system files +/.idea/ +/.vscode/ +.DS_Store diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md new file mode 100644 index 000000000..e7dec9bde --- /dev/null +++ b/apps/memtrace-memos-integration/README.md @@ -0,0 +1,265 @@ +# MemOS smartcomment integration + +Trace how memories move through MemOS add, Search API, and asynchronous MemRead operations. This external plugin listens to existing MemOS Hooks and writes a smartcomment execution graph as JSON for each trace. Installing the plugin does not enable it automatically. + +**Dependency:** use the [public `smartcomment` package on PyPI](https://pypi.org/project/smartcomment/). The plugin requires `smartcomment>=0.1.2`; you do not need a smartcomment source checkout. The integration in this directory is installed separately. + +## Quick start + +### 1. Install into the MemOS environment + +Use Python 3.12 or newer. Activate the **same environment that runs MemOS**, then run these commands from the MemOS repository root: + +```bash +# Skip this line if MemOS is already installed in the active environment. +python -m pip install -e . +python -m pip install "smartcomment>=0.1.2" +python -m pip install -e apps/memtrace-memos-integration +``` + +The integration package also declares both MemoryOS and smartcomment as dependencies, but installing the public smartcomment package explicitly makes its source clear. If your MemOS environment was created with `uv` and has no `pip` module, use its interpreter explicitly instead: + +```bash +uv pip install --python .venv/bin/python -e . +uv pip install --python .venv/bin/python "smartcomment>=0.1.2" +uv pip install --python .venv/bin/python -e apps/memtrace-memos-integration +``` + +Skip the first line if MemOS is already installed there. + +Check that both packages and the MemOS plugin entry point are visible to this interpreter: + +```bash +python -c 'from importlib.metadata import version, entry_points; print("smartcomment", version("smartcomment")); print("integration", version("memos-smartcomment-integration")); print("plugin", [ep.name for ep in entry_points(group="memos.plugins") if ep.name == "smartcomment"])' +``` + +The last line should show `plugin ['smartcomment']`. If it does not, install the integration again with the interpreter used to start MemOS. + +### 2. Enable the plugin and start MemOS + +Configure the MemOS service as described in the [self-hosting guide](../../README.md). Set the plugin variables **before** starting the service: + +```bash +export MEMOS_ENABLED_PLUGINS=smartcomment +export MEMOS_SMARTCOMMENT_OUTPUT_DIR=.memos/memtrace +python -m uvicorn memos.api.server_api:app --host 127.0.0.1 --port 8000 +``` + +`MEMOS_ENABLED_PLUGINS` accepts comma-separated names if you use other opt-in plugins. The output path is relative to the service's working directory; use an absolute path when that directory may change. Startup should log `smartcomment plugin loaded`. + +### 3. Run a memory operation and inspect the graph + +Call a working MemOS `/product/add` or `/product/search` endpoint (available in the service's `/docs` page). Add records the message → extraction → persistence flow; Search records retrieval, filtering, deduplication, and reranking. MemRead scheduler events may arrive later and extend the same trace. + +After the operation, list the saved graphs: + +```bash +find .memos/memtrace -type f -name '*.json' -print +``` + +Files are stored under a user-specific directory, one JSON graph per MemOS `trace_id`. Open a file to inspect `data.nodes`, `data.edges`, `data.operations`, and `data.sessions`. User and trace path components contain a readable prefix plus a hash, so the filename is not necessarily the raw trace ID. The graph's `graph_id` contains the trace ID. + +## Configuration + +Set these environment variables before loading the plugin. Restart MemOS after changing them. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `MEMOS_ENABLED_PLUGINS` | unset | Include `smartcomment` to enable this opt-in plugin. | +| `MEMOS_SMARTCOMMENT_OUTPUT_DIR` | `.memos/memtrace` | Directory for JSON graphs. | +| `MEMOS_SMARTCOMMENT_QUEUE_SIZE` | `1024` | Maximum queued trace events; a full queue drops events with a warning. | +| `MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS` | `20000` | Maximum length of each captured string. | +| `MEMOS_SMARTCOMMENT_STRICT` | `false` | Enable smartcomment identity checks. | +| `MEMOS_SMARTCOMMENT_PROJECT_ID` | `memos` | Project identifier stored in graphs. | +| `MEMOS_SMARTCOMMENT_MAX_CACHED_TRACES` | `64` | Maximum graphs retained in memory. | +| `MEMOS_SMARTCOMMENT_MAX_CACHED_GRAPH_ITEMS` | `50000` | Maximum cached nodes, edges, operations, and sessions. | +| `MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS` | `300` | Idle cache expiration in seconds. | + +Snapshots are written atomically. An idle or full in-memory cache can be evicted; a later event restores the saved graph and appends to it. Cache limits do not delete JSON files. Use a different output directory for each writer process, including when running multiple server workers. + +Credential-named fields are redacted, embedding/vector fields are omitted, and long values are bounded before events enter the queue. JSON strings in `argument` and `arguments` are decoded and recursively redacted; unparseable argument strings are omitted. **Free-form message and memory text can still appear in graphs**, up to the length limit. Store the output directory with access appropriate for that data. + +## If no graph appears + +1. Confirm the service uses the interpreter that passed the package and entry-point check above, and that `MEMOS_ENABLED_PLUGINS` included `smartcomment` before startup. +2. Check for `smartcomment plugin loaded` and any writer warnings in the service logs. A full queue drops events; failed writes are logged. +3. Confirm that an add or Search API operation ran successfully, then check the configured output directory relative to the service's working directory. Scheduler updates can arrive asynchronously. + +## Development and graph semantics + +The sections below describe the Hook mapping and explicit data-flow contract for maintainers. MemOS business modules do not import smartcomment. See also [`memos_smartcomment/CONVENTIONS.md`](memos_smartcomment/CONVENTIONS.md). + +### Handler layout + +Hook callbacks live in `memos_smartcomment/handlers/`: + +- `add_handler.py`: `AddHandler` observes add, extraction, persistence, and subsequent + MemRead scheduler operations; it owns pending message-batch correlation. +- `search_handler.py`: `SearchHandler` observes the Search API result pipeline and + owns its operation-scoped stage cache. +- `base.py`: shared context parsing, snapshotting, event construction, and submission. +- `__init__.py`: `HookHandlers` combines both flows and clears both caches at shutdown. + The existing `from memos_smartcomment.handlers import HookHandlers` import is unchanged. + +Each handler lists Hook callbacks before its internal helpers. `AddHandler` and +`SearchHandler` can also be instantiated independently with an event submission callable. + +### Captured memory flow + +```text +message batch + -- mem_reader.extract --> extracted memory + -- text_memory.persist --> persisted memory + -- scheduler.mem_read.fine_transfer --> enhanced memory + -- scheduler.mem_read.add_enhanced --> persisted memory +``` + +The synchronous add flow emits three semantic memory categories: + +- `message_batch`: messages received at `add.before` and consumed by + `mem_reader.extract.after`; +- `extracted_memory`: a MemReader output passed into the TextMemory writer; +- `persisted_memory`: the memory after `text_memory.add.after` confirms the write. + +The asynchronous MemRead scheduler also emits: + +- `enhanced_memory`: a fine-transfer result reused as the input to the enhanced-memory + database write; +- `scheduler_result`: a string result for operations such as archive, delete, refresh, + or failure that do not otherwise expose a graphable output value. + +Scheduler fine-transfer inputs reuse the `memory::` identity created +by the earlier TextMemory persistence event. This connects the original persisted memory +to its enhanced successor without creating a duplicate memory node. Archive, delete, +refresh, and failed MemRead operations are also recorded; failures include only the error +type, not the error message or an error-value node. + +A persisted memory unit is one immutable graph anchor per cube and memory ID. Its first +complete snapshot is retained; different later observations log a warning without replacing +it or creating a new version, including in strict mode. If an asynchronous reference arrives +before the write event, its ID-only placeholder is completed once without changing the node +ID or existing edges. Restoring older snapshots also joins split versions of persisted units +to their earliest anchor and retains the first complete snapshot and all incident edges. + +Fine-transfer Hooks declare `operation_input.result_grouping`: `per_input` means +one group per input in input order, including empty groups for failures; `batch` +means the whole input batch contributes to its outputs. The MemRead scheduler Hook +identifies SimpleStructMemReader's transfer implementation (including inherited uses +such as StrategyStructMemReader) as `per_input`, and MultiModalStructMemReader's as +`batch`. A Reader's explicit `fine_transfer_result_grouping` declaration takes +precedence; custom transfer implementations otherwise remain `unknown`. If grouping is +unknown or inconsistent for multiple inputs, the plugin preserves the nodes and +marks lineage as unresolved instead of guessing source edges. Failed enhanced +writes retain their enhanced-memory inputs and links to the failure status. + +The integration follows the current MemOS operation-boundary Hook contract: callbacks +receive correlation-only `HookContext` data plus explicit business keyword arguments. +Successful MemReader, TextMemory, and scheduler callbacks observe the piped `result`; +their matching `failed` callbacks record a status node without changing the propagated +exception. MemReader and TextMemory failures also omit exception messages. + +Database write exceptions propagate through `MemoryManager.add`, including batch and +parallel single-node writes. A failed add emits the existing failure Hook instead of a +successful persistence event, and a failed enhanced write skips source-memory deletion +for that MemRead operation. Some concurrent writes may already have committed; failure +describes the operation outcome and does not imply rollback of those writes. + +The Search API emits this ordered result pipeline: + +```text +search_query + -- memos.search.retrieve --> raw memory nodes + -- memos.search.threshold_filter --> threshold memory nodes; removed nodes -> "filtered" + -- memos.search.deduplicate --> deduplicated memory nodes; removed nodes -> "filtered" + -- memos.search.rerank --> reranked memory nodes +``` + +Every Search stage materializes its current candidates as separate, stage-scoped +`search_memory` nodes. The node value contains the memory ID, text, score, result type, and +cube ID. Its comment records the one-based `rank`, stage, rank scope, and global position. +Rank scope is one result-type/cube bucket because the Search API returns bucketed results. +The current API rerank stage is recorded as a descending `metadata.relativity` sort +(`relativity_sort`). Search nodes describe the Search pipeline and do not reference +persisted memory units. + +Search stages use explicit links rather than the input/output Cartesian product. Threshold +and deduplication link each surviving source to its new stage node, while removed memories +link to one stage-scoped `"filtered"` string node. Rerank likewise links each +source to the new node with the same memory ID, including knowledge memories whose text is +expanded with original document content. Matching uses result type, cube and memory ID; +candidates without an ID use a content fingerprint computed before truncation. New +candidates are retained even when no source edge is known. This keeps every Hook visible as +one step in the Search flow, including stages whose values did not change. + +Search observation ends at `search.results.after_rerank`, which releases the operation's +cached stage nodes. All Search hooks use the shared `HookContext.operation_id` as their graph and cache +scope, so multiple searches in one trace cannot reuse or clear each other's stage nodes. +Post-processing failures clear only the failed operation's cached stage nodes. + +The exported graph omits smartcomment's internal `COMMENT:NONE@1` sentinel node, its +incident edges, and its sentinel session. Root inputs and outputs remain normal semantic +nodes; the internal placeholder does not participate in the MemOS graph. + +The node `comment` explains its meaning and MemOS stage. Request wrappers, Hook boundary +objects, `info`, backend objects, errors, and scheduler variables are intentionally not +represented as graph nodes. Search queries and stage-scoped memory results are represented, +but raw backend objects are not. Scheduler operations +are not connected merely because one ran after another; the graph contains only explicit +data-flow edges. When an operation has no data output, its `scheduler_result` string is used +as the target of edges from the memory values it consumed. + +Each add receives a fresh UUID for its message batch, including repeated adds that reuse +the same Python list. A bounded registry correlates the live request with MemReader +callbacks for each target cube. The registry uses weak references for API requests and +releases entries after extraction (including empty and failed results), `add.after`, or +shutdown. At most 1024 pending adds are retained; after eviction, an extraction receives +an independent input node instead of borrowing another request's identity. + +Without a request trace context, stages use `standalone::`, with +`default_session` matching MemOS's default. The task ID is a fallback only when the user +is unavailable, because MemReader does not receive the add request's task ID. +An explicit Hook context always takes precedence over a worker's ambient request context, +including when the Hook has no trace ID. This prevents one message from borrowing another +message's trace during scheduler batch processing. Hook callbacks capture detached snapshots +synchronously and enqueue them; the graph writer alone runs asynchronously. + +Extracted and persisted memories use different identities so the database write remains +visible as a transition. Both synchronous and scheduler writes match returned IDs to +unique input memory IDs, allowing skipped inputs and reordered results. Unknown or +ambiguous IDs retain an ID-only persisted node without inferred content or source edges; +operation metadata marks lineage as `resolved`, `partial`, or `unresolved`. + +### Event and link contract + +All flows use the same edge path: Hooks build `TraceLink(source, target)`, and the +recorder writes each link with smartcomment's `comment_link` inside one +`comment_op_scope` per event. Link category and comment default to the operation's; +link metadata extends and can override the operation metadata on that edge. + +`TraceEvent.links` defaults to `()`: omitted or empty links create no edges. +`inputs` and `outputs` register nodes without inferring dependencies. Link endpoints +are registered even when they are not separately listed as inputs or outputs. +Independent nodes and operations with no links are retained. + +```python +event = TraceEvent( + operation="memos.text_memory.persist", + category="memory_persistence", + trace_id=trace_id, + inputs=(extracted,), + outputs=(persisted,), + links=(TraceLink(source=extracted, target=persisted),), +) +``` + +Callers that previously relied on omitted `links` or `links=None` for automatic +input/output edges must now provide explicit links. Match endpoints using each +operation's data contract; unknown lineage keeps its nodes without a guessed edge. + +## Run tests + +From the MemOS repository root, in the same Python environment: + +```bash +python -m pytest apps/memtrace-memos-integration/tests -q +``` diff --git a/apps/memtrace-memos-integration/__init__.py b/apps/memtrace-memos-integration/__init__.py new file mode 100644 index 000000000..88be0f013 --- /dev/null +++ b/apps/memtrace-memos-integration/__init__.py @@ -0,0 +1,8 @@ +# ruff: noqa: N999 +# 兼容旧版应用骨架的导入入口; 实际插件实现位于 memos_smartcomment/plugin.py. +"""Compatibility import for the original app skeleton.""" + +from memos_smartcomment.plugin import SmartCommentPlugin + + +__all__ = ["SmartCommentPlugin"] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md new file mode 100644 index 000000000..5dcb0d8e8 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -0,0 +1,382 @@ +# memos_smartcomment 追踪数据约定 + +本文按当前实现整理,供阅读执行图、编写可视化和扩展 handler 时查阅。字段名和取值以代码为准;下列 `category`、`class_name` 都是字符串约定,未定义为强制枚举。 + +代码入口:[事件结构](events.py)、[公共处理](handlers/base.py)、[添加链路](handlers/add_handler.py)、[搜索链路](handlers/search_handler.py)、[图写入](recorder.py)。 + +## 1. 公共模型与身份 + +调用链为 `MemOS Hook → HookHandlers → TraceEvent → AsyncTraceAdapter → SmartCommentRecorder → JSON`。`HookHandlers` 组合 `AddHandler` 和 `SearchHandler`,两者也可以独立使用。 + +| 对象 | 表示什么 | 关键字段 | +| --- | --- | --- | +| `TraceValue` | 一份业务值快照,最终注册为图节点 | `name`、`value`、`identity`、`class_name`、`category`、`identity_only`、`comment`、`metadata`、`match_key` | +| `TraceEvent` | 一次 Hook 观测,最终建立一个 session 和一个 operation | `operation`、`category`、`trace_id`、用户/会话/cube 关联信息、`inputs`、`outputs`、`links`、`metadata`、`event_id`、`created_at` | +| `TraceLink` | 来源值到目标值的显式数据依赖 | `source`、`target`、`category`、`comment`、`metadata` | + +`inputs` 和 `outputs` 只注册节点,**不会自动生成输入与输出之间的边**。只有 `links` 生成边;未单独列入输入/输出的 link 端点也会注册。操作发生的时间先后不构成连线依据。 + +### 1.1 `identity`、`name`、`full_node_id` 的区别 + +当前 `TraceValue` 的字段叫 **`identity`,没有 `identity_id` 字段**。本地 SmartComment 的节点导出也使用 `name`、`full_name`、`node_id`、`full_node_id`。如果讨论中说「identity_id」,应先明确是业务身份还是带版本的图节点 ID。 + +Recorder 用 `id_strategy` 返回 `TraceValue.identity`,因此映射如下: + +| 字段 | 当前含义/构成 | +| --- | --- | +| `TraceValue.name` | handler 内的业务名称,如 `messages`、`query`、`raw_text_mem_1`;当前 recorder 不将它传给 SmartComment | +| `TraceValue.identity` | handler 构造的业务身份字符串 | +| 导出节点 `name` | 等于 `TraceValue.identity`,不是 `TraceValue.name` | +| 导出节点 `full_name` | `{class_name}:{identity}`;没有 `class_name` 时为 `{identity}` | +| 导出节点 `node_id` | `{identity}@{version}` | +| 导出节点 `full_node_id` | `{class_name}:{identity}@{version}`;边的 source/target 使用此字段 | + +例如 `class_name="memory"`、`identity="memory:cube-1:mem-1"` 时,首版 `full_node_id` 是 `memory:memory:cube-1:mem-1@1`。两个 `memory` 分别来自类型命名空间和业务身份前缀。 + +这些身份目前通过冒号直接拼接,没有转义或解析协议。`trace_id` 本身也可能含冒号,读取图时不宜靠固定位置的 `split(":")` 还原字段。 + +### 1.2 图、操作、事件与 session 的作用域 + +| 标识 | 规则 | +| --- | --- | +| 图的隔离键 | Adapter 按 `(user_id, trace_id)` 管理图;`graph_id = trace_id`,`project_id` 来自配置,默认 `memos` | +| `operation_id` | 来自 Hook 业务上下文,用于关联一次业务操作;存在时由 `_event()` 补入 `TraceEvent.metadata` | +| `event_id` | 每次创建事件都生成一个 `uuid4().hex`,标识本次观测;不是业务 `operation_id` | +| `created_at` | 事件创建时的 UTC ISO 时间 | +| SmartComment `session_id` | 每个事件使用 `event-{event_id}`,session 名称是 `event.operation` | +| MemOS `session_id` | 业务会话 ID,导出时放在 session/operation metadata 的 `memos_session_id` 中 | +| SmartComment `op_id` | 由 SmartComment 创建 operation 时分配;业务 `operation_id` 仍在 metadata 中 | + +即使两个 trace 出现相同 `identity`,也不会跨图合并。持久化记忆 identity 不含 trace,并不意味着全局只有一个节点。 + +`trace_id` 解析顺序: + +1. 使用 subject 上非空且不等于占位值 `"trace-id"` 的 `trace_id`。 +2. **仅当 subject 完全没有 `trace_id` 字段**时,尝试环境请求上下文。显式 `None`、空值或 `"trace-id"` 不会借用环境 trace。 +3. 无有效 trace 且存在 `user_id`:`standalone:{user_id}:{session_id or 'default_session'}`。 +4. 无 user 但有 task:`task:{task_id}`;两者都没有:`standalone:unknown:{session_id or 'default_session'}`。 + +`_hook_subject()` 保留显式空 trace 的语义。Search 另用请求补充 Hook 上为空的 `trace_id/user_id/session_id/task_id`,所以 Hook 的空 trace 可以由请求的有效 trace 补齐。 + +cube 列表取第一个非空字段:`writable_cube_ids → readable_cube_ids → mem_cube_ids → cube_ids`,转字符串并按原顺序去重;均无值则取 `mem_cube_id or cube_id`。单条记忆的默认 cube 只在恰好一个 cube 时确定;Add 的特定入口还可使用 `user_name` 作为存储 cube 的兜底,不能把它直接当成 `user_id`。 + +### 1.3 `identity_only` 与版本 + +| 情况 | 行为 | +| --- | --- | +| `identity_only=True` | 按身份引用已有节点;尚不存在时仍可创建节点,并非「不记录 value」 | +| 普通节点,身份和编码后的 value 相同 | SmartComment 复用已有版本 | +| 普通节点,身份相同但 value 不同,`identity_only=False` | 非 strict 模式创建新版本;strict 模式报一致性错误 | +| `category="persisted_memory"` | recorder 强制按身份复用,使用下面的不可变锚点规则 | + +持久化记忆由 `(cube_id, memory_id)` 标识,保留第一次完整正文快照。仅有 ID 时先创建占位;完整快照到达后补全一次,保留原节点 ID 和连线。此后内容冲突只告警,不覆盖或升版,strict 模式也相同。恢复旧图时,同一持久化身份的多个版本合并到最早锚点,并保留首个完整快照及关联边。 + +普通节点复用时,不会因为新一次观测的 metadata/comment 不同就自动更新它们;逐次操作信息应查看对应 operation/edge。持久化占位补全是 recorder 的专门处理。 + +## 2. `TraceValue` 节点总表 + +下面的 `{cube}` 在 Add 链中表示 `cube_id or 'unknown'`;`{trace}` 为解析后的 trace。`{search_scope}` 优先使用非 `None` 的 `operation_id`,否则使用 trace。`{status}` 在调度链中为 `after` 或 `failed`。 + +### 2.1 AddHandler:输入、提取、持久化与后台增强 + +| `TraceValue.name` | `category` | `class_name` | `identity` | `value` | +| --- | --- | --- | --- | --- | +| `messages` | `message_batch` | `message_batch` | `message_batch:{trace}:{uuid4.hex}` | messages 的快照 | +| `extracted_memory` | `extracted_memory` | `memory` | `extracted_memory:{trace}:{cube}:{memory_id}` | 记忆正文 | +| `persisted_memory` | `persisted_memory` | `memory` | `memory:{cube}:{memory_id}` | 完整时为正文;引用/来源未知时为 `{"memory_id": ..., "cube_id": ...}` | +| `enhanced_memory` | `enhanced_memory` | `memory` | `enhanced_memory:{trace}:{cube}:{memory_id}` | 生成时为正文;后续持久化引用时为 ID/cube 字典 | +| `scheduler_result` | `scheduler_result` | `str` | `scheduler_result:{operation_id or 'unknown'}:{status}` | `"{operation_name} completed"` 或 `"{operation_name} failed"` | + +正文统一优先取 `memory` 字段,仅当其为 `None` 时取 `text`;空字符串是有效正文。Add 遍历记录时要求同时有 ID 和正文,ID 优先取 `memory_id`,再取 `id`;不会递归遍历 `metadata/info/internal_info` 包装字段。 + +| 节点 | `TraceValue.metadata` 实际记录什么 | +| --- | --- | +| `message_batch`,在 `add.before` 创建 | `writable_cube_ids`、`async_mode`、`mode`、`custom_tags`、`info`、`chat_history`,均取自 request;字段缺失时值可为 `None` | +| `message_batch`,MemReader 引用 | 当前传入 `{}`;已有节点通常保留最初 metadata | +| `extracted_memory`,提取成功时创建 | `type`、`info`、`mode`、`user_name`,取自 MemReader Hook 参数 | +| `extracted_memory`,写入阶段引用 | 当前传入 `{}`,仍携带正文作为 value | +| `persisted_memory`,写入 Hook 输出 | `memos_stage`、`cube_id`、`memory_id`;stage 为 `text_memory.add.after` 或 `scheduler.mem_read.add_enhanced_memories.after` | +| `persisted_memory`,后台按 ID 引用 | `cube_id`、`memory_id`;此构造函数不添加 `memos_stage` | +| `enhanced_memory` | `cube_id`、`memory_id` | +| `scheduler_result` | `operation_id`、`operation_name`、`status` | + +Recorder 对导出的 `persisted_memory.metadata` 额外添加 `snapshot_status="reference"` 或 `"complete"`。这个字段不在 handler 原始 `TraceValue.metadata` 中。 + +**这里的 metadata 不是源记忆 `memory.metadata` 的完整副本。** 例如 `memory_type`、`sources`、时间戳不会自动进入提取/持久化节点;`extracted_memory` 甚至不单独添加 `cube_id/memory_id` metadata,它们目前编码在 identity 中。 + +消息身份的关联规则:每次 `add.before` 分配新 UUID,即便重复使用同一个 messages 列表。短期缓存键为 `(trace_id, user_id, id(messages))`,验证请求仍持有同一个 messages 对象,再让各 cube 的 MemReader 引用同一批次。`id(messages)` 仅用于内存关联,不是图身份;未命中时分配独立 UUID。缓存默认最多 1024 个 pending add,各 cube 消费完、`add.after` 或 shutdown 时清理。 + +### 2.2 SearchHandler:查询与各阶段候选 + +| `TraceValue.name` | `category` | `class_name` | `identity` | `value` | +| --- | --- | --- | --- | --- | +| `query` | `search_query` | `str` | `search_query:{search_scope}` | 请求 query;无字段时默认空字符串 | +| `{stage}_{result_type}_{rank}` | `search_memory` | `search_memory` | `search_memory:{search_scope}:{stage}:{result_type}:{cube_scope}:{occurrence_id}:{global_position}` | `memory_id`、`memory`、`result_type`、`cube_id`、`score` 五字段字典 | +| `{stage}_filtered` | `search_filter_result` | `str` | `search_filter_result:{search_scope}:{stage}` | 固定字符串 `"filtered"` | + +| 搜索身份/位置字段 | 规则 | +| --- | --- | +| `stage` | `raw`、`threshold`、`dedup`、`rerank` | +| `result_type` | results 顶层 key 转字符串,例如 `text_mem`、`pref_mem`;handler 没有限定枚举,也不是源记忆的 `memory_type` | +| `cube_scope` | cube ID 非 `None` 时使用该 ID,否则使用 `bucket-{bucket_index}` | +| `occurrence_id` | memory ID 非 `None` 时使用该 ID,否则 `anonymous`;Search 取 ID 的顺序为 `id or memory_id` | +| `bucket_index` | 同一 result type 内的桶序号,从 0 开始 | +| `rank` | 当前桶内排名,从 1 开始;不代表所有 cube 合并后的排名 | +| `rank_scope` | `f"{result_type}:{cube_id or bucket_index}"`,用于说明排名所在桶;它与 `cube_scope` 的兜底写法不同 | +| `global_position` | 当前阶段遍历所有 result type 和桶时的位置,从 1 开始;不是全局按分数排序的名次 | +| `score` | 依次取源记忆 `metadata.relativity → metadata.score → memory.score`,仅遇到 `None` 才继续兜底,因此 `0` 有效 | + +| 节点 | `TraceValue.metadata` | +| --- | --- | +| `search_query` | `memos_stage="search.before"` | +| `search_memory` | `stage`、`result_type`、`cube_id`、`memory_id`、`rank`、`rank_scope`、`global_position`、`score` | +| `search_filter_result` | `stage`、`status="filtered"` | + +候选的 `comment` 同时写明 stage、rank、rank_scope、global_position,以及非 `None` 的 score。搜索节点的阶段字段叫 `stage`,不是 `memos_stage`。 + +Search 消费的形状是 `result_type → 桶列表 → memories 列表`,例如: + +```python +results = { + "text_mem": [ + { + "cube_id": "cube-1", + "memories": [ + {"id": "mem-1", "memory": "示例记忆", "metadata": {"relativity": 0.8}} + ], + } + ] +} +``` + +当前实现不解包 `{"data": results}`。传入回调时应提供业务 results 字典。 + +每个阶段都会建立新的 `search_memory` 节点,即使正文和顺序未变;同 ID 重复出现也由 `global_position` 区分。它们不复用 Add 的 `persisted_memory` 节点。 + +跨阶段配对使用另外的 `match_key`: + +```text +SHA256(UTF8(JSON([result_type, cube_scope, ["id", str(memory_id)]]))) +无 ID 时,最后一项改为 ["text", 原始未截断正文] +``` + +具体 JSON 编码使用 `json.dumps(..., ensure_ascii=False)`。匹配不包含 stage、rank、score;有 ID 时正文改变仍可关联。重复项按 FIFO 一对一消费。`match_key` 保存在 `TraceValue` 中供 handler 缓存使用,当前 recorder 不将它导出到节点或 metadata。 + +### 2.3 BaseHandler:通用失败状态 + +| 字段 | 取值 | +| --- | --- | +| `name / category` | 均为 `operation_status` | +| `class_name / value` | `str` / `"failed"` | +| `identity` | `operation_status:{operation_id or trace_id}:{stage}` | +| `metadata` | `memos_stage=stage`、`status="failed"` | +| 当前 stage | `mem_reader.extract.failed`、`text_memory.add.failed`、`search.post_process.failed` | + +MemRead 调度失败使用上一节的 `scheduler_result`,不使用这个节点。异常类型 `error_type` 放在事件/操作 metadata 中;异常对象和异常消息不作为失败节点内容。 + +当前 handler 共使用 **4 种 class_name**:`message_batch`、`memory`、`str`、`search_memory`;共使用 **9 种节点 category**:以上表格中的五种 Add、三种 Search 和一种通用失败状态。`TraceValue` 自身的默认 `category="variable"`、`class_name=None` 供自定义调用使用,当前 handler 显式指定这两个字段。 + +## 3. AddHandler 的 operation 与连线 + +### 3.1 同步添加 + +```mermaid +flowchart LR + M[message_batch] -->|memory_extraction| E[extracted_memory] + E -->|memory_persistence| P[persisted_memory] + M -->|operation_failed| F1[operation_status: 提取失败] + E -->|operation_failed| F2[operation_status: 写入失败] +``` + +图中失败分支按实际失败 Hook 出现,不会因为存在成功分支自动生成。 + +| Hook | `TraceEvent.operation` | 事件/操作 `category` | 输入 → 输出 / 边 category | +| --- | --- | --- | --- | +| `add.before` | `memos.add.message_input` | `memory_add_request` | 创建 message_batch 根节点,无边 | +| `add.after` | 不发事件 | — | 只清理 pending 消息关联 | +| `mem_reader.extract.after` | `memos.mem_reader.extract` | `memory_extraction` | message_batch → 每条 extracted_memory;`memory_extraction` | +| `mem_reader.extract.failed` | `memos.mem_reader.extract.failed` | `memory_extraction` | message_batch → operation_status;`operation_failed` | +| `text_memory.add.after` | `memos.text_memory.persist` | `memory_persistence` | 匹配到的 extracted_memory → persisted_memory;`memory_persistence` | +| `text_memory.add.failed` | `memos.text_memory.persist.failed` | `memory_persistence` | 各 extracted_memory → operation_status;`operation_failed` | + +| 操作 | 由 handler 添加的 `TraceEvent.metadata`,不含公共补充字段 | +| --- | --- | +| `memos.add.message_input` | `writable_cube_ids`、`async_mode`、`mode`、`custom_tags`、`info`、`chat_history` | +| `memos.mem_reader.extract` | `type`、`info`、`mode`、`user_name` | +| `memos.mem_reader.extract.failed` | 上述四项,加 `memos_stage="mem_reader.extract.failed"`、`status="failed"`、`error_type` | +| `memos.text_memory.persist` | `memos_stage="text_memory.add.after"`、`memory_count`、`lineage_status`、`backend`(text_memory 的类名) | +| `memos.text_memory.persist.failed` | `memos_stage="text_memory.add.failed"`、`status="failed"`、`error_type`、`backend` | + +注意 `memos_stage` 不是所有事件或节点的必有字段;成功的 add 输入、MemReader 提取事件目前不自动补它。 + +触发边界:反馈请求和空 messages 不产生 add 输入事件;MemReader 只观察 `type="chat"`,成功时要求消息和提取结果都非空。写入成功时,输入/输出都为空就不发事件;写入失败时,无可识别候选输入则不发失败事件。失败表示该操作失败,不承诺底层已经提交的部分写入被回滚。 + +这里的「写入成功」表示收到了成功 Hook 和返回 ID,插件不独立查询数据库确认。当前工作区的 `MemoryManager._submit_batches()` 捕获批量写入异常后只记录日志,可能导致失败写入仍走成功 Hook。因此不能仅凭 persisted_memory 节点认定实际写入成功;失败节点也依赖业务层正确传播异常。本次核对时,`test_write_failure_hooks.py` 的 4 个用例因此未通过。 + +### 3.2 持久化配对 + +同步写入和后台增强写入都按 **返回 memory ID 匹配唯一输入记录**,不按返回位置配对。返回值当前只接受 list/tuple 作为 ID 序列。 + +| 情况 | 节点和边 | +| --- | --- | +| 返回 ID 唯一匹配输入 | 使用该输入正文建立 persisted_memory,连来源边 | +| 返回 ID 不认识,或输入中同 ID 出现多次 | 保留只有 ID/cube 的 persisted_memory,不推测正文和来源边 | +| 部分输入没有对应返回 ID | 输入节点保留,不推测写入成功 | + +持久化边额外包含 `pair_index`(返回 ID 序列下标)和 `input_index`(展开输入记录下标),均从 0 开始。匹配索引按 memory ID 构建;输入含多个相同 ID 时,即使来自不同 cube,也按歧义处理。 + +`lineage_status` 表示本次持久化输出的来源配对情况:所有输出都有来源边为 `resolved`,只有部分有边为 `partial`,有输出但无边为 `unresolved`。没有持久化输出和边时也为 `resolved`。`memory_count` 是构造出的输出数量,在 `_unique_values()` 去重前计数,不保证等于最终图中新增加的节点数。 + +### 3.3 MemRead 后台处理 + +```mermaid +flowchart LR + P[persisted_memory] -->|memory_refinement| E[enhanced_memory] + E -->|memory_persistence| P2[persisted_memory] + P -->|归档、删除或其他操作结果| S[scheduler_result] + E -->|scheduler_operation_failed| SF[scheduler_result: failed] +``` + +只接受 `hook_context.source` 以 `scheduler.mem_read.` 开头的调度 Hook。去掉前缀得到 `operation_name`。 + +| 字段 | 约定 | +| --- | --- | +| 订阅 Hook | `scheduler.memory.operation.after` / `scheduler.memory.operation.failed` | +| `event.operation` | `memos.scheduler.mem_read.{operation_name}.{status}` | +| `event.category` | 固定 `scheduler_memory_operation` | +| 共同 metadata | `memos_stage="scheduler.mem_read.{operation_name}.{status}"`、`handler="mem_read"`、`operation_name`、`operation`、`target`、`status` | +| `operation / target` | 直接记录 Hook 参数,例如 `fine/add/archive/delete/soft_delete` 和 `textual_memory`;不是插件内强制枚举 | +| 失败额外 metadata | `error_type`,仅异常类名 | + +| `operation_name` / 情况 | 输入、输出与边 category | +| --- | --- | +| `fine_transfer_simple_mem` 成功且有记忆输出 | operation_input.memories 中的持久化记忆引用 → enhanced_memory;`memory_refinement` | +| `add_enhanced_memories` 成功且有返回 ID | enhanced_memory → persisted_memory;`memory_persistence`,沿用唯一 ID 配对 | +| `archive_merged_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_archival` | +| `remove_memories` / `remove_source_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_deletion` | +| 其他操作成功,如 `refresh_memory_size` | memory_ids 对应持久化记忆 → scheduler_result;`scheduler_operation_result` | +| 失败 | 当前操作输入 → scheduler_result;`scheduler_operation_failed`;增强写入失败保留 enhanced_memory 输入 | + +handler 并没有封闭的 operation_name 枚举:上述特殊分支之外,其他符合 source 前缀的名称按通用结果分支处理。成功的 fine-transfer/增强写入如果没有记忆输出,也会生成 scheduler_result;没有输入时保留独立结果节点和 operation,不补造边。 + +fine-transfer 额外记录 `result_grouping`,取自 `operation_input`,缺省 `unknown`: + +| `result_grouping` / 输入情况 | 连线规则 | +| --- | --- | +| `per_input` 且外层结果组数等于输入数 | 按输入顺序逐组连接,一条输入可产生多条输出;失败/无输出对应的空组必须保留 | +| `batch` | 整批输入共同连接到每个输出 | +| 只有一个输入 | 即使 grouping 未知,也可以把输出连接到唯一输入 | +| 多输入且 grouping 未知,或 per_input 组数不一致 | 保留输入/输出,但不猜来源边 | + +fine-transfer 成功且存在 enhanced_memory 输出时,`lineage_status` 取 `resolved`(存在来源边)或 `unresolved`(无来源边),没有 `partial` 分支。增强写入成功则使用上一节的持久化 lineage 规则。插件本身消费 Hook 提供的 grouping,不识别 Reader 的具体类。 + +## 4. SearchHandler 的 operation 与连线 + +```mermaid +flowchart LR + Q[search_query] -->|memory_retrieval| R[raw] + R -->|memory_threshold_passed| T[threshold] + T -->|memory_deduplicated| D[dedup] + D -->|memory_reranked| RR[rerank] + R -->|memory_filtered| F1[threshold: filtered] + T -->|memory_filtered| F2[dedup: filtered] + D -->|memory_filtered| F3[rerank: filtered] +``` + +raw、threshold、dedup、rerank 框都表示 `search_memory` 节点。 + +| Hook | `TraceEvent.operation` | 事件/操作 `category` | 主要边 category | +| --- | --- | --- | --- | +| `search.before` | `memos.search.request` | `search_request` | query 根节点,无边 | +| `search.memory_results` | `memos.search.retrieve` | `memory_retrieval` | query → raw;`memory_retrieval` | +| `search.results.after_threshold` | `memos.search.threshold_filter` | `memory_filtering` | raw → threshold;`memory_threshold_passed` | +| `search.results.after_dedup` | `memos.search.deduplicate` | `memory_deduplication` | threshold → dedup;`memory_deduplicated` | +| `search.results.after_rerank` | `memos.search.rerank` | `memory_reranking` | dedup → rerank;`memory_reranked` | +| `search.post_process.failed` | `memos.search.post_process.failed` | `search_failure` | query → operation_status;`operation_failed` | + +threshold、dedup、rerank 中,前一阶段未匹配到的候选都以 `memory_filtered` 连到当前阶段共用的 `search_filter_result`。因此 `filtered` 不只表示阈值未通过,也包括去重删除及 rerank 阶段消失的候选。新出现且找不到来源的候选仍保留,不连推测边。 + +| 操作 | 由 handler 添加的 `TraceEvent.metadata`,不含公共补充字段 | +| --- | --- | +| request | `memos_stage="search.before"` | +| retrieve | `memos_stage="search.memory_results"`、`top_k` | +| threshold_filter | `memos_stage="search.results.after_threshold"`、`threshold`(request.relativity,空值按 0)、`score_field="metadata.relativity"`、`applied=(threshold > 0)` | +| deduplicate | `memos_stage="search.results.after_dedup"`、`method`(request.dedup,缺省/空值按 `no`)、`applied=(method in {sim, mmr})`、`top_k` | +| rerank | `memos_stage="search.results.after_rerank"`、`algorithm="relativity_sort"`、`order_by="metadata.relativity"`、`direction="descending"`、`top_k` | +| post_process.failed | `memos_stage="search.post_process.failed"`、`status="failed"`、`error_type`、`handler`(API handler 的类名) | + +四个结果阶段另加 `link_strategy="explicit_pairwise"` 和 `filtered_count`。raw 阶段 filtered_count 为 0;其余阶段是前一阶段未匹配候选数。rerank 的 algorithm/order_by/direction 是插件当前固定写入的描述,不是对实际后端算法的动态识别;插件也不自己重新过滤、去重或排序。`top_k` 是当前 Hook 请求中的值,可能已经被业务层扩充。 + +缓存键为 `(trace_id, search_scope_id, stage)`。rerank 和 post_process.failed 清理本次搜索的所有阶段缓存,shutdown 清理全部。若缺少 `operation_id`,search_scope 退回 trace,此时同 trace 多次搜索不再具有独立身份,调用方应提供一次搜索各 Hook 共享的 operation_id。 + +观测终点为 `search.results.after_rerank`;插件没有订阅 `search.context.render` 或 `search.after`。 + +## 5. metadata 分层与合并规则 + +| 层级 | 内容 | 自动继承情况 | +| --- | --- | --- | +| 节点 `TraceValue.metadata` | 第 2 节列出的值身份、阶段、排名或输入参数 | 不自动继承事件 metadata、HookContext.attributes 或源记忆的完整 metadata | +| 事件 `TraceEvent.metadata` | 第 3/4 节列出的本次操作参数和结果说明 | `_event()` 在 subject.operation_id 非 `None` 时用 setdefault 补 `operation_id` | +| 导出 operation metadata | 事件 metadata,加关联公共字段 | 公共字段覆盖事件 metadata 同名项 | +| 导出 edge metadata | operation metadata,加 `TraceLink.metadata` | link 字段优先;持久化边额外有 pair_index/input_index | +| 导出 session metadata | 单独构造的事件关联字段 | 不自动复制完整 event metadata | + +Recorder 的公共字段如下: + +```python +operation_metadata = { + **event.metadata, + "trace_event_id": event.event_id, + "memos_trace_id": event.trace_id, + "memos_session_id": event.session_id, + "task_id": event.task_id, + "cube_ids": list(event.cube_ids), +} +edge_metadata = {**operation_metadata, **link.metadata} + +session_metadata = { + "memos_session_id": event.session_id, + "task_id": event.task_id, + "cube_ids": list(event.cube_ids), + "trace_event_id": event.event_id, + "trace_event_created_at": event.created_at, +} +``` + +`user_id` 由 graph 持有并进入 SmartComment 对象的顶层字段,不在以上公共 metadata 字典中。link 未指定 category/comment 时沿用 operation;`event.category` 同时作为 operation 和 session 的 category,**不会把节点 category 改成操作 category**。 + +目前只有持久化配对边显式添加 link metadata,其他 handler 边使用空字典并继承 operation metadata。 + +## 6. 快照、导出与边界 + +`_value()` 对 value 和 metadata、`_event()` 对事件 metadata,在 Hook 当前线程内执行快照;后台线程负责写图。dataclass 的 frozen 只禁止字段重新赋值,不深度冻结嵌套字典。 + +| 规则 | 当前行为 | +| --- | --- | +| 字符串长度 | 默认保留前 20,000 字符,超出追加 `…`;限制按每个字符串计算 | +| 容器大小 | 默认每层最多 200 项,附截断提示;不等于一个事件最多 200 个节点 | +| 敏感字段 | 字段名规范化后包含 api_key/apikey/access_key/password/secret/authorization/cookie/credential/token 时,值替换为 `` | +| 向量字段 | embedding/embeddings/vector/vectors 的值替换为 `` | +| argument/arguments JSON 字符串 | 先解析并递归脱敏,再编码和截断;不能解析为 dict/list 则替换为 `` | +| 自由文本 | 按长度截断,不扫描正文中的任意凭据 | +| 其他类型 | 循环引用用 ``,bytes 用长度标记,非有限浮点数转字符串,模型/对象尽可能展开字段 | +| 导出节点 value | 用 `json.dumps(value, ensure_ascii=False, sort_keys=True)` 编码为字符串;消费导出 JSON 时需对 node.value 再做一次 JSON 解码 | +| 内部哨兵 | 导出移除 `COMMENT:NONE@1` 及其关联边、`__none_op__` 和 `__none__` session | + +身份和 comment 并不经过 `_value()` 的 value/metadata 快照流程。队列入满会丢弃追踪事件并告警,所以图用于记录已观测到的数据依赖,不能把缺失节点/边直接解释为业务步骤从未发生。 + +输出文件按 user/trace 隔离,目录名和文件名为经过清洗且最多 80 字符的可读前缀,加原始 ID 的 JSON 编码的 SHA-256;文件原子替换,后续事件可恢复已有图。缓存淘汰不删除磁盘快照。配置和运行方式见 [应用 README](../README.md)。 + +## 7. 查代码与扩展时的检查点 + +| 问题 | 主要实现 | +| --- | --- | +| trace/cube 从哪里来 | [base.py](handlers/base.py):`_current_trace_id`、`_cube_ids`、`_hook_subject` | +| Add 身份与 metadata | [add_handler.py](handlers/add_handler.py):`_message_value`、`_extracted_memory_value`、`_persisted_memory_value`、`_enhanced_memory_value` | +| 返回 ID 怎么连回输入 | [add_handler.py](handlers/add_handler.py):`_persistence_values`、`_persistence_lineage` | +| 后台增强的来源关系 | [add_handler.py](handlers/add_handler.py):`_fine_transfer_values`、`_on_scheduler_memory_operation` | +| 搜索节点、排名、跨阶段配对 | [search_handler.py](handlers/search_handler.py):`_search_result_items`、`_search_memory_value`、`_search_memory_match_key`、`_match_search_nodes` | +| 图中身份、metadata、占位补全 | [recorder.py](recorder.py):`_as_comment_item`、`record`、`_complete_memory_snapshots` | +| 快照与落盘 | [serialization.py](serialization.py)、[adapter.py](adapter.py) | + +新增节点时要明确身份作用域、value 形状、class_name、节点 category 和引用方式;新增操作时要独立定义 operation/category/metadata,并依据已知业务来源显式创建 TraceLink。来源无法确认时保留节点,不用操作顺序或输入输出下标推测边。 diff --git a/apps/memtrace-memos-integration/memos_smartcomment/__init__.py b/apps/memtrace-memos-integration/memos_smartcomment/__init__.py new file mode 100644 index 000000000..9d219ff6e --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/__init__.py @@ -0,0 +1,19 @@ +# ruff: noqa: N999 +# 包入口: 对外只暴露 SmartCommentPlugin, 具体实现按需导入. +"""MemOS adapter for asynchronous smartcomment execution-graph tracing.""" + +from __future__ import annotations + +from typing import Any + + +__all__ = ["SmartCommentPlugin"] + + +# 只有访问 SmartCommentPlugin 时才导入插件及其依赖, 避免导入包时立即加载完整集成. +def __getattr__(name: str) -> Any: + if name == "SmartCommentPlugin": + from memos_smartcomment.plugin import SmartCommentPlugin + + return SmartCommentPlugin + raise AttributeError(name) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py new file mode 100644 index 000000000..a0e49838d --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py @@ -0,0 +1,250 @@ +# 异步传输与落盘层: 多个 Hook 线程提交事件, 一个后台线程串行更新和保存执行图. +# 图按 (user_id, trace_id) 隔离; 内存缓存可淘汰, 后续事件可以从磁盘快照继续追加. +from __future__ import annotations + +import hashlib +import json +import os +import queue +import re +import threading +import time + +from collections import OrderedDict +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any, Protocol +from uuid import uuid4 + +from memos_smartcomment.recorder import SmartCommentRecorder + +from memos.exceptions import MemOSError +from memos.log import get_logger + + +if TYPE_CHECKING: + from collections.abc import Callable + from pathlib import Path + + from memos_smartcomment.config import SmartCommentSettings + from memos_smartcomment.events import TraceEvent + + +logger = get_logger(__name__) + + +# 写入器需要满足的最小接口; 测试可注入替代实现, 不必直接依赖 SmartComment. +class Recorder(Protocol): + def record(self, event: TraceEvent) -> None: ... + + def export_graph(self) -> dict[str, Any]: ... + + def restore_graph(self, data: dict[str, Any]) -> None: ... + + +# accepted 为入队数, dropped 为队满丢弃数, rejected 为关闭后拒收数. +# processed 为处理成功数, failed 为后台处理异常数; 入队成功不代表已经落盘. +@dataclass(slots=True) +class AdapterStats: + accepted: int = 0 + dropped: int = 0 + processed: int = 0 + failed: int = 0 + rejected: int = 0 + + +# 记录单张图、最近使用时间和图元素数量, 用于 LRU/TTL 与容量淘汰. +@dataclass(slots=True) +class _CachedRecorder: + recorder: Recorder + last_used: float + graph_items: int + + +# 可读前缀后追加原始 ID 的 SHA-256, 避免不同 ID 经字符替换或截断后重名. +def _safe_path_component(value: str | None, fallback: str) -> str: + normalized = re.sub(r"[^A-Za-z0-9._-]+", "_", value or fallback).strip("._") + prefix = (normalized or fallback)[:80] + digest = hashlib.sha256(json.dumps(value, ensure_ascii=False).encode("utf-8")).hexdigest() + return f"{prefix}-{digest}" + + +class AsyncTraceAdapter: + """Bounded event queue and graph cache, with one snapshot-writing worker.""" + + # 这里只初始化队列与状态; 真正启动线程由 start 或有待处理事件的 close 触发. + def __init__( + self, + settings: SmartCommentSettings, + *, + recorder_factory: Callable[[TraceEvent], Recorder] | None = None, + ) -> None: + self.settings = settings + self._recorder_factory = recorder_factory or ( + lambda event: SmartCommentRecorder(settings, event) + ) + self._queue: queue.Queue[TraceEvent] = queue.Queue(maxsize=settings.queue_size) + self._recorders: OrderedDict[tuple[str | None, str], _CachedRecorder] = OrderedDict() + self._cached_graph_items = 0 + self._thread: threading.Thread | None = None + self._started = False + self._closed = False + # 此锁保护启动/关闭状态和统计; 图缓存只由后台线程访问, 不需要逐图加锁. + self._state_lock = threading.Lock() + self._stats = AdapterStats() + + # 在锁内返回统计副本, 防止调用者直接修改内部计数. + @property + def stats(self) -> AdapterStats: + with self._state_lock: + return replace(self._stats) + + # 调用方已持有状态锁; 一个写入线程独占图缓存, 避免并发修改同一张图. + def _start_worker(self) -> None: + self._thread = threading.Thread( + target=self._run, name="memos-smartcomment-writer", daemon=True + ) + self._thread.start() + self._started = True + + # 允许重复启动调用, 但关闭后的实例不能重新启动. + def start(self) -> None: + with self._state_lock: + if self._closed: + raise MemOSError("The smartcomment adapter is already closed") + if not self._started: + self._start_worker() + + # 非阻塞入队; 与 close 共用状态锁, 确保关闭后不再接受新事件. + def submit(self, event: TraceEvent) -> bool: + with self._state_lock: + if self._closed: + self._stats.rejected += 1 + return False + try: + # 队满时丢弃追踪事件而不等待写盘, 避免追踪积压拖住 MemOS 业务请求. + self._queue.put_nowait(event) + except queue.Full: + self._stats.dropped += 1 + else: + self._stats.accepted += 1 + return True + logger.warning("Dropping smartcomment event because the queue is full: %s", event.operation) + return False + + # 每个用户目录下每条 trace 一个 JSON 文件; 匿名用户同样参与哈希隔离. + def output_path(self, trace_id: str, user_id: str | None = None) -> Path: + user_part = _safe_path_component(user_id, "anonymous") + trace_part = _safe_path_component(trace_id, "trace") + return self.settings.output_dir / user_part / f"{trace_part}.json" + + # 只读取带哈希的新路径; 文件不存在时从空图开始. + def _read_snapshot(self, event: TraceEvent) -> dict[str, Any] | None: + path = self.output_path(event.trace_id, event.user_id) + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return None + + # 先完整写入同目录临时文件, 再原子替换目标快照, 避免读到半份 JSON. + def _export(self, event: TraceEvent, recorder: Recorder) -> int: + exported = recorder.export_graph() + output_path = self.output_path(event.trace_id, event.user_id) + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = output_path.with_name(f".{output_path.name}.{uuid4().hex}.tmp") + try: + temporary_path.write_text( + json.dumps(exported, ensure_ascii=False, indent=2), encoding="utf-8" + ) + os.replace(temporary_path, output_path) + finally: + temporary_path.unlink(missing_ok=True) + # 缓存容量计入节点、边、操作和会话, 不只计算记忆节点数量. + data = exported.get("data", {}) + return sum(len(data.get(key, ())) for key in ("nodes", "edges", "operations", "sessions")) + + # OrderedDict 左侧是最久未使用的图; 任一容量限制或空闲期限触发时从左侧淘汰. + def _trim_cache(self) -> None: + now = time.monotonic() + while self._recorders: + oldest = next(iter(self._recorders.values())) + if ( + len(self._recorders) <= self.settings.max_cached_traces + and self._cached_graph_items <= self.settings.max_cached_graph_items + and now - oldest.last_used < self.settings.cache_ttl_seconds + ): + break + _, removed = self._recorders.popitem(last=False) + self._cached_graph_items -= removed.graph_items + + # 加载或复用目标图, 追加本次事件, 成功落盘后才重新纳入缓存. + def _process(self, event: TraceEvent) -> None: + key = (event.user_id, event.trace_id) + # 先从缓存移出; 如果后续记录或导出失败, 下次会重新读取最近一次成功快照. + cached = self._recorders.pop(key, None) + if cached is None: + recorder = self._recorder_factory(event) + snapshot = self._read_snapshot(event) + if snapshot is not None: + recorder.restore_graph(snapshot) + else: + self._cached_graph_items -= cached.graph_items + recorder = cached.recorder + # Cache only committed snapshots; failures reload the last durable graph. + recorder.record(event) + graph_items = self._export(event, recorder) + self._recorders[key] = _CachedRecorder(recorder, time.monotonic(), graph_items) + self._cached_graph_items += graph_items + self._trim_cache() + + # 消费循环同时负责空闲淘汰与退出清理; 单个事件失败会计数并继续处理下一条. + def _run(self) -> None: + try: + while True: + try: + event = self._queue.get(timeout=0.05) + except queue.Empty: + self._trim_cache() + with self._state_lock: + if self._closed and self._queue.empty(): + return + continue + try: + self._process(event) + except Exception: + with self._state_lock: + self._stats.failed += 1 + logger.exception("Failed to record smartcomment event: %s", event.operation) + else: + with self._state_lock: + self._stats.processed += 1 + finally: + # 无论成功失败都减少 unfinished_tasks, 否则 flush 可能一直等待. + self._queue.task_done() + finally: + self._recorders.clear() + self._cached_graph_items = 0 + + # 等待所有已入队任务处理结束, 超时返回 False. + # 失败事件也会完成任务计数, 所以返回 True 不表示所有写入成功; 需结合 stats.failed 判断. + def flush(self, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + with self._queue.all_tasks_done: + while self._queue.unfinished_tasks: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._queue.all_tasks_done.wait(remaining) + return True + + # 先拒收新事件, 再等待线程排空队列; 等待超时只告警, 不强行中断正在进行的写入. + def close(self, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + with self._state_lock: + self._closed = True + # 兼容先 submit 后 close 的调用顺序: 即使尚未启动也要处理已接受的事件. + if not self._started and not self._queue.empty(): + self._start_worker() + thread = self._thread + if thread is not None: + thread.join(timeout=max(0, deadline - time.monotonic())) + if thread.is_alive(): + logger.warning("smartcomment writer did not stop within %.1f seconds", timeout) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/config.py b/apps/memtrace-memos-integration/memos_smartcomment/config.py new file mode 100644 index 000000000..6c7c63bec --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/config.py @@ -0,0 +1,64 @@ +# 插件独立配置: 启动时读取 MEMOS_SMARTCOMMENT_* 环境变量, 不修改 MemOS 的业务配置. +from __future__ import annotations + +import os + +from dataclasses import dataclass +from pathlib import Path + + +# 整数配置缺失、解析失败或不大于零时使用默认值, 保证队列和缓存上限有效. +def _positive_int(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError: + return default + return value if value > 0 else default + + +# 只将 1/true/yes/on 视为开启, 忽略大小写和两端空格; 其他已设置的值视为关闭. +def _as_bool(name: str, default: bool = False) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +# 集中描述写图位置、快照大小、队列容量和内存缓存限制. +@dataclass(frozen=True, slots=True) +class SmartCommentSettings: + """Runtime settings owned by the external tracing plugin.""" + + output_dir: Path = Path(".memos/memtrace") + # 队列限制待处理事件数; 单条事件的字符串长度另由 max_value_chars 限制. + queue_size: int = 1024 + max_value_chars: int = 20_000 + # 传给 SmartComment 的身份校验开关; 持久化记忆仍按 recorder 的不可变锚点规则处理. + strict: bool = False + project_id: str = "memos" + # 缓存按图数量、图元素总数和空闲时间淘汰; 已写出的 JSON 文件不会随缓存删除. + max_cached_traces: int = 64 + max_cached_graph_items: int = 50_000 + cache_ttl_seconds: float = 300 + + # 一次性生成配置快照; 修改环境变量后需重新加载插件才能使用新值. + @classmethod + def from_env(cls) -> SmartCommentSettings: + return cls( + output_dir=Path(os.getenv("MEMOS_SMARTCOMMENT_OUTPUT_DIR", ".memos/memtrace")), + queue_size=_positive_int("MEMOS_SMARTCOMMENT_QUEUE_SIZE", 1024), + max_value_chars=_positive_int( + "MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS", + 20_000, + ), + strict=_as_bool("MEMOS_SMARTCOMMENT_STRICT"), + project_id=os.getenv("MEMOS_SMARTCOMMENT_PROJECT_ID", "memos"), + max_cached_traces=_positive_int("MEMOS_SMARTCOMMENT_MAX_CACHED_TRACES", 64), + max_cached_graph_items=_positive_int( + "MEMOS_SMARTCOMMENT_MAX_CACHED_GRAPH_ITEMS", 50_000 + ), + cache_ttl_seconds=_positive_int("MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS", 300), + ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/events.py b/apps/memtrace-memos-integration/memos_smartcomment/events.py new file mode 100644 index 000000000..bee4dd6fe --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/events.py @@ -0,0 +1,65 @@ +# 事件契约: handlers 生成这些数据对象, adapter 传递它们, recorder 将其转换为图. +# TraceValue 是节点描述, TraceLink 是明确的数据依赖, TraceEvent 是一次操作的观测结果. +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + + +# 一个业务值的快照, 例如消息批次或记忆正文. +# frozen 只禁止字段重新赋值, 不会冻结嵌套字典; 脱离业务对象的快照由 handlers 负责生成. +@dataclass(frozen=True, slots=True) +class TraceValue: + """One semantic variable consumed or produced by a MemOS operation.""" + + name: str + value: Any + # identity 决定图中复用哪个节点; name 是业务名称, 不承担唯一标识职责. + identity: str + category: str = "variable" + class_name: str | None = None + # True 表示按身份引用已有节点; 首次出现也可以创建占位节点. + # 持久化记忆的占位补全规则由 recorder 单独处理. + identity_only: bool = False + comment: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + # 搜索跨阶段匹配使用截断前的指纹, 防止不同长文本在截断后被误认为相同. + match_key: str | None = None # Correlation key computed before snapshot truncation. + + +# source -> target 表示目标使用了来源数据; 它不表示两个操作仅在时间上相邻. +# category/comment 未指定时沿用操作默认值; metadata 可覆盖同名操作元数据. +@dataclass(frozen=True, slots=True) +class TraceLink: + """One explicit variable-to-variable dependency within an operation.""" + + source: TraceValue + target: TraceValue + category: str | None = None + comment: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +# 一次 Hook 观测对应一个事件, 同一 trace 下的多个事件最终进入同一张图. +# 即使没有连线也保留输入输出节点; 只有显式 links 才产生边. +@dataclass(frozen=True, slots=True) +class TraceEvent: + """Framework-neutral event; only explicit links create graph edges.""" + + operation: str + category: str + trace_id: str + session_id: str | None = None + task_id: str | None = None + user_id: str | None = None + cube_ids: tuple[str, ...] = () + inputs: tuple[TraceValue, ...] = () + outputs: tuple[TraceValue, ...] = () + # Inputs/outputs preserve standalone nodes; they never imply dependencies. + links: tuple[TraceLink, ...] = () + metadata: dict[str, Any] = field(default_factory=dict) + # event_id 标识本次观测; operation_id 若存在则保存在 metadata 中, 用于关联业务操作. + event_id: str = field(default_factory=lambda: uuid4().hex) + created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat()) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py new file mode 100644 index 000000000..6ac803084 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py @@ -0,0 +1,25 @@ +# ruff: noqa: N999 +"""按业务链组织 Hook 回调, 并保留插件使用的统一入口。 + +AddHandler / SearchHandler 可独立使用; HookHandlers 提供两者的全部回调。 +""" + +from memos_smartcomment.handlers.add_handler import AddHandler +from memos_smartcomment.handlers.search_handler import SearchHandler + + +__all__ = ["AddHandler", "HookHandlers", "SearchHandler"] + + +class HookHandlers(AddHandler, SearchHandler): + """Combine add and search callbacks behind the existing plugin interface. + + Both handlers use super() so initialization follows + AddHandler -> SearchHandler -> BaseHandler, sharing one event sink while + keeping their correlation caches separate. + """ + + def clear(self) -> None: + """Release both flows' correlation state at shutdown.""" + AddHandler.clear(self) + SearchHandler.clear(self) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py new file mode 100644 index 000000000..7630e74fc --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -0,0 +1,790 @@ +"""添加链路及其后台 MemRead 增强处理。 + +消息 -> 提取记忆 -> 持久化记忆 -> 增强记忆 -> 持久化记忆。 +先列出 Hook 回调, 再列出消息批次关联、节点构造和来源配对函数。 +""" + +from __future__ import annotations + +import weakref + +from collections import OrderedDict, defaultdict +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from threading import Lock +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue +from memos_smartcomment.handlers.base import ( + BaseHandler, + _cube_ids, + _current_trace_id, + _get, + _memory_text, +) + + +if TYPE_CHECKING: + from memos.plugins.hook_context import HookContext + + +_MEM_READ_SOURCE_PREFIX = "scheduler.mem_read." + + +# 仅在明确只有一个 cube 时给出默认值, 多 cube 场景不猜测某条记忆的归属. +def _single_cube_id(subject: Any) -> str | None: + cube_ids = _cube_ids(subject) + return cube_ids[0] if len(cube_ids) == 1 else None + + +# 递归展开各种嵌套返回结构, 产出 (记忆对象, cube_id, memory_id). +# 仅接受同时具有 ID 和正文的记录; 遍历时沿用外层 cube, 并跳过元数据包装字段. +def _walk_memory_records( + value: Any, cube_id: str | None = None +) -> Iterator[tuple[Any, str | None, str]]: + if value is None: + return + + current_cube = _get(value, "cube_id", cube_id) or _get(value, "user_name", cube_id) + memory_id = _get(value, "memory_id") or _get(value, "id") + if memory_id is not None and _memory_text(value) is not None: + yield value, str(current_cube) if current_cube else None, str(memory_id) + return + + if isinstance(value, Mapping): + for key, item in value.items(): + if key not in {"metadata", "info", "internal_info"}: + yield from _walk_memory_records(item, current_cube) + elif isinstance(value, list | tuple | set): + for item in value: + yield from _walk_memory_records(item, current_cube) + + +# 保存一次 add 的消息身份及尚未完成提取的 cube, 供后续 MemReader 回调接续同一输入节点. +@dataclass(slots=True) +class _MessageBatch: + owner: Callable[[], Any] + identity: str + pending_cubes: set[str] + + +class AddHandler(BaseHandler): + """Capture add, extraction, persistence, and asynchronous MemRead memory flows.""" + + def __init__( + self, + submit: Callable[[TraceEvent], Any], + *, + max_value_chars: int = 20_000, + max_pending_adds: int = 1024, + ) -> None: + super().__init__(submit, max_value_chars=max_value_chars) + self._max_pending_adds = max_pending_adds + self._message_batches: OrderedDict[tuple[str, str | None, int], _MessageBatch] = ( + OrderedDict() + ) + self._message_batches_lock = Lock() + + def clear(self) -> None: + """Release pending message batches at shutdown.""" + with self._message_batches_lock: + self._message_batches.clear() + + # 添加链路入口: 仅跟踪非反馈且非空的消息批次, 首次创建图的根输入节点. + def on_add_before(self, *, request: Any, **_kwargs: Any) -> None: + if bool(_get(request, "is_feedback", False)): + return + messages = _get(request, "messages") + if not messages: + return + metadata = { + "writable_cube_ids": _get(request, "writable_cube_ids"), + "async_mode": _get(request, "async_mode"), + "mode": _get(request, "mode"), + "custom_tags": _get(request, "custom_tags"), + "info": _get(request, "info"), + "chat_history": _get(request, "chat_history"), + } + message = self._message_value(request, messages, identity_only=False, metadata=metadata) + self._emit( + self._event( + request, + operation="memos.add.message_input", + category="memory_add_request", + outputs=(message,), + metadata=metadata, + ) + ) + + # add 结束后的兜底清理, 覆盖跳过 MemReader 等未消费批次关联的路径. + def on_add_after(self, *, request: Any, **_kwargs: Any) -> None: + """Clear pending batches if the add path did not invoke MemReader.""" + key = self._message_batch_key(request, _get(request, "messages")) + with self._message_batches_lock: + self._message_batches.pop(key, None) + + # 只处理 chat 场景, 将同一消息批次连向每条提取出的记忆; 不记录 Reader 内部变量. + def on_mem_reader_extract_after( + self, + *, + hook_context: HookContext, + scene_data: Any, + type: str, + info: Any, + mode: Any, + user_name: Any, + result: Any, + **_kwargs: Any, + ) -> None: + if type != "chat": + return + + messages = self._messages_from_scene_data(scene_data) + subject = self._hook_subject(hook_context, user_name=user_name) + message = self._message_value(subject, messages, identity_only=True) if messages else None + default_cube_id = str(user_name) if user_name else None + metadata = { + "type": type, + "info": info, + "mode": mode, + "user_name": user_name, + } + extracted = tuple( + self._extracted_memory_value( + subject, + memory, + cube_id=cube_id or default_cube_id, + memory_id=memory_id, + identity_only=False, + metadata=metadata, + ) + for memory, cube_id, memory_id in _walk_memory_records(result, default_cube_id) + ) + # 消息关联已在构造 message 时消费, 因此空提取结果也不会遗留这次 cube 的待处理状态. + if message is None or not extracted: + return + + self._emit( + self._event( + subject, + operation="memos.mem_reader.extract", + category="memory_extraction", + inputs=(message,), + outputs=extracted, + links=tuple( + TraceLink(source=message, target=memory, category="memory_extraction") + for memory in extracted + ), + metadata=metadata, + ) + ) + + # chat 提取失败时消费消息批次关联并连向失败状态, 只保留异常类型. + def on_mem_reader_extract_failed( + self, + *, + hook_context: HookContext, + scene_data: Any, + type: str, + info: Any, + mode: Any, + user_name: Any, + error: BaseException, + **_kwargs: Any, + ) -> None: + if type != "chat": + return + messages = self._messages_from_scene_data(scene_data) + if not messages: + return + subject = self._hook_subject(hook_context, user_name=user_name) + message = self._message_value(subject, messages, identity_only=True) + status = self._operation_status_value(subject, stage="mem_reader.extract.failed") + self._emit( + self._event( + subject, + operation="memos.mem_reader.extract.failed", + category="memory_extraction", + inputs=(message,), + outputs=(status,), + links=(TraceLink(source=message, target=status, category="operation_failed"),), + metadata={ + "memos_stage": "mem_reader.extract.failed", + "status": "failed", + "error_type": error.__class__.__name__, + "type": type, + "info": info, + "mode": mode, + "user_name": user_name, + }, + ) + ) + + # 根据写入成功 Hook 的返回 ID 建立 extracted_memory -> persisted_memory. + # 这是对 Hook 通知的观测; 能否反映真实写入结果依赖 MemOS 正确传播数据库异常. + def on_text_memory_add_after( + self, + *, + hook_context: HookContext, + text_memory: Any, + memories: Any, + kwargs: Any, + result: Any, + ) -> None: + subject = self._hook_subject(hook_context) + cube_id = _single_cube_id(subject) or _get(kwargs, "user_name") + inputs, outputs, links = self._persistence_values( + subject, memories, result, cube_id=str(cube_id) if cube_id is not None else None + ) + + if not inputs and not outputs: + return + self._emit( + self._event( + subject, + operation="memos.text_memory.persist", + category="memory_persistence", + inputs=self._unique_values(inputs), + outputs=self._unique_values(outputs), + links=tuple(links), + metadata={ + "memos_stage": "text_memory.add.after", + "memory_count": len(outputs), + "lineage_status": self._persistence_lineage(outputs, links), + "backend": type(text_memory).__name__, + }, + ) + ) + + # 写入失败 Hook 将候选记忆连到失败状态, 不依据输入数量推断成功落库节点. + # 失败描述本次操作结果; 如果底层有部分写入已提交, 这里不表示那些写入被回滚. + def on_text_memory_add_failed( + self, + *, + hook_context: HookContext, + text_memory: Any, + memories: Any, + kwargs: Any, + error: BaseException, + ) -> None: + subject = self._hook_subject(hook_context) + cube_id = _single_cube_id(subject) or _get(kwargs, "user_name") + inputs, _, _ = self._persistence_values( + subject, memories, (), cube_id=str(cube_id) if cube_id is not None else None + ) + if not inputs: + return + status = self._operation_status_value(subject, stage="text_memory.add.failed") + self._emit( + self._event( + subject, + operation="memos.text_memory.persist.failed", + category="memory_persistence", + inputs=tuple(inputs), + outputs=(status,), + links=tuple( + TraceLink(source=value, target=status, category="operation_failed") + for value in inputs + ), + metadata={ + "memos_stage": "text_memory.add.failed", + "status": "failed", + "error_type": error.__class__.__name__, + "backend": text_memory.__class__.__name__, + }, + ) + ) + + # 转发成功操作及 Hook 已提供的 result, 不重新调用调度器业务方法. + def on_scheduler_memory_operation_after( + self, + *, + hook_context: HookContext, + operation: Any, + target: Any, + operation_input: Any, + result: Any, + ) -> None: + self._on_scheduler_memory_operation( + hook_context=hook_context, + operation=operation, + target=target, + operation_input=operation_input, + result=result, + status="after", + ) + + # 转发失败观测, 让输入连向失败状态; 原始异常的继续传播由 MemOS 调用方处理. + def on_scheduler_memory_operation_failed( + self, + *, + hook_context: HookContext, + operation: Any, + target: Any, + operation_input: Any, + error: BaseException, + ) -> None: + self._on_scheduler_memory_operation( + hook_context=hook_context, + operation=operation, + target=target, + operation_input=operation_input, + error=error, + status="failed", + ) + + # Python 对象地址只用于短期查找同一消息批次, 不直接作为图中永久身份. + @staticmethod + def _message_batch_key(subject: Any, messages: Any) -> tuple[str, str | None, int]: + return _current_trace_id(subject), _get(subject, "user_id"), id(messages) + + # reference=False 表示新 add, 分配新 UUID; True 表示提取回调, 尝试消费已有批次关联. + def _message_identity(self, subject: Any, messages: Any, *, reference: bool) -> str: + key = self._message_batch_key(subject, messages) + with self._message_batches_lock: + if reference: + batch = self._message_batches.get(key) + # Validate the live owner, so an abandoned request cannot match a + # subsequently allocated list at the same address. + if batch is not None and _get(batch.owner(), "messages") is messages: + cubes = _cube_ids(subject) + # 一个 add 可向多个 cube 提取; 等各 cube 都消费完关联后才移除, 避免后续 cube 丢失输入身份. + batch.pending_cubes.difference_update(cubes) + if not cubes or not batch.pending_cubes: + del self._message_batches[key] + return batch.identity + self._message_batches.pop(key, None) + # 每次新 add 使用新 UUID, 即使调用方重复使用同一个 messages 列表也不会复用上次节点. + # 引用未命中时也生成独立身份, 不把来源不明的提取结果误接到其他请求. + identity = f"message_batch:{key[0]}:{uuid4().hex}" + if not reference: + owner: Callable[[], Any] + try: + # 弱引用允许已丢弃请求被回收; 查询时还会验证其 messages 就是当前对象, 防止地址复用误配. + owner = weakref.ref(subject) + except TypeError: + # Plain dict/namespace callers cannot be weakly referenced. + # Their entries still have the same bounded, per-add lifetime. + def owner() -> Any: + return subject + + cubes = set(_cube_ids(subject)) or {str(_get(subject, "user_id", "unknown"))} + self._message_batches[key] = _MessageBatch(owner, identity, cubes) + self._message_batches.move_to_end(key) + # 异常退出或缺少后续 Hook 时也受数量上限约束; 最旧的待处理关联会被淘汰. + while len(self._message_batches) > self._max_pending_adds: + self._message_batches.popitem(last=False) + return identity + + # 兼容 Reader 的单组包装 [messages], 解包后尽可能保留原消息对象以关联 add 节点. + @staticmethod + def _messages_from_scene_data(scene_data: Any) -> Any: + if isinstance(scene_data, list | tuple) and len(scene_data) == 1: + messages = scene_data[0] + if isinstance(messages, list | tuple | str): + return messages + return scene_data + + # 消息批次在 add.before 创建, 在 MemReader 回调中以 identity_only 引用同一节点. + def _message_value( + self, + subject: Any, + messages: Any, + *, + identity_only: bool, + metadata: dict[str, Any] | None = None, + ) -> TraceValue: + return self._value( + name="messages", + value=messages, + identity=self._message_identity(subject, messages, reference=identity_only), + category="message_batch", + class_name="message_batch", + identity_only=identity_only, + comment=("Message batch received at MemOS stage add.before."), + metadata=metadata, + ) + + # 提取结果属于当前 trace 和 cube; 身份带 extracted_memory 前缀, 让后续落库转换可见. + def _extracted_memory_value( + self, + subject: Any, + memory: Any, + *, + cube_id: str | None, + memory_id: str, + identity_only: bool, + metadata: dict[str, Any] | None = None, + ) -> TraceValue: + trace_id = _current_trace_id(subject) + return self._value( + name="extracted_memory", + value=_memory_text(memory), + identity=f"extracted_memory:{trace_id}:{cube_id or 'unknown'}:{memory_id}", + category="extracted_memory", + class_name="memory", + identity_only=identity_only, + comment=("Memory candidate produced at MemOS stage mem_reader.extract.after."), + metadata=metadata, + ) + + # 按 cube + memory_id 建立持久化锚点; 返回 ID 无法匹配输入时只记录 ID 占位. + # 身份不含 trace, 使同一图中的同步写入和延迟调度可以接到同一节点; 不会跨图合并. + def _persisted_memory_value( + self, + memory: Any, + *, + cube_id: str | None, + memory_id: str, + stage: str = "text_memory.add.after", + ) -> TraceValue: + return self._value( + name="persisted_memory", + value=( + _memory_text(memory) + if memory is not None + else {"memory_id": memory_id, "cube_id": cube_id} + ), + identity=f"memory:{cube_id or 'unknown'}:{memory_id}", + category="persisted_memory", + class_name="memory", + identity_only=memory is None, + comment=( + f"Textual memory successfully persisted at MemOS stage {stage}." + if memory is not None + else f"Write returned this memory ID at {stage}; its source and content are unresolved." + ), + metadata={ + "memos_stage": stage, + "cube_id": cube_id, + "memory_id": memory_id, + }, + ) + + # 后台操作按数据库身份引用原记忆; 即使写入事件尚未到达, 也可先建立占位与连线. + def _persisted_memory_reference( + self, + *, + cube_id: str | None, + memory_id: str, + ) -> TraceValue: + return self._value( + name="persisted_memory", + value={"memory_id": memory_id, "cube_id": cube_id}, + identity=f"memory:{cube_id or 'unknown'}:{memory_id}", + category="persisted_memory", + class_name="memory", + identity_only=True, + comment="Reference to a memory unit previously persisted by MemOS.", + metadata={"cube_id": cube_id, "memory_id": memory_id}, + ) + + # 精细处理结果先使用 enhanced_memory 身份, 写库时引用它, 再连向新的持久化锚点. + def _enhanced_memory_value( + self, + subject: Any, + memory: Any, + *, + cube_id: str | None, + memory_id: str, + identity_only: bool, + ) -> TraceValue: + value = ( + {"memory_id": memory_id, "cube_id": cube_id} if identity_only else _memory_text(memory) + ) + return self._value( + name="enhanced_memory", + value=value, + identity=( + f"enhanced_memory:{_current_trace_id(subject)}:{cube_id or 'unknown'}:{memory_id}" + ), + category="enhanced_memory", + class_name="memory", + identity_only=identity_only, + comment="Memory produced by the MemRead scheduler's fine-transfer stage.", + metadata={"cube_id": cube_id, "memory_id": memory_id}, + ) + + # 归档、删除、刷新或失败等没有记忆输出的操作, 使用带操作身份的字符串结果作为终点. + def _scheduler_result_value( + self, + hook_context: HookContext, + *, + operation_name: str, + status: str, + ) -> TraceValue: + operation_id = _get(hook_context, "operation_id") or "unknown" + result = f"{operation_name} {'completed' if status == 'after' else 'failed'}" + return self._value( + name="scheduler_result", + value=result, + identity=f"scheduler_result:{operation_id}:{status}", + category="scheduler_result", + class_name="str", + identity_only=False, + comment=f"String result for MemRead scheduler operation {operation_name}.", + metadata={ + "operation_id": operation_id, + "operation_name": operation_name, + "status": status, + }, + ) + + # 把已落库输入与精细处理输出关联; 连线方式依赖 Hook 显式声明的结果分组契约. + def _fine_transfer_values( + self, + subject: Any, + operation_input: Mapping[str, Any], + result: Any, + *, + cube_id: str | None, + ) -> tuple[list[TraceValue], list[TraceValue], list[TraceLink]]: + inputs = self._scheduler_persisted_inputs( + "fine_transfer_simple_mem", operation_input, cube_id=cube_id + ) + outputs: list[TraceValue] = [] + links: list[TraceLink] = [] + grouping = operation_input.get("result_grouping", "unknown") + result_groups = result if isinstance(result, list | tuple) else [] + # per_input 的外层结果按输入顺序一一分组, 空组也必须保留, 否则会把后续结果接错来源. + if grouping == "per_input" and len(result_groups) == len(inputs): + groups = [ + (inputs[index : index + 1], group) for index, group in enumerate(result_groups) + ] + # batch 明确声明整批输入共同产生输出; 只有一个输入时也能确定所有输出的来源. + elif grouping == "batch" or len(inputs) == 1: + groups = [(inputs, result)] + else: + # 多输入而分组未知或数量不一致时保留输出, 使用空来源列表, 后续元数据标记 unresolved. + groups = [([], result)] + + for source_values, group in groups: + for memory, record_cube, memory_id in _walk_memory_records(group, cube_id): + target = self._enhanced_memory_value( + subject, + memory, + cube_id=record_cube or cube_id, + memory_id=memory_id, + identity_only=False, + ) + outputs.append(target) + links.extend( + TraceLink( + source=source, + target=target, + category="memory_refinement", + comment="Refine persisted memory in the MemRead scheduler.", + ) + for source in source_values + ) + return inputs, outputs, links + + # 同步写入与增强写入共用配对规则: 用返回 ID 匹配唯一输入, 不按返回顺序猜测来源. + def _persistence_values( + self, + subject: Any, + memories: Any, + result: Any, + *, + cube_id: str | None, + enhanced: bool = False, + ) -> tuple[list[TraceValue], list[TraceValue], list[TraceLink]]: + records = list(_walk_memory_records(memories, cube_id)) + result_ids = list(result) if isinstance(result, list | tuple) else [] + inputs: list[TraceValue] = [] + outputs: list[TraceValue] = [] + links: list[TraceLink] = [] + # 索引保留每个 ID 对应的所有输入位置, 用于识别重复 ID 引起的歧义. + candidates: dict[str, list[int]] = defaultdict(list) + source_value = self._enhanced_memory_value if enhanced else self._extracted_memory_value + for source_index, (memory, record_cube, candidate_id) in enumerate(records): + resolved_cube = record_cube or cube_id + source = source_value( + subject, + memory, + cube_id=resolved_cube, + memory_id=candidate_id, + identity_only=True, + ) + candidates[candidate_id].append(source_index) + inputs.append(source) + + stage = ( + "scheduler.mem_read.add_enhanced_memories.after" + if enhanced + else "text_memory.add.after" + ) + for index, result_id in enumerate(result_ids): + memory_id = str(result_id) + matching = candidates.get(memory_id, []) + # 返回 ID 可能乱序、跳过部分输入或没有已知来源; 只有唯一匹配才继承正文并建立连线. + source_index = matching[0] if len(matching) == 1 else None + memory, record_cube, _ = ( + records[source_index] if source_index is not None else (None, cube_id, memory_id) + ) + target = self._persisted_memory_value( + memory, + cube_id=record_cube or cube_id, + memory_id=memory_id, + stage=stage, + ) + outputs.append(target) + if source_index is not None: + links.append( + TraceLink( + source=inputs[source_index], + target=target, + category="memory_persistence", + comment=f"Persist the memory with the matching ID at {stage}.", + metadata={"pair_index": index, "input_index": source_index}, + ) + ) + return inputs, outputs, links + + # 来源关系完整为 resolved, 部分可确认是 partial, 有输出但无连线是 unresolved. + # 没有输出也没有连线时视为 resolved, 因为不存在需要解释来源的输出. + @staticmethod + def _persistence_lineage(outputs: list[TraceValue], links: list[TraceLink]) -> str: + if len(links) == len(outputs): + return "resolved" + return "partial" if links else "unresolved" + + # 精细处理从完整 memories 提取 ID; 其他操作从 memory_ids 引用持久化节点. + def _scheduler_persisted_inputs( + self, + operation_name: str, + operation_input: Mapping[str, Any], + *, + cube_id: str | None, + ) -> list[TraceValue]: + if operation_name == "fine_transfer_simple_mem": + records = _walk_memory_records(operation_input.get("memories"), cube_id) + return [ + self._persisted_memory_reference( + cube_id=record_cube or cube_id, + memory_id=memory_id, + ) + for _memory, record_cube, memory_id in records + ] + memory_ids = operation_input.get("memory_ids") or [] + return [ + self._persisted_memory_reference(cube_id=cube_id, memory_id=str(memory_id)) + for memory_id in memory_ids + ] + + # MemRead 成功/失败事件的共同入口; 根据操作类型构造输入输出与数据依赖. + # 后台任务的执行先后不自动产生边, 只有实际消费的记忆才连向输出或状态节点. + def _on_scheduler_memory_operation( + self, + *, + hook_context: HookContext, + operation: Any, + target: Any, + operation_input: Any, + result: Any | None = None, + error: BaseException | None = None, + status: str, + ) -> None: + source = str(_get(hook_context, "source", "")) + # 通用调度 Hook 也会承载其他 handler 的操作; 本插件这里只解释 MemRead 的业务契约. + if not source.startswith(_MEM_READ_SOURCE_PREFIX): + return + + operation_name = source.removeprefix(_MEM_READ_SOURCE_PREFIX) + if not isinstance(operation_input, Mapping): + operation_input = {} + subject = self._hook_subject(hook_context, user_name=operation_input.get("user_name")) + cube_id = _single_cube_id(subject) + + inputs: list[TraceValue] = [] + outputs: list[TraceValue] = [] + links: list[TraceLink] = [] + if status == "after" and operation_name == "fine_transfer_simple_mem": + inputs, outputs, links = self._fine_transfer_values( + subject, + operation_input, + result, + cube_id=cube_id, + ) + elif operation_name == "add_enhanced_memories": + inputs, outputs, links = self._persistence_values( + subject, + operation_input.get("memories"), + result if status == "after" else (), + cube_id=cube_id, + enhanced=True, + ) + else: + inputs = self._scheduler_persisted_inputs( + operation_name, + operation_input, + cube_id=cube_id, + ) + + # 归档/删除/刷新或失败时通常没有记忆输出; 用状态节点承接每条实际输入, 保持操作可见. + if not outputs: + if status == "failed": + category = "scheduler_operation_failed" + comment = "The MemRead scheduler operation failed while processing this memory." + elif operation_name in { + "archive_merged_memories", + "remove_memories", + "remove_source_memories", + }: + category = ( + "memory_archival" + if operation_name == "archive_merged_memories" + else "memory_deletion" + ) + comment = f"Apply {operation_name} to a persisted memory unit." + else: + category = "scheduler_operation_result" + comment = f"Produce the result of {operation_name} from its memory input." + result_value = self._scheduler_result_value( + hook_context, + operation_name=operation_name, + status=status, + ) + outputs.append(result_value) + links.extend( + TraceLink( + source=value, + target=result_value, + category=category, + comment=comment, + ) + for value in inputs + ) + + metadata = { + "memos_stage": f"{source}.{status}", + "handler": "mem_read", + "operation_name": operation_name, + "operation": operation, + "target": target, + "status": status, + } + if operation_name == "fine_transfer_simple_mem": + metadata["result_grouping"] = operation_input.get("result_grouping", "unknown") + if status == "after" and any(value.category == "enhanced_memory" for value in outputs): + metadata["lineage_status"] = "resolved" if links else "unresolved" + elif operation_name == "add_enhanced_memories" and status == "after": + metadata["lineage_status"] = self._persistence_lineage( + [value for value in outputs if value.category == "persisted_memory"], + [link for link in links if link.category == "memory_persistence"], + ) + if status == "failed": + # 异常消息可能包含业务正文或后端细节, 图中只记录异常类名, 不保存错误对象或消息. + metadata["error_type"] = type(error).__name__ + self._emit( + self._event( + subject, + operation=f"memos.{source}.{status}", + category="scheduler_memory_operation", + inputs=self._unique_values(inputs), + outputs=self._unique_values(outputs), + links=tuple(links), + metadata=metadata, + ) + ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py new file mode 100644 index 000000000..a44d33e81 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py @@ -0,0 +1,197 @@ +"""添加与搜索共用的上下文解析、快照和事件构造。 + +这里只处理追踪数据, 不保存业务链的关联状态, 也不执行数据库操作。 +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue +from memos_smartcomment.serialization import to_jsonable + +from memos.log import get_logger + + +logger = get_logger(__name__) +# 哨兵区分没有 trace_id 字段与显式 trace_id=None, 后者不能借用其他任务的线程上下文。 +_MISSING_TRACE = object() + + +# 兼容字典与对象属性, 让回调可处理 MemOS 模型、HookContext 以及测试替身. +def _get(value: Any, name: str, default: Any | None = None) -> Any: + if isinstance(value, Mapping): + return value.get(name, default) + return getattr(value, name, default) + + +# 统一正文入口: 优先 memory 字段, 仅在其为 None 时退回 text 字段. +def _memory_text(memory: Any) -> Any: + text = _get(memory, "memory") + return text if text is not None else _get(memory, "text") + + +# 决定事件归属哪张图: 显式 trace 优先, 缺少字段时才查线程上下文, 最后按用户/会话兜底. +def _current_trace_id(subject: Any | None = None) -> str: + trace_id = _get(subject, "trace_id", _MISSING_TRACE) + if trace_id is not _MISSING_TRACE and trace_id and trace_id != "trace-id": + return str(trace_id) + + # 只有调用方未提供 trace 字段时才读取环境上下文; 避免调度批次中串用其他消息的 trace. + if trace_id is _MISSING_TRACE: + try: + from memos.context.context import get_current_trace_id + + trace_id = get_current_trace_id() + if trace_id and trace_id != "trace-id": + return str(trace_id) + except Exception: + logger.debug("MemOS request context is unavailable", exc_info=True) + + # 无 trace 时优先使用 MemReader 和 add 共有的 user/session; task_id 仅在无 user 时兜底. + user_id = _get(subject, "user_id") + if user_id is None: + if task_id := _get(subject, "task_id"): + return f"task:{task_id}" + user_id = "unknown" + session_id = _get(subject, "session_id") or "default_session" + return f"standalone:{user_id}:{session_id}" + + +# 兼容添加、搜索和 HookContext 的不同 cube 字段名; 去重时保留原顺序. +def _cube_ids(subject: Any) -> tuple[str, ...]: + raw = ( + _get(subject, "writable_cube_ids") + or _get(subject, "readable_cube_ids") + or _get(subject, "mem_cube_ids") + or _get(subject, "cube_ids") + ) + if raw: + return tuple(dict.fromkeys(str(item) for item in raw)) + cube_id = _get(subject, "mem_cube_id") or _get(subject, "cube_id") + return (str(cube_id),) if cube_id else () + + +class BaseHandler: + """Build detached trace events and submit them without changing business results.""" + + def __init__( + self, + submit: Callable[[TraceEvent], Any], + *, + max_value_chars: int = 20_000, + ) -> None: + self._submit = submit + self._max_value_chars = max_value_chars + + # 在当前 Hook 线程同步复制并脱敏, 防止排队期间原始业务对象被其他线程修改. + def _snapshot(self, value: Any) -> Any: + return to_jsonable(value, max_value_chars=self._max_value_chars) + + # 统一节点构造入口: 正文和元数据都经过快照处理, 不把业务对象引用交给写入线程. + def _value( + self, + *, + name: str, + value: Any, + identity: str, + category: str, + class_name: str, + identity_only: bool, + comment: str, + metadata: dict[str, Any] | None = None, + ) -> TraceValue: + return TraceValue( + name=name, + value=self._snapshot(value), + identity=identity, + category=category, + class_name=class_name, + identity_only=identity_only, + comment=comment, + metadata=self._snapshot(metadata or {}), + ) + + # 统一补齐 trace、用户、会话、cube 和 operation_id 等关联信息; 节点与连线由调用方提供. + def _event( + self, + subject: Any, + *, + operation: str, + category: str, + inputs: tuple[TraceValue, ...] = (), + outputs: tuple[TraceValue, ...] = (), + links: tuple[TraceLink, ...] = (), + metadata: dict[str, Any] | None = None, + ) -> TraceEvent: + event_metadata = dict(metadata or {}) + if (operation_id := _get(subject, "operation_id")) is not None: + event_metadata.setdefault("operation_id", operation_id) + return TraceEvent( + operation=operation, + category=category, + trace_id=_current_trace_id(subject), + session_id=_get(subject, "session_id"), + task_id=_get(subject, "task_id"), + user_id=_get(subject, "user_id"), + cube_ids=_cube_ids(subject), + inputs=inputs, + outputs=outputs, + links=links, + metadata=self._snapshot(event_metadata), + ) + + # 追踪提交失败只记录日志, 不让队列出口的异常改变原业务调用结果. + def _emit(self, event: TraceEvent) -> None: + try: + self._submit(event) + except Exception: + logger.exception("Failed to enqueue smartcomment event: %s", event.operation) + + # 从 HookContext 提取关联字段, 不把上下文对象本身画成节点. + # 这里的 user_name 是记忆存储使用的 cube 标识兜底, 与 user_id 是不同概念. + @staticmethod + def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, Any]: + subject = { + key: value + for key in ( + "trace_id", + "task_id", + "user_id", + "session_id", + "operation_id", + ) + if (value := _get(context, key)) is not None + } + # 保留显式 None trace 的语义, 让 _current_trace_id 不误用工作线程残留的 trace. + if context is not None: + subject["trace_id"] = _get(context, "trace_id") + cube_ids = _get(context, "cube_ids") + if cube_ids: + subject["mem_cube_ids"] = cube_ids + if user_name is not None: + subject.setdefault("cube_id", user_name) + return subject + + # 没有成功业务输出时, 用 failed 字符串作为图中终点; 异常对象不作为节点保存. + def _operation_status_value(self, subject: Any, *, stage: str) -> TraceValue: + operation_id = _get(subject, "operation_id") or _current_trace_id(subject) + return self._value( + name="operation_status", + value="failed", + identity=f"operation_status:{operation_id}:{stage}", + category="operation_status", + class_name="str", + identity_only=False, + comment=f"MemOS operation failed at stage {stage}.", + metadata={"memos_stage": stage, "status": "failed"}, + ) + + # 按业务 identity 合并节点描述; 同身份保留最后一份值, 列表位置沿用首次出现顺序. + @staticmethod + def _unique_values(values: list[TraceValue]) -> tuple[TraceValue, ...]: + d = {} + for value in values: + d[value.identity] = value + return tuple(d.values()) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py new file mode 100644 index 000000000..4101fc638 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -0,0 +1,537 @@ +"""Search API 各阶段的候选快照、排序和过滤关系。 + +查询 -> raw -> threshold -> dedup -> rerank。 +先列出 Hook 回调, 再列出上下文解析、候选节点构造和跨阶段配对函数。 +""" + +from __future__ import annotations + +import hashlib +import json + +from collections import defaultdict, deque +from collections.abc import Callable, Mapping +from dataclasses import replace +from threading import Lock +from typing import TYPE_CHECKING, Any + +from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue +from memos_smartcomment.handlers.base import ( + BaseHandler, + _cube_ids, + _current_trace_id, + _get, + _memory_text, +) + + +if TYPE_CHECKING: + from memos.plugins.hook_context import HookContext + + +_SEARCH_LINK_CATEGORIES = { + "threshold": "memory_threshold_passed", + "dedup": "memory_deduplicated", + "rerank": "memory_reranked", +} + + +class SearchHandler(BaseHandler): + """Capture Search API stages with operation-scoped candidate correlation.""" + + def __init__( + self, + submit: Callable[[TraceEvent], Any], + *, + max_value_chars: int = 20_000, + ) -> None: + super().__init__(submit, max_value_chars=max_value_chars) + self._search_stage_nodes: dict[tuple[str, str, str], tuple[TraceValue, ...]] = {} + self._search_stage_items_lock = Lock() + + def clear(self) -> None: + """Release cached search-stage nodes at shutdown.""" + with self._search_stage_items_lock: + self._search_stage_nodes.clear() + + # 记录搜索的根输入 query; 请求包装对象不进入图. + def on_search_before( + self, + *, + request: Any, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + if request is None: + return + subject = self._search_subject(hook_context, request) + query = self._search_query_value(subject, request, identity_only=False) + self._emit( + self._event( + subject, + operation="memos.search.request", + category="search_request", + outputs=(query,), + metadata={"memos_stage": "search.before"}, + ) + ) + + # 召回结果作为 raw 阶段; 每个候选都由查询节点指向. + def on_search_memory_results( + self, + *, + search_req: Any, + results: Any, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + self._on_search_results( + request=search_req, + results=results, + hook_context=hook_context, + operation="memos.search.retrieve", + category="memory_retrieval", + stage="raw", + previous_stage=None, + metadata={ + "memos_stage": "search.memory_results", + "top_k": _get(search_req, "top_k"), + }, + ) + + # 记录 MemOS 阈值过滤后的结果和所用阈值; 此回调本身不执行过滤. + def on_search_results_after_threshold( + self, + *, + search_req: Any, + results: Any, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + threshold = _get(search_req, "relativity", 0) or 0 + self._on_search_results( + request=search_req, + results=results, + hook_context=hook_context, + operation="memos.search.threshold_filter", + category="memory_filtering", + stage="threshold", + previous_stage="raw", + metadata={ + "memos_stage": "search.results.after_threshold", + "threshold": threshold, + "score_field": "metadata.relativity", + "applied": threshold > 0, + }, + ) + + # 记录去重后的候选; sim/mmr 表示启用去重, 关闭时仍保留该阶段节点. + def on_search_results_after_dedup( + self, + *, + search_req: Any, + results: Any, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + method = _get(search_req, "dedup", "no") or "no" + self._on_search_results( + request=search_req, + results=results, + hook_context=hook_context, + operation="memos.search.deduplicate", + category="memory_deduplication", + stage="dedup", + previous_stage="threshold", + metadata={ + "memos_stage": "search.results.after_dedup", + "method": str(method), + "applied": method in {"sim", "mmr"}, + "top_k": _get(search_req, "top_k"), + }, + ) + + # 记录 API 当前按 metadata.relativity 降序排序的结果, 插件不重新计算分数. + def on_search_results_after_rerank( + self, + *, + search_req: Any, + results: Any, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + self._on_search_results( + request=search_req, + results=results, + hook_context=hook_context, + operation="memos.search.rerank", + category="memory_reranking", + stage="rerank", + previous_stage="dedup", + metadata={ + "memos_stage": "search.results.after_rerank", + "algorithm": "relativity_sort", + "order_by": "metadata.relativity", + "direction": "descending", + "top_k": _get(search_req, "top_k"), + }, + ) + + # 后处理失败时连接 query 与失败状态节点, 仅记录异常类型, 随后释放本次搜索缓存. + def on_search_post_process_failed( + self, + *, + handler: Any, + search_req: Any, + error: BaseException, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + stage = "search.post_process.failed" + subject = self._search_subject(hook_context, search_req) + query = self._search_query_value(subject, search_req, identity_only=True) + status = self._operation_status_value(subject, stage=stage) + self._emit( + self._event( + subject, + operation=f"memos.{stage}", + category="search_failure", + inputs=(query,), + outputs=(status,), + links=( + TraceLink( + source=query, + target=status, + category="operation_failed", + comment=f"Search processing failed at MemOS stage {stage}.", + ), + ), + metadata={ + "memos_stage": stage, + "status": "failed", + "error_type": error.__class__.__name__, + "handler": handler.__class__.__name__, + }, + ) + ) + self._clear_search_stages(subject) + + # 搜索使用 Hook 的关联信息, 再用请求字段补充空缺及可读 cube 列表. + def _search_subject( + self, + hook_context: HookContext | None, + request: Any, + ) -> dict[str, Any]: + subject = self._hook_subject(hook_context) + for key in ("trace_id", "user_id", "session_id", "task_id"): + if subject.get(key) is None and (value := _get(request, key)) is not None: + subject[key] = value + if cube_ids := _cube_ids(request): + subject["mem_cube_ids"] = cube_ids + return subject + + # 一条 trace 可能发生多次搜索; 优先用 operation_id 区分每次搜索, 缺失时才退回 trace. + @staticmethod + def _search_scope_id(subject: Any) -> str: + """Identify one search operation within a possibly shared trace.""" + operation_id = _get(subject, "operation_id") + return str(operation_id) if operation_id is not None else _current_trace_id(subject) + + # 以 trace + 搜索操作 + 阶段定位缓存, 避免同一 trace 内的多次搜索互相接错节点. + def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: + return ( + _current_trace_id(subject), + self._search_scope_id(subject), + stage, + ) + + # search.before 创建查询节点, 原始召回阶段按同一搜索身份引用它. + def _search_query_value( + self, + subject: Any, + request: Any, + *, + identity_only: bool, + ) -> TraceValue: + search_scope_id = self._search_scope_id(subject) + return self._value( + name="query", + value=_get(request, "query", ""), + identity=f"search_query:{search_scope_id}", + category="search_query", + class_name="str", + identity_only=identity_only, + comment="User Query submitted to the MemOS Search API.", + metadata={"memos_stage": "search.before"}, + ) + + # 将 result_type -> cube 分桶 -> memories 的结构展平, 同时保留桶内名次和遍历位置. + @staticmethod + def _search_result_items(results: Any) -> list[dict[str, Any]]: + if not isinstance(results, Mapping): + return [] + + items: list[dict[str, Any]] = [] + global_position = 0 + for result_type, buckets in results.items(): + if not isinstance(buckets, list | tuple): + continue + for bucket_index, bucket in enumerate(buckets): + memories = _get(bucket, "memories") + if not isinstance(memories, list | tuple): + continue + cube_id = _get(bucket, "cube_id") + # rank 从 1 开始且仅在本结果类型的当前 cube 桶内有效; global_position 是展平遍历顺序. + rank_scope = f"{result_type}:{cube_id or bucket_index}" + for rank, memory in enumerate(memories, start=1): + global_position += 1 + metadata = _get(memory, "metadata", {}) + # 按 API 常用字段优先级取分数; 0 也是有效分数, 所以后续用 is None 判断缺失. + score = _get(metadata, "relativity") + if score is None: + score = _get(metadata, "score") + if score is None: + score = _get(memory, "score") + items.append( + { + "result_type": str(result_type), + "bucket_index": bucket_index, + "cube_id": str(cube_id) if cube_id is not None else None, + "memory_id": _get(memory, "id") or _get(memory, "memory_id"), + "memory": _memory_text(memory), + "rank": rank, + "rank_scope": rank_scope, + "global_position": global_position, + "score": score, + } + ) + return items + + # 每一阶段都创建独立的搜索候选节点, 因而可展示分数与排名变化. + # 搜索节点使用 search_memory 身份, 不复用添加链路中的持久化记忆锚点. + def _search_memory_value( + self, + subject: Any, + *, + stage: str, + item: Mapping[str, Any], + ) -> TraceValue: + search_scope_id = self._search_scope_id(subject) + result_type = str(item.get("result_type", "memory")) + cube_id = item.get("cube_id") + bucket_index = item.get("bucket_index", 0) + rank = item.get("rank") + rank_scope = item.get("rank_scope") + global_position = item.get("global_position") + memory_id = item.get("memory_id") + # 节点身份还包含 stage 和 global_position, 因而同 ID 在不同阶段或重复出现时仍可单独展示. + occurrence_id = memory_id if memory_id is not None else "anonymous" + cube_scope = cube_id if cube_id is not None else f"bucket-{bucket_index}" + score = item.get("score") + comment_parts = [ + f"Search result memory at stage={stage}", + f"rank={rank}", + f"rank_scope={rank_scope}", + f"global_position={global_position}", + ] + if score is not None: + comment_parts.append(f"score={score}") + + node = self._value( + name=f"{stage}_{result_type}_{rank}", + value={ + "memory_id": memory_id, + "memory": item.get("memory"), + "result_type": result_type, + "cube_id": cube_id, + "score": score, + }, + identity=( + f"search_memory:{search_scope_id}:{stage}:{result_type}:{cube_scope}:" + f"{occurrence_id}:{global_position}" + ), + category="search_memory", + class_name="search_memory", + identity_only=False, + comment="; ".join(comment_parts) + ".", + metadata={ + "stage": stage, + "result_type": result_type, + "cube_id": cube_id, + "memory_id": memory_id, + "rank": rank, + "rank_scope": rank_scope, + "global_position": global_position, + "score": score, + }, + ) + + # 使用原始 item 计算指纹, 不用已截断的 node.value; 长正文截断不会影响跨阶段匹配. + return replace(node, match_key=self._search_memory_match_key(item)) + + # 跨阶段匹配使用结果类型 + cube + 记忆 ID; 没有 ID 时退回未截断正文的指纹. + # 不把排名、分数或阶段纳入匹配键, 使同一候选在重排后仍能找到来源. + @staticmethod + def _search_memory_match_key(value: Mapping[str, Any]) -> str: + memory_id = value.get("memory_id") + identity = ( + ("id", str(memory_id)) if memory_id is not None else ("text", value.get("memory")) + ) + cube_id = value.get("cube_id") + cube_scope = cube_id if cube_id is not None else f"bucket-{value.get('bucket_index', 0)}" + raw = json.dumps([value.get("result_type"), cube_scope, identity], ensure_ascii=False) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + # 为每个匹配键建立先进先出的候选池, 逐个消费重复项, 避免一个来源匹配多个目标. + # 返回与当前项等长的来源列表 (可能含 None), 以及上一阶段未被匹配的移除项. + def _match_search_nodes( + self, + previous_nodes: tuple[TraceValue, ...], + current_items: tuple[dict[str, Any], ...], + ) -> tuple[tuple[TraceValue | None, ...], tuple[TraceValue, ...]]: + pools: dict[str, deque[TraceValue]] = defaultdict(deque) + for node in previous_nodes: + pools[node.match_key or self._search_memory_match_key(node.value)].append(node) + + matched: list[TraceValue | None] = [] + matched_identities: set[str] = set() + for item in current_items: + candidates = pools[self._search_memory_match_key(item)] + node = candidates.popleft() if candidates else None + matched.append(node) + if node is not None: + matched_identities.add(node.identity) + + removed = tuple(node for node in previous_nodes if node.identity not in matched_identities) + return tuple(matched), removed + + # 每个搜索阶段共用一个 filtered 状态节点, 承接这一阶段被移除候选的边. + def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: + search_scope_id = self._search_scope_id(subject) + return self._value( + name=f"{stage}_filtered", + value="filtered", + identity=f"search_filter_result:{search_scope_id}:{stage}", + category="search_filter_result", + class_name="str", + identity_only=False, + comment=f"Memory was removed from Search API results at stage={stage}.", + metadata={"stage": stage, "status": "filtered"}, + ) + + # 搜索结果阶段共用的处理流程: 展平结果、读取前阶段、配对连线、保存当前阶段并发出事件. + def _on_search_results( + self, + *, + request: Any, + results: Any, + hook_context: HookContext | None, + operation: str, + category: str, + stage: str, + previous_stage: str | None, + metadata: dict[str, Any], + ) -> None: + if request is None: + return + subject = self._search_subject(hook_context, request) + current_items = tuple(self._search_result_items(results)) + with self._search_stage_items_lock: + previous_nodes = ( + self._search_stage_nodes.get( + self._search_stage_key(subject, previous_stage), + (), + ) + if previous_stage is not None + else () + ) + + inputs: tuple[TraceValue, ...] + outputs: tuple[TraceValue, ...] + links: tuple[TraceLink, ...] + removed_nodes: tuple[TraceValue, ...] = () + current_nodes = tuple( + self._search_memory_value(subject, stage=stage, item=item) for item in current_items + ) + + # 首个结果阶段由 query 指向召回候选; 后续阶段必须按候选身份逐一配对. + if previous_stage is None: + query = self._search_query_value(subject, request, identity_only=True) + inputs = (query,) + outputs = current_nodes + links = tuple( + TraceLink( + source=query, + target=node, + category="memory_retrieval", + comment="Search query retrieved this memory candidate.", + ) + for node in current_nodes + ) + else: + matched_nodes, removed_nodes = self._match_search_nodes( + previous_nodes, + current_items, + ) + + paired_links: list[TraceLink] = [] + for source, candidate in zip(matched_nodes, current_nodes, strict=True): + if source is not None: + paired_links.append( + TraceLink( + source=replace(source, identity_only=True), + target=candidate, + category=_SEARCH_LINK_CATEGORIES[stage], + comment=f"Carry this Search candidate into stage={stage}.", + ) + ) + # 未找到来源的新候选也保留节点, 但不补造来源边; 未被匹配的旧候选则连向 filtered. + outputs = list(current_nodes) + if removed_nodes: + filtered = self._search_filter_result_value(subject, stage=stage) + outputs.append(filtered) + paired_links.extend( + TraceLink( + source=replace(node, identity_only=True), + target=filtered, + category="memory_filtered", + comment=f"This memory was filtered out at stage={stage}.", + ) + for node in removed_nodes + ) + links = tuple(paired_links) + inputs = self._unique_values([link.source for link in links]) + outputs = tuple(outputs) + + with self._search_stage_items_lock: + self._search_stage_nodes[self._search_stage_key(subject, stage)] = current_nodes + + event_metadata = { + **metadata, + "link_strategy": "explicit_pairwise", + "filtered_count": len(removed_nodes), + } + self._emit( + self._event( + subject, + operation=operation, + category=category, + inputs=inputs, + outputs=outputs, + links=links, + metadata=event_metadata, + ) + ) + # rerank 是观测链路的正常终点; 异常终点由 on_search_post_process_failed 清理. + if stage == "rerank": + self._clear_search_stages(subject) + + # 只清理当前搜索 operation 的缓存, 保留同一 trace 中其他搜索的中间状态. + def _clear_search_stages(self, subject: Any) -> None: + trace_id = _current_trace_id(subject) + search_scope_id = self._search_scope_id(subject) + with self._search_stage_items_lock: + for key in [ + key for key in self._search_stage_nodes if key[:2] == (trace_id, search_scope_id) + ]: + del self._search_stage_nodes[key] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py new file mode 100644 index 000000000..cd3556915 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py @@ -0,0 +1,96 @@ +# 集成入口: MemOS 从 memos.plugins entry point 发现此插件, 启用后调用 on_load. +# 调用链: MemOS Hook -> HookHandlers -> AsyncTraceAdapter -> SmartCommentRecorder -> JSON. +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from memos_smartcomment.adapter import AsyncTraceAdapter +from memos_smartcomment.config import SmartCommentSettings +from memos_smartcomment.handlers import HookHandlers + +from memos.log import get_logger +from memos.plugins.base import MemOSPlugin +from memos.plugins.hook_defs import H + + +logger = get_logger(__name__) +AdapterFactory = Callable[[SmartCommentSettings], AsyncTraceAdapter] + + +# 以观察者身份订阅现有 Hook; 回调只提交追踪事件, 不提供替换业务结果的返回值. +class SmartCommentPlugin(MemOSPlugin): + """Trace existing MemOS semantic Hooks without changing business results.""" + + name = "smartcomment" + version = "0.1.0" + description = "Trace MemOS memory flows with smartcomment" + # 安装不等于启用; 需要通过 MEMOS_ENABLED_PLUGINS=smartcomment 显式开启. + enabled_by_default = False + + # adapter_factory 是可注入的创建函数, 便于测试时替换真实后台写入器. + def __init__(self, adapter_factory: AdapterFactory | None = None) -> None: + self._adapter_factory = adapter_factory or AsyncTraceAdapter + self.adapter: AsyncTraceAdapter | None = None + self.handlers: HookHandlers | None = None + self.context: dict[str, Any] = {"shared": {}, "configs": {}} + + # 读取配置、启动写入线程, 再把业务边界 Hook 绑定到对应回调. + def on_load(self) -> None: + settings = SmartCommentSettings.from_env() + self.adapter = self._adapter_factory(settings) + # 回调在当前业务线程中生成快照; submit 只负责入队, 实际写图由后台线程执行. + self.handlers = HookHandlers( + self.adapter.submit, + max_value_chars=settings.max_value_chars, + ) + self.adapter.start() + + # 三组观测点: 添加/提取/持久化, MemRead 后台操作, Search 各个结果阶段. + # 成功与失败分别注册, 使失败状态也能与其输入记忆建立数据依赖. + for hook, callback in ( + (H.ADD_BEFORE, self.handlers.on_add_before), + (H.ADD_AFTER, self.handlers.on_add_after), + (H.MEM_READER_EXTRACT_AFTER, self.handlers.on_mem_reader_extract_after), + (H.MEM_READER_EXTRACT_FAILED, self.handlers.on_mem_reader_extract_failed), + (H.TEXT_MEMORY_ADD_AFTER, self.handlers.on_text_memory_add_after), + (H.TEXT_MEMORY_ADD_FAILED, self.handlers.on_text_memory_add_failed), + (H.SCHEDULER_MEMORY_OPERATION_AFTER, self.handlers.on_scheduler_memory_operation_after), + ( + H.SCHEDULER_MEMORY_OPERATION_FAILED, + self.handlers.on_scheduler_memory_operation_failed, + ), + (H.SEARCH_BEFORE, self.handlers.on_search_before), + (H.SEARCH_MEMORY_RESULTS, self.handlers.on_search_memory_results), + (H.SEARCH_RESULTS_AFTER_THRESHOLD, self.handlers.on_search_results_after_threshold), + (H.SEARCH_RESULTS_AFTER_DEDUP, self.handlers.on_search_results_after_dedup), + (H.SEARCH_RESULTS_AFTER_RERANK, self.handlers.on_search_results_after_rerank), + (H.SEARCH_POST_PROCESS_FAILED, self.handlers.on_search_post_process_failed), + ): + self.register_hook(hook, callback) + logger.info( + "smartcomment plugin loaded; output_dir=%s queue_size=%d", + settings.output_dir, + settings.queue_size, + ) + + # 接收插件框架提供的共享组件上下文, 供插件生命周期持有. + def init_components(self, context: dict[str, Any]) -> None: + self.context = context + + # 先释放 Hook 关联缓存, 再在等待预算内排空已接受事件, 最后记录处理统计. + def on_shutdown(self) -> None: + if self.handlers is not None: + self.handlers.clear() + if self.adapter is not None: + self.adapter.close() + stats = getattr(self.adapter, "stats", None) + if stats is not None: + logger.info( + "smartcomment plugin stopped; accepted=%d processed=%d dropped=%d failed=%d", + stats.accepted, + stats.processed, + stats.dropped, + stats.failed, + ) + self.context = {"shared": {}, "configs": {}} diff --git a/apps/memtrace-memos-integration/memos_smartcomment/recorder.py b/apps/memtrace-memos-integration/memos_smartcomment/recorder.py new file mode 100644 index 000000000..8aeb3188b --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/recorder.py @@ -0,0 +1,261 @@ +# SmartComment 桥接层: 每个 recorder 管理一张 trace 图, 把 TraceValue/TraceLink 写入运行时. +# handlers 决定业务节点与依赖, 此处负责节点身份、操作作用域、快照恢复和导出. +from __future__ import annotations + +import json + +from typing import TYPE_CHECKING, Any + +from memos.exceptions import MemOSError +from memos.log import get_logger + + +if TYPE_CHECKING: + from memos_smartcomment.config import SmartCommentSettings + from memos_smartcomment.events import TraceEvent, TraceValue + + +_NONE_FULL_NODE_ID = "COMMENT:NONE@1" +_NONE_SESSION_ID = "__none__" +_NONE_OPERATION_ID = "__none_op__" +logger = get_logger(__name__) + + +# 固定字典键顺序, 让相同结构的快照有稳定编码, 用于比较节点内容是否冲突. +def _encode(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +# 与 _encode 配对, 供 SmartComment 将节点字符串还原为 JSON 值. +def _decode(value: str) -> Any: + return json.loads(value) + + +class SmartCommentRecorder: + """Own one smartcomment graph and append normalized MemOS events to it.""" + + # 延迟导入 SmartComment 并创建图; trace、用户和项目身份取自首个事件及插件配置. + def __init__(self, settings: SmartCommentSettings, first_event: TraceEvent) -> None: + from smartcomment import ( + comment_graph, + comment_link, + comment_op_scope, + comment_session, + comment_variable, + ) + from smartcomment.runtime import ExecNetwork + + self._comment_graph = comment_graph + self._comment_link = comment_link + self._comment_op_scope = comment_op_scope + self._comment_session = comment_session + self._comment_variable = comment_variable + self._graph = ExecNetwork( + graph_id=first_event.trace_id, + user_id=first_event.user_id, + project_id=settings.project_id, + strict=settings.strict, + ) + + # 将插件节点描述转换为 comment_variable 的值与参数, 以业务 identity 控制节点复用. + @staticmethod + def _as_comment_item(value: TraceValue) -> tuple[Any, dict[str, Any]]: + options: dict[str, Any] = { + "id_strategy": lambda _value, identity=value.identity: identity, + "encoding_fn": _encode, + "decoding_fn": _decode, + "category": value.category, + "identity_only": value.identity_only, + "metadata": value.metadata, + } + if value.class_name is not None: + options["class_name"] = value.class_name + if value.comment is not None: + options["comment"] = value.comment + # 持久化记忆采用不可变锚点: 始终按身份复用, 不因再次观测到不同正文就生成新版本. + # snapshot_status 另外区分仅含 ID 的 reference 与有正文的 complete. + if value.category == "persisted_memory": + # A database memory ID is an immutable graph anchor, not a versioned view. + options["identity_only"] = True + options["metadata"] = { + **value.metadata, + "snapshot_status": "reference" if value.identity_only else "complete", + } + return value.value, options + + # 一次事件对应一个 SmartComment session 和一个 operation; 这不是 MemOS 对话会话. + # MemOS 的 session_id 作为元数据保留, 方便从执行图反查业务上下文. + def record(self, event: TraceEvent) -> None: + session_metadata = { + "memos_session_id": event.session_id, + "task_id": event.task_id, + "cube_ids": list(event.cube_ids), + "trace_event_id": event.event_id, + "trace_event_created_at": event.created_at, + } + operation_metadata = { + **event.metadata, + "trace_event_id": event.event_id, + "memos_trace_id": event.trace_id, + "memos_session_id": event.session_id, + "task_id": event.task_id, + "cube_ids": list(event.cube_ids), + } + nodes: dict[int, Any] = {} + completions: dict[str, TraceValue] = {} + + # 按本次事件中的 TraceValue 对象缓存注册结果; 跨事件复用由业务 identity 决定. + def register(value: TraceValue) -> Any: + key = id(value) + if key not in nodes: + raw_value, options = self._as_comment_item(value) + node = self._comment_variable(raw_value, to_runtime=True, **options) + nodes[key] = node + if ( + not value.identity_only + and getattr(node, "category", None) == "persisted_memory" + ): + if node.metadata.get("snapshot_status") == "reference": + completions.setdefault(node.full_node_id, value) + # 完整快照出现冲突时只告警, 保留第一次完整内容; strict 模式也遵循这条规则. + elif node.raw_value != _encode(value.value): + logger.warning( + "Ignoring a conflicting snapshot for immutable memory unit %s", + value.identity, + ) + return nodes[key] + + with ( + self._comment_graph(graph=self._graph), + self._comment_session( + session_id=f"event-{event.event_id}", + session_name=event.operation, + category=event.category, + comment=f"MemOS event for {event.operation}.", + metadata=session_metadata, + ), + self._comment_op_scope( + op_name=event.operation, + category=event.category, + comment=f"MemOS semantic operation: {event.operation}.", + metadata=operation_metadata, + ), + ): + for value in (*event.inputs, *event.outputs): + register(value) + # 只按显式 TraceLink 建边, 不做 inputs x outputs 全连接. + # 端点即使未单独列入 inputs/outputs 也会注册; link 元数据优先于 operation 元数据. + for link in event.links: + self._comment_link( + source=register(link.source), + target=register(link.target), + category=link.category, + comment=link.comment, + edge_metadata={**operation_metadata, **link.metadata}, + ) + # 离开图作用域后再通过导出/导入补全占位, 避免活动作用域仍引用被替换的运行时图. + if completions: + self._complete_memory_snapshots(completions) + + # 异步引用先到时只有 ID 占位; 完整写入快照到达后补一次正文, 保留节点 ID 和已有连线. + def _complete_memory_snapshots(self, completions: dict[str, TraceValue]) -> None: + """Fill an ID-only placeholder once, preserving its node ID and incident edges.""" + exported = self._graph.export_graph() + for node in exported["data"]["nodes"]: + if (value := completions.get(node["full_node_id"])) is not None: + node["value"] = _encode(value.value) + node["comment"] = value.comment + node["metadata"] = { + **node.get("metadata", {}), + **value.metadata, + "snapshot_status": "complete", + } + # Use the public graph round-trip API after exiting the active graph scope. + self.restore_graph(exported) + + # 恢复前验证 trace、用户与项目一致; 导入后继续沿用当前配置的严格校验开关. + def restore_graph(self, data: dict[str, Any]) -> None: + from smartcomment.runtime import ExecNetwork + + for field in ("graph_id", "user_id", "project_id"): + if data.get(field) != getattr(self._graph, field): + raise MemOSError(f"Snapshot {field} does not match the requested graph") + graph = ExecNetwork.import_graph(self._normalize_memory_units(data)) + graph.strict = self._graph.strict + self._graph = graph + + # 兼容旧版把同一持久化记忆拆成多版本的图: 以最早版本作锚点, 保留首个完整快照. + @staticmethod + def _normalize_memory_units(data: dict[str, Any]) -> dict[str, Any]: + """Reconnect legacy versions to the earliest anchor and first complete snapshot.""" + graph_data = data["data"] + anchors: dict[tuple[str | None, str], dict[str, Any]] = {} + aliases: dict[str, str] = {} + units = [node for node in graph_data["nodes"] if node["category"] == "persisted_memory"] + for node in sorted(units, key=lambda node: node["version"]): + metadata = dict(node.get("metadata", {})) + if "snapshot_status" not in metadata: + is_reference = "memory_id" in metadata and _decode(node["value"]) == { + "memory_id": metadata["memory_id"], + "cube_id": metadata.get("cube_id"), + } + metadata["snapshot_status"] = "reference" if is_reference else "complete" + # SmartComment 导出节点的 name 承载 id_strategy 产生的身份, 用它聚合同一记忆的版本. + key = (node.get("class_name"), node["name"]) + if key not in anchors: + anchors[key] = {**node, "metadata": metadata} + anchor = anchors[key] + aliases[node["full_node_id"]] = anchor["full_node_id"] + if ( + anchor["metadata"]["snapshot_status"] == "reference" + and metadata["snapshot_status"] == "complete" + ): + anchor.update(value=node["value"], comment=node.get("comment"), metadata=metadata) + + nodes = [] + for node in graph_data["nodes"]: + if node["category"] != "persisted_memory": + nodes.append(node) + elif aliases[node["full_node_id"]] == node["full_node_id"]: + nodes.append(anchors[(node.get("class_name"), node["name"])]) + return { + **data, + "data": { + **graph_data, + "nodes": nodes, + "edges": [ + { + **edge, + **{ + # 旧版本的入边和出边都重定向到锚点, 合并节点时仍保留原始数据依赖. + field: aliases.get(edge[field], edge[field]) + for field in ("source_full_node_id", "target_full_node_id") + }, + } + for edge in graph_data["edges"] + ], + }, + } + + # 导出面向 MemOS 的执行图, 移除 SmartComment 内部 NONE 哨兵及相关边、操作和会话. + def export_graph(self) -> dict[str, Any]: + exported = self._graph.export_graph() + data = exported["data"] + data["nodes"] = [ + node for node in data["nodes"] if node["full_node_id"] != _NONE_FULL_NODE_ID + ] + data["edges"] = [ + edge + for edge in data["edges"] + if edge["source_full_node_id"] != _NONE_FULL_NODE_ID + and edge["target_full_node_id"] != _NONE_FULL_NODE_ID + ] + data["operations"] = [ + operation + for operation in data["operations"] + if operation["op_id"] != _NONE_OPERATION_ID + ] + data["sessions"] = [ + session for session in data["sessions"] if session["session_id"] != _NONE_SESSION_ID + ] + return exported diff --git a/apps/memtrace-memos-integration/memos_smartcomment/serialization.py b/apps/memtrace-memos-integration/memos_smartcomment/serialization.py new file mode 100644 index 000000000..ec2bba2f7 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/serialization.py @@ -0,0 +1,153 @@ +# 快照层: 将业务对象复制为可序列化的基础值, 在事件入队前按字段脱敏并限制大小. +# 普通消息正文只截断, 不扫描其中任意位置的凭据; 脱敏依据是结构化字段名. +from __future__ import annotations + +import dataclasses +import json +import math + +from collections.abc import Mapping +from datetime import date, datetime +from enum import Enum +from pathlib import Path +from typing import Any + + +_SECRET_PARTS = ( + "api_key", + "apikey", + "access_key", + "password", + "secret", + "authorization", + "cookie", + "credential", + "token", +) +_VECTOR_KEYS = {"embedding", "embeddings", "vector", "vectors"} +_ARGUMENT_KEYS = {"argument", "arguments"} + + +# 统一字段大小写和连接符后做子串匹配; 含 token 等敏感词的字段会整体脱敏. +def _is_secret_key(key: str) -> bool: + normalized = key.lower().replace("-", "_") + return any(part in normalized for part in _SECRET_PARTS) + + +# 按字符数保留前缀并追加省略号; 限制作用于每个字符串, 不是整张图的总字节数. +def _truncate(value: str, max_value_chars: int) -> str: + if len(value) <= max_value_chars: + return value + return value[:max_value_chars] + "…" + + +# 递归处理字典、集合、dataclass、Pydantic 模型和普通对象. +# max_items 限制每层容器的输出项数; _seen 检测当前递归路径上的循环引用. +def to_jsonable( + value: Any, + *, + max_value_chars: int = 20_000, + max_items: int = 200, + _seen: set[int] | None = None, +) -> Any: + """Create a bounded JSON-compatible snapshot without secrets or vectors.""" + + if value is None or isinstance(value, bool | int): + return value + # JSON 不支持标准的 NaN/Infinity 数字表示, 因此把非有限浮点数转为字符串. + if isinstance(value, float): + return value if math.isfinite(value) else str(value) + if isinstance(value, str): + return _truncate(value, max_value_chars) + if isinstance(value, bytes): + return f"" + if isinstance(value, date | datetime | Path | Enum): + return str(getattr(value, "value", value)) + + seen = _seen if _seen is not None else set() + value_id = id(value) + if value_id in seen: + return "" + seen.add(value_id) + + try: + # 先把模型展开为字段映射, 再统一递归处理, 让嵌套字段也经过脱敏. + if dataclasses.is_dataclass(value) and not isinstance(value, type): + value = {field.name: getattr(value, field.name) for field in dataclasses.fields(value)} + elif hasattr(value, "model_dump"): + try: + value = value.model_dump(mode="json") + except TypeError: + value = value.model_dump() + + if isinstance(value, Mapping): + result: dict[str, Any] = {} + for index, (raw_key, item) in enumerate(value.items()): + if index >= max_items: + result[""] = f"{len(value) - max_items} more items" + break + key = str(raw_key) + if _is_secret_key(key): + result[key] = "" + # 向量只保留长度提示, 避免高维数组占用追踪队列和图文件. + elif key.lower() in _VECTOR_KEYS: + length = len(item) if hasattr(item, "__len__") else "unknown" + result[key] = f"" + # 工具调用的 arguments 可能是内嵌 JSON 字符串; 必须先解析和脱敏, 再截断. + # 无法解析时省略整段参数, 避免保留无法检查敏感字段的原文. + elif key.lower() in _ARGUMENT_KEYS and isinstance(item, str): + # OpenAI tool calls carry a JSON string, which must be decoded + # before key-based redaction. Incomplete arguments are not safe + # to retain because their secret fields cannot be inspected. + try: + arguments = json.loads(item) + except (ValueError, RecursionError): + arguments = None + if isinstance(arguments, dict | list): + sanitized = to_jsonable( + arguments, + max_value_chars=max_value_chars, + max_items=max_items, + _seen=seen, + ) + result[key] = _truncate( + json.dumps(sanitized, ensure_ascii=False), max_value_chars + ) + else: + result[key] = "" + else: + result[key] = to_jsonable( + item, + max_value_chars=max_value_chars, + max_items=max_items, + _seen=seen, + ) + return result + + if isinstance(value, list | tuple | set | frozenset): + items = list(value) + result = [ + to_jsonable( + item, + max_value_chars=max_value_chars, + max_items=max_items, + _seen=seen, + ) + for item in items[:max_items] + ] + if len(items) > max_items: + result.append(f"<{len(items) - max_items} more items>") + return result + + # 未知对象优先读取属性字典; 没有属性字典时使用截断后的 repr 作为兜底表示. + if hasattr(value, "__dict__"): + return to_jsonable( + vars(value), + max_value_chars=max_value_chars, + max_items=max_items, + _seen=seen, + ) + return _truncate(repr(value), max_value_chars) + finally: + # 退出当前分支就释放标记, 因此兄弟分支共享同一个对象不会被误判为循环. + seen.discard(value_id) diff --git a/apps/memtrace-memos-integration/pyproject.toml b/apps/memtrace-memos-integration/pyproject.toml new file mode 100644 index 000000000..cf0df0f4c --- /dev/null +++ b/apps/memtrace-memos-integration/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "memos-smartcomment-integration" +version = "0.1.0" +description = "Non-invasive smartcomment execution-graph tracing for MemOS" +readme = "README.md" +requires-python = ">=3.12" +license = {text = "Apache-2.0"} +dependencies = [ + "MemoryOS>=2.0.27", + "smartcomment>=0.1.2", +] + +[project.entry-points."memos.plugins"] +smartcomment = "memos_smartcomment.plugin:SmartCommentPlugin" + +[tool.setuptools.packages.find] +where = ["."] +include = ["memos_smartcomment*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +extend = "../../pyproject.toml" +target-version = "py312" + +[tool.ruff.lint] +extend-ignore = ["N999"] diff --git a/src/memos/mem_reader/simple_struct.py b/src/memos/mem_reader/simple_struct.py index 14521cf47..4db01d90f 100644 --- a/src/memos/mem_reader/simple_struct.py +++ b/src/memos/mem_reader/simple_struct.py @@ -815,7 +815,7 @@ def fine_transfer_simple_mem( if not input_memories: return [] - memory_list = [] + memory_list = [[] for _ in input_memories] if type == "chat": processing_func = self._process_transfer_chat_data @@ -826,15 +826,15 @@ def fine_transfer_simple_mem( # Process Q&A pairs concurrently with context propagation with ContextThreadPoolExecutor() as executor: - futures = [ - executor.submit(processing_func, scene_data_info, custom_tags, **kwargs) - for scene_data_info in input_memories - ] + futures = { + executor.submit(processing_func, scene_data_info, custom_tags, **kwargs): index + for index, scene_data_info in enumerate(input_memories) + } for future in concurrent.futures.as_completed(futures): try: res_memory = future.result() if res_memory is not None: - memory_list.append(res_memory) + memory_list[futures[future]] = res_memory except Exception as e: logger.error(f"Task failed with exception: {e}") logger.error(traceback.format_exc()) diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index a61744785..022872a00 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -182,7 +182,11 @@ def _process_memories_with_reader( operation="fine", target="textual_memory", operation_context=operation_context, - operation_input={"memories": memory_items, "type": "chat"}, + operation_input={ + "memories": memory_items, + "type": "chat", + "result_grouping": self._fine_transfer_result_grouping(mem_reader), + }, capture_failure=True, ) as observation: observation.result( @@ -574,3 +578,18 @@ def _try_submit_organize_task( ) except Exception as e: logger.error("Failed to enqueue MEM_ORGANIZE task: %s", e, exc_info=True) + + def _fine_transfer_result_grouping(self, mem_reader: Any) -> str: + """Describe known transfer implementations at the observation boundary.""" + + from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader + from memos.mem_reader.simple_struct import SimpleStructMemReader + + # Match the bound implementation so inherited methods work, while overrides + # with a different grouping contract remain unknown unless explicitly declared. + transfer = getattr(mem_reader.fine_transfer_simple_mem, "__func__", None) + if transfer is MultiModalStructMemReader.fine_transfer_simple_mem: + return "batch" + if transfer is SimpleStructMemReader.fine_transfer_simple_mem: + return "per_input" + return "unknown" diff --git a/tests/mem_reader/test_fine_transfer_order.py b/tests/mem_reader/test_fine_transfer_order.py new file mode 100644 index 000000000..2beacfbe2 --- /dev/null +++ b/tests/mem_reader/test_fine_transfer_order.py @@ -0,0 +1,44 @@ +"""Fine-transfer groups retain their input index when tasks finish out of order.""" + +from types import SimpleNamespace + +import pytest + +from memos.mem_reader.simple_struct import SimpleStructMemReader + + +@pytest.mark.parametrize("failed_result", [None, ValueError("synthetic failure")]) +def test_fine_transfer_keeps_failed_slots_and_input_order(monkeypatch, failed_result): + monkeypatch.setattr( + "memos.mem_reader.simple_struct.concurrent.futures.as_completed", + lambda futures: iter(reversed(list(futures))), + ) + + def process(source, custom_tags, **kwargs): + if source == "missing": + if isinstance(failed_result, Exception): + raise failed_result + return failed_result + return [source + "-enhanced"] + + reader = SimpleNamespace(_process_transfer_chat_data=process) + result = SimpleStructMemReader.fine_transfer_simple_mem( + reader, ["first", "missing", "last"], "chat" + ) + assert result == [["first-enhanced"], [], ["last-enhanced"]] + + +def test_multimodal_fine_transfer_returns_one_group_for_the_batch(): + from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader + + sources = ["first", "last"] + enhanced = ["combined-enhanced"] + + def process(memories, custom_tags, **kwargs): + assert memories is sources + return enhanced + + reader = SimpleNamespace(_process_transfer_multi_modal_data=process) + result = MultiModalStructMemReader.fine_transfer_simple_mem(reader, sources, "chat") + assert result == [enhanced] + assert result[0] is enhanced diff --git a/tests/mem_scheduler/test_mem_read_grouping.py b/tests/mem_scheduler/test_mem_read_grouping.py new file mode 100644 index 000000000..06561c060 --- /dev/null +++ b/tests/mem_scheduler/test_mem_read_grouping.py @@ -0,0 +1,139 @@ +"""Fine-transfer Hooks describe the actual Reader result grouping.""" + +from collections import defaultdict +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.mem_reader.simple_struct import SimpleStructMemReader +from memos.mem_reader.strategy_struct import StrategyStructMemReader +from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import ( + MemReadMessageHandler, +) +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.plugins.hook_defs import H +from memos.plugins.hooks import register_hook + + +class CustomTransferReader(SimpleStructMemReader): + def fine_transfer_simple_mem(self, *args, **kwargs): + return self.transfer_result + + +@pytest.fixture +def captured_operations(monkeypatch): + monkeypatch.setattr("memos.plugins.hooks._hooks", defaultdict(list)) + captured = [] + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, lambda **kwargs: captured.append(kwargs)) + return captured + + +def memory(text): + return TextualMemoryItem( + memory=text, + metadata=TreeNodeTextualMemoryMetadata(memory_type="LongTermMemory"), + ) + + +@pytest.mark.parametrize( + "reader_class,declared_grouping,expected_grouping", + [ + (SimpleStructMemReader, None, "per_input"), + (StrategyStructMemReader, None, "per_input"), + (MultiModalStructMemReader, None, "batch"), + (CustomTransferReader, None, "unknown"), + (CustomTransferReader, "batch", "batch"), + (SimpleStructMemReader, "unknown", "unknown"), + ], +) +def test_scheduler_hook_reports_reader_grouping_without_changing_results( + monkeypatch, captured_operations, reader_class, declared_grouping, expected_grouping +): + sources = [memory("first"), memory("missing"), memory("last")] + enhanced = [memory("first enhanced"), memory("last enhanced")] + reader = object.__new__(reader_class) + reader.save_rawfile = False + reader.graph_db = None + reader.memory_version_switch = "off" + reader.transfer_result = [enhanced] + if declared_grouping is not None: + reader.fine_transfer_result_grouping = declared_grouping + + def process(source, custom_tags, **kwargs): + if source is sources[1]: + return None + return [enhanced[0] if source is sources[0] else enhanced[1]] + + reader._process_transfer_chat_data = process + reader._process_transfer_multi_modal_data = lambda *args, **kwargs: enhanced + monkeypatch.setattr( + "memos.mem_reader.simple_struct.concurrent.futures.as_completed", + lambda futures: iter(reversed(list(futures))), + ) + text_mem = Mock() + text_mem.get.side_effect = sources + text_mem.add.return_value = [item.id for item in enhanced] + text_mem.memory_manager.reorganizer.is_reorganize = False + handler = MemReadMessageHandler( + SimpleNamespace(get_mem_reader=lambda: reader, services=Mock(), get_mem_cube=Mock()) + ) + handler._process_memories_with_reader( + mem_ids=[item.id for item in sources], + user_id="test-user", + mem_cube_id="test-cube", + text_mem=text_mem, + user_name="test-cube", + operation_context={ + "trace_id": "test-trace", + "user_id": "test-user", + "cube_id": "test-cube", + }, + ) + + transfer = next( + event + for event in captured_operations + if event["hook_context"].source == "scheduler.mem_read.fine_transfer_simple_mem" + ) + assert transfer["operation_input"]["result_grouping"] == expected_grouping + assert transfer["operation_input"]["memories"] == sources + if reader_class in (SimpleStructMemReader, StrategyStructMemReader): + assert transfer["result"] == [[enhanced[0]], [], [enhanced[1]]] + else: + assert transfer["result"] == [enhanced] + text_mem.add.assert_called_once_with(enhanced, user_name="test-cube") + text_mem.delete.assert_called_once_with([item.id for item in sources], user_name="test-cube") + text_mem.memory_manager.remove_and_refresh_memory.assert_called_once_with(user_name="test-cube") + + +def test_scheduler_failed_hook_keeps_grouping_and_original_error(monkeypatch, captured_operations): + error = ValueError("synthetic transfer failure") + reader = CustomTransferReader.__new__(CustomTransferReader) + reader.fine_transfer_result_grouping = "batch" + + def fail(*args, **kwargs): + raise error + + reader.fine_transfer_simple_mem = fail + failed = [] + register_hook(H.SCHEDULER_MEMORY_OPERATION_FAILED, lambda **kwargs: failed.append(kwargs)) + text_mem = Mock() + text_mem.get.return_value = memory("source") + handler = MemReadMessageHandler(SimpleNamespace(get_mem_reader=lambda: reader)) + handler._process_memories_with_reader( + mem_ids=[text_mem.get.return_value.id], + user_id="test-user", + mem_cube_id="test-cube", + text_mem=text_mem, + user_name="test-cube", + ) + assert len(failed) == 1 + assert failed[0]["error"] is error + assert failed[0]["operation_input"]["result_grouping"] == "batch" + assert all( + event["hook_context"].source != "scheduler.mem_read.fine_transfer_simple_mem" + for event in captured_operations + ) + text_mem.add.assert_not_called() From 305e3ea7e8c8785eec642068e1f67d6e2da612b0 Mon Sep 17 00:00:00 2001 From: Chen Jiang Date: Sun, 27 Sep 2026 16:47:01 +0800 Subject: [PATCH 03/22] refactor(smartcomment): simplify tracing and refine hook tests Simplify internal tracing helpers while preserving graph identity, lineage, snapshots, lifecycle and hook boundaries. Refine integration-owned hook tests, preserve upstream tests, and document the changes and validation limits. --- .../README.changes.md | 79 ++++ apps/memtrace-memos-integration/README.md | 41 ++- apps/memtrace-memos-integration/__init__.py | 1 - .../memos_smartcomment/CONVENTIONS.md | 8 +- .../memos_smartcomment/__init__.py | 2 - .../memos_smartcomment/adapter.py | 40 +-- .../memos_smartcomment/config.py | 10 +- .../memos_smartcomment/events.py | 18 +- .../memos_smartcomment/handlers/__init__.py | 5 +- .../handlers/add_handler.py | 93 ++--- .../memos_smartcomment/handlers/base.py | 37 +- .../handlers/search_handler.py | 93 ++--- .../memos_smartcomment/plugin.py | 14 +- .../memos_smartcomment/recorder.py | 38 +- .../memos_smartcomment/serialization.py | 17 +- .../handlers/mem_read_handler.py | 2 +- tests/api/test_search_hook_context.py | 178 +++++++++ tests/api/test_search_pipeline_hooks.py | 183 +--------- tests/conftest.py | 11 + tests/mem_reader/test_hook_pipeline.py | 84 ++--- tests/mem_scheduler/test_mem_read_grouping.py | 25 +- .../mem_scheduler/test_operation_observer.py | 296 ++++++++++----- .../multi_mem_cube/test_single_cube_hooks.py | 70 ++-- tests/plugins/test_hook_context.py | 338 ++++++------------ 24 files changed, 802 insertions(+), 881 deletions(-) create mode 100644 apps/memtrace-memos-integration/README.changes.md create mode 100644 tests/api/test_search_hook_context.py create mode 100644 tests/conftest.py diff --git a/apps/memtrace-memos-integration/README.changes.md b/apps/memtrace-memos-integration/README.changes.md new file mode 100644 index 000000000..a6fd65db4 --- /dev/null +++ b/apps/memtrace-memos-integration/README.changes.md @@ -0,0 +1,79 @@ +# SmartComment 集成优化说明 + +基线:`feat/smartcomment-integration` 分支的 `e80b7ced`。本文仅记录在该版本之上的修改,不重复介绍基线已有的 Hook 和插件功能。 + +## 1. 修改范围 + +- 在现有架构内精简插件实现,合入 Hook 测试优化,修正文档与实现不一致的描述。 +- 保持公开接口、配置及默认值、Hook 位置与顺序、异常处理边界不变。 +- 保留图的业务身份、来源关系、顺序、分类和输出说明,以及脱敏、快照隔离、锁、弱引用、队列、缓存、原子落盘和旧图恢复机制。 +- 不修改 MemTrace,不修改依赖约束或锁文件。MemOS 核心仅纠正一行 Reader 分组注释,可执行逻辑未改。 + +## 2. 插件代码修改 + +以下路径相对本目录的 `memos_smartcomment/`。可执行修改集中在 4 个文件、9 个私有函数。 + +| 文件 / 函数 | 修改内容 | 等价依据 | +| --- | --- | --- | +| `handlers/add_handler.py` · `_fine_transfer_values` | 将按索引切片取输入改为 `zip(..., strict=True)` 直接配对。 | 仅在 `per_input` 且数量相等时执行;顺序和空组保留。写入结果仍按唯一 ID 匹配。 | +| `handlers/add_handler.py` · `_on_scheduler_memory_operation` | 直接区分归档和删除;在添加状态节点前计算持久化来源状态,删除事后两次筛选。 | 分类和说明字符串不变;完整、部分、未知来源及空结果的判断相同。 | +| `handlers/base.py` · `_hook_subject` | 删除 `trace_id` 的重复提取。 | 保留显式 `None` 的语义;内部字典键顺序变化,但现有调用方仅按字段名读取,不影响业务输出。 | +| `handlers/base.py` · `_unique_values` | 用字典推导替代逐项赋值循环。 | 同身份保留最后一份值,位置仍按首次出现顺序排列。 | +| `handlers/search_handler.py` · `_search_memory_value` | 直接读取已规范化字段,删除重复默认值和字符串转换。 | 唯一构造入口 `_search_result_items` 已填齐字段并完成类型转换;外部输入检查保留。 | +| `handlers/search_handler.py` · `_search_memory_match_key` | 直接读取规范化字段。 | ID / 原始正文的选择、cube 范围、JSON 编码和 SHA-256 算法均未变。 | +| `handlers/search_handler.py` · `_match_search_nodes` | 接收已构造节点,复用 `match_key`;删除重复计算和缺失键兜底,调整私有参数及类型标注。 | 候选缓存仅写入已计算匹配键的节点;重复候选 FIFO 配对及过滤规则不变。 | +| `handlers/search_handler.py` · `_on_search_results` | 删除本地列表的多余 tuple 转换;传递当前节点;直接拼接输出 tuple。 | 内容和顺序不变;过滤节点不进入候选缓存,业务值的快照复制仍保留。 | +| `recorder.py` · `_normalize_memory_units` | 将嵌套字典推导改为直接更新边的来源和目标 ID。 | 使用相同别名映射,保留其他边属性、边顺序和快照恢复规则。 | + +其余插件 Python 文件仅整理注释、文档字符串或导入分组标记,没有修改执行逻辑。`adapter.py`、`handlers/base.py`、`plugin.py` 中的三处 `isort: split` 用于兼容根目录与插件的 Ruff 配置,原有绝对导入及执行顺序不变。 + +## 3. Hook 测试修改 + +以下路径相对仓库根目录。以 `93e40823` 核对上游测试归属,仅优化本次集成新增的测试。 + +| 文件 | 修改内容 | +| --- | --- | +| `tests/conftest.py` | 新增显式启用的 `clean_hooks` fixture,替代重复清理代码;不使用全局 autouse。 | +| `tests/plugins/test_hook_context.py` | 注册表列齐 14 个 Hook;合并上下文字段及复制检查;保留新旧签名兼容、单次调用、同步和异步上下文复用;删除 UUID 格式、类型提示等低价值检查。 | +| `tests/api/test_search_pipeline_hooks.py` | 撤出追加的 Hook 测试,保留上游六个测试及辅助代码;相对上游仅保留 Hook 接口所需的两处 `hook_context` 参数断言调整。 | +| `tests/api/test_search_hook_context.py` | 独立验证七个成功 Hook 的顺序与共享上下文、请求/响应替换、五个结果管道对最终响应的影响及失败传播。 | +| `tests/mem_reader/test_hook_pipeline.py` | 用一个成功用例检查参数、上下文和结果替换;用一个失败用例检查原异常及后续成功 Hook 不执行。 | +| `tests/multi_mem_cube/test_single_cube_hooks.py` | 使用固定有效 ID;验证写入参数、返回 ID 替换及后续调度消费;精简失败路径重复断言。 | +| `tests/mem_scheduler/test_operation_observer.py` | 保留 Observer 成功管道与失败传播;执行 MemRead 处理逻辑,覆盖 fine-transfer、增强写入、归档、delete、soft-delete,并验证替换结果被后续步骤使用。 | +| `tests/mem_scheduler/test_mem_read_grouping.py` | 删除“显式声明优先”的过期预期,按实际实现验证 `per_input`、`batch`、`unknown`;保留输入顺序、失败空组及异常传播检查。 | + +其余上游原有测试未修改、跳过或放宽。七个共享 Hook 测试文件已同步到独立 Hook 工作树,Hook PR 不依赖插件即可验证。 + +## 4. 文档纠偏 + +更新插件 `README.md` 和 `memos_smartcomment/CONVENTIONS.md`: + +- Reader 分组由绑定的方法实现自动识别,不读取显式分组声明。 +- 成功 Hook 表示被观察的方法返回,不独立证明数据库写入成功;失败 Hook 只能观察传播至该边界的异常。 +- 当前 MemRead refresh 调用没有独立观测点,不单独生成刷新事件。 +- 移除未随仓库提供的插件测试命令和过时失败数量,保留安装、启用及 JSON 产物检查说明。 + +`src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py` 仅删除注释中“除非显式声明”的过期表述,自动分组和 Reader 顺序保持逻辑未改。 + +## 5. 验证结果与边界 + +使用 Python 3.12.14、Poetry 2.1.3、现有锁文件及 SmartComment 0.1.2。原版与优化版在同一依赖环境的独立进程中运行相同插件回归,并核对模块加载路径,避免混用工作树代码。 + +| 验证范围 | 结果 | +| --- | --- | +| 独立 Hook 工作树 | 41 passed | +| 集成工作树 Hook + Reader 顺序/分组 | 49 passed | +| 本地插件回归:原版 / 优化版 | 31 passed / 31 passed | +| 相关离线 MemOS 模块 | 266 passed,1 skipped(上游已有) | +| 根目录与插件配置的定点 Ruff、格式检查 | 通过 | +| 两工作树 `git diff --check` | 通过 | + +上述测试集合存在重叠,不应累加通过数量。插件回归包含真实 SmartComment 图构造和 JSON 写入/读回;数据库、LLM 等外部边界使用替身。插件本地回归覆盖添加关联、来源配对、搜索阶段匹配、旧图恢复、脱敏、快照隔离及生命周期。 + +等价结论针对现有公开使用路径和业务输出;源码位置字段 `filename`、`lineno`、`trigger_point` 可随路径和行号变化,不承诺 JSON 字节完全一致。 + +完整 `make test` 在 Ollama 外部调用处达到 180 秒诊断上限后终止,未完成全量测试,也未进行真实部署环境的端到端验收。此限制不应写成“全量通过”。 + +## 6. 提交范围 + +提交内容为上述插件清理、Hook 测试优化、文档及本说明。插件目录下本地维护的 `tests/`、`tmp/` 和日志继续被 Git 忽略,不纳入提交。通用 Hook 与独立插件仍按两个上游 PR 的定位准备,不在本轮重新划分基线已有提交。 diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md index e7dec9bde..46c9514eb 100644 --- a/apps/memtrace-memos-integration/README.md +++ b/apps/memtrace-memos-integration/README.md @@ -119,20 +119,21 @@ The synchronous add flow emits three semantic memory categories: - `message_batch`: messages received at `add.before` and consumed by `mem_reader.extract.after`; - `extracted_memory`: a MemReader output passed into the TextMemory writer; -- `persisted_memory`: the memory after `text_memory.add.after` confirms the write. +- `persisted_memory`: the memory observed when `text_memory.add.after` returns its ID. The asynchronous MemRead scheduler also emits: - `enhanced_memory`: a fine-transfer result reused as the input to the enhanced-memory database write; -- `scheduler_result`: a string result for operations such as archive, delete, refresh, +- `scheduler_result`: a string result for operations such as archive, delete, or failure that do not otherwise expose a graphable output value. Scheduler fine-transfer inputs reuse the `memory::` identity created by the earlier TextMemory persistence event. This connects the original persisted memory to its enhanced successor without creating a duplicate memory node. Archive, delete, -refresh, and failed MemRead operations are also recorded; failures include only the error -type, not the error message or an error-value node. +soft-delete, and failed MemRead operations are also recorded; failures include only the +error type, not the error message or an error-value node. The current MemRead refresh +call is not wrapped by an operation Hook and is not recorded separately. A persisted memory unit is one immutable graph anchor per cube and memory ID. Its first complete snapshot is retained; different later observations log a warning without replacing @@ -146,8 +147,8 @@ one group per input in input order, including empty groups for failures; `batch` means the whole input batch contributes to its outputs. The MemRead scheduler Hook identifies SimpleStructMemReader's transfer implementation (including inherited uses such as StrategyStructMemReader) as `per_input`, and MultiModalStructMemReader's as -`batch`. A Reader's explicit `fine_transfer_result_grouping` declaration takes -precedence; custom transfer implementations otherwise remain `unknown`. If grouping is +`batch`. Grouping is inferred from the bound transfer implementation; custom overrides +remain `unknown`. If grouping is unknown or inconsistent for multiple inputs, the plugin preserves the nodes and marks lineage as unresolved instead of guessing source edges. Failed enhanced writes retain their enhanced-memory inputs and links to the failure status. @@ -155,14 +156,15 @@ writes retain their enhanced-memory inputs and links to the failure status. The integration follows the current MemOS operation-boundary Hook contract: callbacks receive correlation-only `HookContext` data plus explicit business keyword arguments. Successful MemReader, TextMemory, and scheduler callbacks observe the piped `result`; -their matching `failed` callbacks record a status node without changing the propagated -exception. MemReader and TextMemory failures also omit exception messages. +their matching `failed` callbacks record a status node without changing MemOS exception +handling. MemReader and TextMemory failures also omit exception messages. -Database write exceptions propagate through `MemoryManager.add`, including batch and -parallel single-node writes. A failed add emits the existing failure Hook instead of a -successful persistence event, and a failed enhanced write skips source-memory deletion -for that MemRead operation. Some concurrent writes may already have committed; failure -describes the operation outcome and does not imply rollback of those writes. +A successful Hook means the observed method returned; the plugin does not independently +verify database persistence. MemOS may catch and log lower-level write errors before +they reach the Hook boundary, so a `persisted_memory` node alone is not proof of a +successful database write. Failure Hooks only observe exceptions reaching their boundary. +If an enhanced-write exception reaches that boundary, subsequent source-memory deletion +is skipped for that operation. Failure does not imply rollback of writes already committed. The Search API emits this ordered result pipeline: @@ -256,10 +258,11 @@ Callers that previously relied on omitted `links` or `links=None` for automatic input/output edges must now provide explicit links. Match endpoints using each operation's data contract; unknown lineage keeps its nodes without a guessed edge. -## Run tests +## Verify the integration -From the MemOS repository root, in the same Python environment: - -```bash -python -m pytest apps/memtrace-memos-integration/tests -q -``` +The plugin regression tests are maintained locally and are not included in this directory. +For an installed integration, follow the Quick start package and entry-point check, enable +the plugin, and run a memory operation. Inspect the resulting JSON graph's `graph_id`, +`data.nodes`, and `data.edges`, and check the service logs for writer failures or dropped +events. Shutdown logs report accepted, processed, dropped, and failed event counts; +queue acceptance alone does not confirm persistence. diff --git a/apps/memtrace-memos-integration/__init__.py b/apps/memtrace-memos-integration/__init__.py index 88be0f013..e200df854 100644 --- a/apps/memtrace-memos-integration/__init__.py +++ b/apps/memtrace-memos-integration/__init__.py @@ -1,5 +1,4 @@ # ruff: noqa: N999 -# 兼容旧版应用骨架的导入入口; 实际插件实现位于 memos_smartcomment/plugin.py. """Compatibility import for the original app skeleton.""" from memos_smartcomment.plugin import SmartCommentPlugin diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md index 5dcb0d8e8..4e8f1f889 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -211,7 +211,7 @@ flowchart LR 触发边界:反馈请求和空 messages 不产生 add 输入事件;MemReader 只观察 `type="chat"`,成功时要求消息和提取结果都非空。写入成功时,输入/输出都为空就不发事件;写入失败时,无可识别候选输入则不发失败事件。失败表示该操作失败,不承诺底层已经提交的部分写入被回滚。 -这里的「写入成功」表示收到了成功 Hook 和返回 ID,插件不独立查询数据库确认。当前工作区的 `MemoryManager._submit_batches()` 捕获批量写入异常后只记录日志,可能导致失败写入仍走成功 Hook。因此不能仅凭 persisted_memory 节点认定实际写入成功;失败节点也依赖业务层正确传播异常。本次核对时,`test_write_failure_hooks.py` 的 4 个用例因此未通过。 +这里的「写入成功」表示被观察的方法返回,并触发成功 Hook 和返回 ID,插件不独立查询数据库确认。MemOS 的底层写入方法可能捕获异常并记录日志,异常未到达 Hook 边界时仍可能触发成功 Hook。因此不能仅凭 persisted_memory 节点认定实际写入成功;失败节点只反映传播至被观察边界的异常,不改变 MemOS 原有异常处理方式。 ### 3.2 持久化配对 @@ -254,13 +254,15 @@ flowchart LR | `add_enhanced_memories` 成功且有返回 ID | enhanced_memory → persisted_memory;`memory_persistence`,沿用唯一 ID 配对 | | `archive_merged_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_archival` | | `remove_memories` / `remove_source_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_deletion` | -| 其他操作成功,如 `refresh_memory_size` | memory_ids 对应持久化记忆 → scheduler_result;`scheduler_operation_result` | +| 其他符合 source 前缀的操作成功 | memory_ids 对应持久化记忆 → scheduler_result;`scheduler_operation_result` | | 失败 | 当前操作输入 → scheduler_result;`scheduler_operation_failed`;增强写入失败保留 enhanced_memory 输入 | -handler 并没有封闭的 operation_name 枚举:上述特殊分支之外,其他符合 source 前缀的名称按通用结果分支处理。成功的 fine-transfer/增强写入如果没有记忆输出,也会生成 scheduler_result;没有输入时保留独立结果节点和 operation,不补造边。 +handler 并没有封闭的 operation_name 枚举:上述特殊分支之外,其他符合 source 前缀的名称按通用结果分支处理。这里描述的是插件收到 Hook 后的处理能力,不代表每个调度调用都有 Hook;当前 MemRead 的 `remove_and_refresh_memory` 没有独立观测点,不生成刷新事件。成功的 fine-transfer/增强写入如果没有记忆输出,也会生成 scheduler_result;没有输入时保留独立结果节点和 operation,不补造边。 fine-transfer 额外记录 `result_grouping`,取自 `operation_input`,缺省 `unknown`: +MemOS 按绑定的 transfer 实现自动判断分组:SimpleStructMemReader 及继承该方法的 Reader 为 `per_input`,MultiModalStructMemReader 为 `batch`,自定义重写为 `unknown`。不读取 Reader 上的显式分组声明。 + | `result_grouping` / 输入情况 | 连线规则 | | --- | --- | | `per_input` 且外层结果组数等于输入数 | 按输入顺序逐组连接,一条输入可产生多条输出;失败/无输出对应的空组必须保留 | diff --git a/apps/memtrace-memos-integration/memos_smartcomment/__init__.py b/apps/memtrace-memos-integration/memos_smartcomment/__init__.py index 9d219ff6e..0ae3a643e 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/__init__.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/__init__.py @@ -1,5 +1,4 @@ # ruff: noqa: N999 -# 包入口: 对外只暴露 SmartCommentPlugin, 具体实现按需导入. """MemOS adapter for asynchronous smartcomment execution-graph tracing.""" from __future__ import annotations @@ -10,7 +9,6 @@ __all__ = ["SmartCommentPlugin"] -# 只有访问 SmartCommentPlugin 时才导入插件及其依赖, 避免导入包时立即加载完整集成. def __getattr__(name: str) -> Any: if name == "SmartCommentPlugin": from memos_smartcomment.plugin import SmartCommentPlugin diff --git a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py index a0e49838d..952c7461d 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py @@ -1,5 +1,4 @@ -# 异步传输与落盘层: 多个 Hook 线程提交事件, 一个后台线程串行更新和保存执行图. -# 图按 (user_id, trace_id) 隔离; 内存缓存可淘汰, 后续事件可以从磁盘快照继续追加. +# One worker serializes events into per-user, per-trace graph snapshots. from __future__ import annotations import hashlib @@ -17,6 +16,8 @@ from memos_smartcomment.recorder import SmartCommentRecorder + +# isort: split from memos.exceptions import MemOSError from memos.log import get_logger @@ -32,7 +33,6 @@ logger = get_logger(__name__) -# 写入器需要满足的最小接口; 测试可注入替代实现, 不必直接依赖 SmartComment. class Recorder(Protocol): def record(self, event: TraceEvent) -> None: ... @@ -41,8 +41,7 @@ def export_graph(self) -> dict[str, Any]: ... def restore_graph(self, data: dict[str, Any]) -> None: ... -# accepted 为入队数, dropped 为队满丢弃数, rejected 为关闭后拒收数. -# processed 为处理成功数, failed 为后台处理异常数; 入队成功不代表已经落盘. +# Queue acceptance does not imply processing or durable storage. @dataclass(slots=True) class AdapterStats: accepted: int = 0 @@ -52,7 +51,6 @@ class AdapterStats: rejected: int = 0 -# 记录单张图、最近使用时间和图元素数量, 用于 LRU/TTL 与容量淘汰. @dataclass(slots=True) class _CachedRecorder: recorder: Recorder @@ -60,7 +58,7 @@ class _CachedRecorder: graph_items: int -# 可读前缀后追加原始 ID 的 SHA-256, 避免不同 ID 经字符替换或截断后重名. +# Hash the original ID to prevent collisions after sanitizing or truncating the prefix. def _safe_path_component(value: str | None, fallback: str) -> str: normalized = re.sub(r"[^A-Za-z0-9._-]+", "_", value or fallback).strip("._") prefix = (normalized or fallback)[:80] @@ -71,7 +69,6 @@ def _safe_path_component(value: str | None, fallback: str) -> str: class AsyncTraceAdapter: """Bounded event queue and graph cache, with one snapshot-writing worker.""" - # 这里只初始化队列与状态; 真正启动线程由 start 或有待处理事件的 close 触发. def __init__( self, settings: SmartCommentSettings, @@ -88,17 +85,16 @@ def __init__( self._thread: threading.Thread | None = None self._started = False self._closed = False - # 此锁保护启动/关闭状态和统计; 图缓存只由后台线程访问, 不需要逐图加锁. + # Protect lifecycle state and stats; only the worker accesses the graph cache. self._state_lock = threading.Lock() self._stats = AdapterStats() - # 在锁内返回统计副本, 防止调用者直接修改内部计数. @property def stats(self) -> AdapterStats: with self._state_lock: return replace(self._stats) - # 调用方已持有状态锁; 一个写入线程独占图缓存, 避免并发修改同一张图. + # The caller holds the state lock. def _start_worker(self) -> None: self._thread = threading.Thread( target=self._run, name="memos-smartcomment-writer", daemon=True @@ -106,7 +102,6 @@ def _start_worker(self) -> None: self._thread.start() self._started = True - # 允许重复启动调用, 但关闭后的实例不能重新启动. def start(self) -> None: with self._state_lock: if self._closed: @@ -114,14 +109,13 @@ def start(self) -> None: if not self._started: self._start_worker() - # 非阻塞入队; 与 close 共用状态锁, 确保关闭后不再接受新事件. def submit(self, event: TraceEvent) -> bool: with self._state_lock: if self._closed: self._stats.rejected += 1 return False try: - # 队满时丢弃追踪事件而不等待写盘, 避免追踪积压拖住 MemOS 业务请求. + # Do not let tracing backpressure block a business request. self._queue.put_nowait(event) except queue.Full: self._stats.dropped += 1 @@ -131,20 +125,18 @@ def submit(self, event: TraceEvent) -> bool: logger.warning("Dropping smartcomment event because the queue is full: %s", event.operation) return False - # 每个用户目录下每条 trace 一个 JSON 文件; 匿名用户同样参与哈希隔离. def output_path(self, trace_id: str, user_id: str | None = None) -> Path: user_part = _safe_path_component(user_id, "anonymous") trace_part = _safe_path_component(trace_id, "trace") return self.settings.output_dir / user_part / f"{trace_part}.json" - # 只读取带哈希的新路径; 文件不存在时从空图开始. def _read_snapshot(self, event: TraceEvent) -> dict[str, Any] | None: path = self.output_path(event.trace_id, event.user_id) if path.exists(): return json.loads(path.read_text(encoding="utf-8")) return None - # 先完整写入同目录临时文件, 再原子替换目标快照, 避免读到半份 JSON. + # Replace atomically so readers cannot observe a partial snapshot. def _export(self, event: TraceEvent, recorder: Recorder) -> int: exported = recorder.export_graph() output_path = self.output_path(event.trace_id, event.user_id) @@ -157,11 +149,10 @@ def _export(self, event: TraceEvent, recorder: Recorder) -> int: os.replace(temporary_path, output_path) finally: temporary_path.unlink(missing_ok=True) - # 缓存容量计入节点、边、操作和会话, 不只计算记忆节点数量. data = exported.get("data", {}) return sum(len(data.get(key, ())) for key in ("nodes", "edges", "operations", "sessions")) - # OrderedDict 左侧是最久未使用的图; 任一容量限制或空闲期限触发时从左侧淘汰. + # Evict the least recently used graph until all limits are satisfied. def _trim_cache(self) -> None: now = time.monotonic() while self._recorders: @@ -175,10 +166,8 @@ def _trim_cache(self) -> None: _, removed = self._recorders.popitem(last=False) self._cached_graph_items -= removed.graph_items - # 加载或复用目标图, 追加本次事件, 成功落盘后才重新纳入缓存. def _process(self, event: TraceEvent) -> None: key = (event.user_id, event.trace_id) - # 先从缓存移出; 如果后续记录或导出失败, 下次会重新读取最近一次成功快照. cached = self._recorders.pop(key, None) if cached is None: recorder = self._recorder_factory(event) @@ -195,7 +184,6 @@ def _process(self, event: TraceEvent) -> None: self._cached_graph_items += graph_items self._trim_cache() - # 消费循环同时负责空闲淘汰与退出清理; 单个事件失败会计数并继续处理下一条. def _run(self) -> None: try: while True: @@ -217,14 +205,12 @@ def _run(self) -> None: with self._state_lock: self._stats.processed += 1 finally: - # 无论成功失败都减少 unfinished_tasks, 否则 flush 可能一直等待. self._queue.task_done() finally: self._recorders.clear() self._cached_graph_items = 0 - # 等待所有已入队任务处理结束, 超时返回 False. - # 失败事件也会完成任务计数, 所以返回 True 不表示所有写入成功; 需结合 stats.failed 判断. + # A successful flush means tasks finished; stats.failed reports failed writes. def flush(self, timeout: float = 5.0) -> bool: deadline = time.monotonic() + timeout with self._queue.all_tasks_done: @@ -235,12 +221,12 @@ def flush(self, timeout: float = 5.0) -> bool: self._queue.all_tasks_done.wait(remaining) return True - # 先拒收新事件, 再等待线程排空队列; 等待超时只告警, 不强行中断正在进行的写入. + # Drain accepted events; a timeout warns without interrupting an active write. def close(self, timeout: float = 5.0) -> None: deadline = time.monotonic() + timeout with self._state_lock: self._closed = True - # 兼容先 submit 后 close 的调用顺序: 即使尚未启动也要处理已接受的事件. + # Accepted events must drain even if start() was never called. if not self._started and not self._queue.empty(): self._start_worker() thread = self._thread diff --git a/apps/memtrace-memos-integration/memos_smartcomment/config.py b/apps/memtrace-memos-integration/memos_smartcomment/config.py index 6c7c63bec..c1ac398cd 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/config.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/config.py @@ -1,4 +1,3 @@ -# 插件独立配置: 启动时读取 MEMOS_SMARTCOMMENT_* 环境变量, 不修改 MemOS 的业务配置. from __future__ import annotations import os @@ -7,7 +6,6 @@ from pathlib import Path -# 整数配置缺失、解析失败或不大于零时使用默认值, 保证队列和缓存上限有效. def _positive_int(name: str, default: int) -> int: raw = os.getenv(name) if raw is None: @@ -19,7 +17,6 @@ def _positive_int(name: str, default: int) -> int: return value if value > 0 else default -# 只将 1/true/yes/on 视为开启, 忽略大小写和两端空格; 其他已设置的值视为关闭. def _as_bool(name: str, default: bool = False) -> bool: raw = os.getenv(name) if raw is None: @@ -27,24 +24,21 @@ def _as_bool(name: str, default: bool = False) -> bool: return raw.strip().lower() in {"1", "true", "yes", "on"} -# 集中描述写图位置、快照大小、队列容量和内存缓存限制. @dataclass(frozen=True, slots=True) class SmartCommentSettings: """Runtime settings owned by the external tracing plugin.""" output_dir: Path = Path(".memos/memtrace") - # 队列限制待处理事件数; 单条事件的字符串长度另由 max_value_chars 限制. queue_size: int = 1024 max_value_chars: int = 20_000 - # 传给 SmartComment 的身份校验开关; 持久化记忆仍按 recorder 的不可变锚点规则处理. + # Enable SmartComment checks without changing immutable persisted-memory semantics. strict: bool = False project_id: str = "memos" - # 缓存按图数量、图元素总数和空闲时间淘汰; 已写出的 JSON 文件不会随缓存删除. + # Cache eviction never deletes durable JSON snapshots. max_cached_traces: int = 64 max_cached_graph_items: int = 50_000 cache_ttl_seconds: float = 300 - # 一次性生成配置快照; 修改环境变量后需重新加载插件才能使用新值. @classmethod def from_env(cls) -> SmartCommentSettings: return cls( diff --git a/apps/memtrace-memos-integration/memos_smartcomment/events.py b/apps/memtrace-memos-integration/memos_smartcomment/events.py index bee4dd6fe..c058da516 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/events.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/events.py @@ -1,5 +1,3 @@ -# 事件契约: handlers 生成这些数据对象, adapter 传递它们, recorder 将其转换为图. -# TraceValue 是节点描述, TraceLink 是明确的数据依赖, TraceEvent 是一次操作的观测结果. from __future__ import annotations from dataclasses import dataclass, field @@ -8,29 +6,25 @@ from uuid import uuid4 -# 一个业务值的快照, 例如消息批次或记忆正文. -# frozen 只禁止字段重新赋值, 不会冻结嵌套字典; 脱离业务对象的快照由 handlers 负责生成. +# Handlers detach nested values; frozen only prevents field reassignment. @dataclass(frozen=True, slots=True) class TraceValue: """One semantic variable consumed or produced by a MemOS operation.""" name: str value: Any - # identity 决定图中复用哪个节点; name 是业务名称, 不承担唯一标识职责. + # Identity controls node reuse; name is a descriptive label. identity: str category: str = "variable" class_name: str | None = None - # True 表示按身份引用已有节点; 首次出现也可以创建占位节点. - # 持久化记忆的占位补全规则由 recorder 单独处理. + # Reference an existing node, or create a placeholder if it has not arrived yet. identity_only: bool = False comment: str | None = None metadata: dict[str, Any] = field(default_factory=dict) - # 搜索跨阶段匹配使用截断前的指纹, 防止不同长文本在截断后被误认为相同. match_key: str | None = None # Correlation key computed before snapshot truncation. -# source -> target 表示目标使用了来源数据; 它不表示两个操作仅在时间上相邻. -# category/comment 未指定时沿用操作默认值; metadata 可覆盖同名操作元数据. +# Category/comment inherit operation defaults; link metadata overrides operation metadata. @dataclass(frozen=True, slots=True) class TraceLink: """One explicit variable-to-variable dependency within an operation.""" @@ -42,8 +36,6 @@ class TraceLink: metadata: dict[str, Any] = field(default_factory=dict) -# 一次 Hook 观测对应一个事件, 同一 trace 下的多个事件最终进入同一张图. -# 即使没有连线也保留输入输出节点; 只有显式 links 才产生边. @dataclass(frozen=True, slots=True) class TraceEvent: """Framework-neutral event; only explicit links create graph edges.""" @@ -60,6 +52,6 @@ class TraceEvent: # Inputs/outputs preserve standalone nodes; they never imply dependencies. links: tuple[TraceLink, ...] = () metadata: dict[str, Any] = field(default_factory=dict) - # event_id 标识本次观测; operation_id 若存在则保存在 metadata 中, 用于关联业务操作. + # event_id identifies this observation; operation_id is correlation metadata. event_id: str = field(default_factory=lambda: uuid4().hex) created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat()) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py index 6ac803084..80d5a00d6 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py @@ -1,8 +1,5 @@ # ruff: noqa: N999 -"""按业务链组织 Hook 回调, 并保留插件使用的统一入口。 - -AddHandler / SearchHandler 可独立使用; HookHandlers 提供两者的全部回调。 -""" +"""Add and Search Hook callbacks behind one plugin interface.""" from memos_smartcomment.handlers.add_handler import AddHandler from memos_smartcomment.handlers.search_handler import SearchHandler diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py index 7630e74fc..c133b2598 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -1,8 +1,4 @@ -"""添加链路及其后台 MemRead 增强处理。 - -消息 -> 提取记忆 -> 持久化记忆 -> 增强记忆 -> 持久化记忆。 -先列出 Hook 回调, 再列出消息批次关联、节点构造和来源配对函数。 -""" +"""Capture message extraction, persistence, and background MemRead flows.""" from __future__ import annotations @@ -32,14 +28,13 @@ _MEM_READ_SOURCE_PREFIX = "scheduler.mem_read." -# 仅在明确只有一个 cube 时给出默认值, 多 cube 场景不猜测某条记忆的归属. +# Only infer a cube when ownership is unambiguous. def _single_cube_id(subject: Any) -> str | None: cube_ids = _cube_ids(subject) return cube_ids[0] if len(cube_ids) == 1 else None -# 递归展开各种嵌套返回结构, 产出 (记忆对象, cube_id, memory_id). -# 仅接受同时具有 ID 和正文的记录; 遍历时沿用外层 cube, 并跳过元数据包装字段. +# Walk ID-bearing memory records, carrying cube ownership through wrappers. def _walk_memory_records( value: Any, cube_id: str | None = None ) -> Iterator[tuple[Any, str | None, str]]: @@ -61,7 +56,6 @@ def _walk_memory_records( yield from _walk_memory_records(item, current_cube) -# 保存一次 add 的消息身份及尚未完成提取的 cube, 供后续 MemReader 回调接续同一输入节点. @dataclass(slots=True) class _MessageBatch: owner: Callable[[], Any] @@ -91,7 +85,6 @@ def clear(self) -> None: with self._message_batches_lock: self._message_batches.clear() - # 添加链路入口: 仅跟踪非反馈且非空的消息批次, 首次创建图的根输入节点. def on_add_before(self, *, request: Any, **_kwargs: Any) -> None: if bool(_get(request, "is_feedback", False)): return @@ -117,14 +110,12 @@ def on_add_before(self, *, request: Any, **_kwargs: Any) -> None: ) ) - # add 结束后的兜底清理, 覆盖跳过 MemReader 等未消费批次关联的路径. def on_add_after(self, *, request: Any, **_kwargs: Any) -> None: """Clear pending batches if the add path did not invoke MemReader.""" key = self._message_batch_key(request, _get(request, "messages")) with self._message_batches_lock: self._message_batches.pop(key, None) - # 只处理 chat 场景, 将同一消息批次连向每条提取出的记忆; 不记录 Reader 内部变量. def on_mem_reader_extract_after( self, *, @@ -161,7 +152,7 @@ def on_mem_reader_extract_after( ) for memory, cube_id, memory_id in _walk_memory_records(result, default_cube_id) ) - # 消息关联已在构造 message 时消费, 因此空提取结果也不会遗留这次 cube 的待处理状态. + # Consume the cube's message reference even when extraction produces no output. if message is None or not extracted: return @@ -180,7 +171,6 @@ def on_mem_reader_extract_after( ) ) - # chat 提取失败时消费消息批次关联并连向失败状态, 只保留异常类型. def on_mem_reader_extract_failed( self, *, @@ -221,8 +211,7 @@ def on_mem_reader_extract_failed( ) ) - # 根据写入成功 Hook 的返回 ID 建立 extracted_memory -> persisted_memory. - # 这是对 Hook 通知的观测; 能否反映真实写入结果依赖 MemOS 正确传播数据库异常. + # A successful Hook observes the method boundary, not an independent database check. def on_text_memory_add_after( self, *, @@ -257,8 +246,7 @@ def on_text_memory_add_after( ) ) - # 写入失败 Hook 将候选记忆连到失败状态, 不依据输入数量推断成功落库节点. - # 失败描述本次操作结果; 如果底层有部分写入已提交, 这里不表示那些写入被回滚. + # Failure does not imply rollback of writes already committed. def on_text_memory_add_failed( self, *, @@ -296,7 +284,6 @@ def on_text_memory_add_failed( ) ) - # 转发成功操作及 Hook 已提供的 result, 不重新调用调度器业务方法. def on_scheduler_memory_operation_after( self, *, @@ -315,7 +302,6 @@ def on_scheduler_memory_operation_after( status="after", ) - # 转发失败观测, 让输入连向失败状态; 原始异常的继续传播由 MemOS 调用方处理. def on_scheduler_memory_operation_failed( self, *, @@ -334,12 +320,10 @@ def on_scheduler_memory_operation_failed( status="failed", ) - # Python 对象地址只用于短期查找同一消息批次, 不直接作为图中永久身份. @staticmethod def _message_batch_key(subject: Any, messages: Any) -> tuple[str, str | None, int]: return _current_trace_id(subject), _get(subject, "user_id"), id(messages) - # reference=False 表示新 add, 分配新 UUID; True 表示提取回调, 尝试消费已有批次关联. def _message_identity(self, subject: Any, messages: Any, *, reference: bool) -> str: key = self._message_batch_key(subject, messages) with self._message_batches_lock: @@ -349,19 +333,18 @@ def _message_identity(self, subject: Any, messages: Any, *, reference: bool) -> # subsequently allocated list at the same address. if batch is not None and _get(batch.owner(), "messages") is messages: cubes = _cube_ids(subject) - # 一个 add 可向多个 cube 提取; 等各 cube 都消费完关联后才移除, 避免后续 cube 丢失输入身份. + # Keep the batch until every target cube has consumed its reference. batch.pending_cubes.difference_update(cubes) if not cubes or not batch.pending_cubes: del self._message_batches[key] return batch.identity self._message_batches.pop(key, None) - # 每次新 add 使用新 UUID, 即使调用方重复使用同一个 messages 列表也不会复用上次节点. - # 引用未命中时也生成独立身份, 不把来源不明的提取结果误接到其他请求. + # Each add gets a fresh identity; unmatched references never borrow another request's. identity = f"message_batch:{key[0]}:{uuid4().hex}" if not reference: owner: Callable[[], Any] try: - # 弱引用允许已丢弃请求被回收; 查询时还会验证其 messages 就是当前对象, 防止地址复用误配. + # Do not keep abandoned requests alive. owner = weakref.ref(subject) except TypeError: # Plain dict/namespace callers cannot be weakly referenced. @@ -372,12 +355,12 @@ def owner() -> Any: cubes = set(_cube_ids(subject)) or {str(_get(subject, "user_id", "unknown"))} self._message_batches[key] = _MessageBatch(owner, identity, cubes) self._message_batches.move_to_end(key) - # 异常退出或缺少后续 Hook 时也受数量上限约束; 最旧的待处理关联会被淘汰. + # Bound pending batches even if no completion Hook arrives. while len(self._message_batches) > self._max_pending_adds: self._message_batches.popitem(last=False) return identity - # 兼容 Reader 的单组包装 [messages], 解包后尽可能保留原消息对象以关联 add 节点. + # Unwrap a single scene without copying the messages used for correlation. @staticmethod def _messages_from_scene_data(scene_data: Any) -> Any: if isinstance(scene_data, list | tuple) and len(scene_data) == 1: @@ -386,7 +369,6 @@ def _messages_from_scene_data(scene_data: Any) -> Any: return messages return scene_data - # 消息批次在 add.before 创建, 在 MemReader 回调中以 identity_only 引用同一节点. def _message_value( self, subject: Any, @@ -406,7 +388,6 @@ def _message_value( metadata=metadata, ) - # 提取结果属于当前 trace 和 cube; 身份带 extracted_memory 前缀, 让后续落库转换可见. def _extracted_memory_value( self, subject: Any, @@ -429,8 +410,7 @@ def _extracted_memory_value( metadata=metadata, ) - # 按 cube + memory_id 建立持久化锚点; 返回 ID 无法匹配输入时只记录 ID 占位. - # 身份不含 trace, 使同一图中的同步写入和延迟调度可以接到同一节点; 不会跨图合并. + # Share persisted anchors within a graph, including references arriving before the write. def _persisted_memory_value( self, memory: Any, @@ -462,7 +442,6 @@ def _persisted_memory_value( }, ) - # 后台操作按数据库身份引用原记忆; 即使写入事件尚未到达, 也可先建立占位与连线. def _persisted_memory_reference( self, *, @@ -480,7 +459,6 @@ def _persisted_memory_reference( metadata={"cube_id": cube_id, "memory_id": memory_id}, ) - # 精细处理结果先使用 enhanced_memory 身份, 写库时引用它, 再连向新的持久化锚点. def _enhanced_memory_value( self, subject: Any, @@ -506,7 +484,6 @@ def _enhanced_memory_value( metadata={"cube_id": cube_id, "memory_id": memory_id}, ) - # 归档、删除、刷新或失败等没有记忆输出的操作, 使用带操作身份的字符串结果作为终点. def _scheduler_result_value( self, hook_context: HookContext, @@ -531,7 +508,6 @@ def _scheduler_result_value( }, ) - # 把已落库输入与精细处理输出关联; 连线方式依赖 Hook 显式声明的结果分组契约. def _fine_transfer_values( self, subject: Any, @@ -547,16 +523,16 @@ def _fine_transfer_values( links: list[TraceLink] = [] grouping = operation_input.get("result_grouping", "unknown") result_groups = result if isinstance(result, list | tuple) else [] - # per_input 的外层结果按输入顺序一一分组, 空组也必须保留, 否则会把后续结果接错来源. + # Empty groups preserve alignment with their corresponding input. if grouping == "per_input" and len(result_groups) == len(inputs): groups = [ - (inputs[index : index + 1], group) for index, group in enumerate(result_groups) + ([source], group) for source, group in zip(inputs, result_groups, strict=True) ] - # batch 明确声明整批输入共同产生输出; 只有一个输入时也能确定所有输出的来源. + # Batch grouping (or one source) gives every output all known inputs. elif grouping == "batch" or len(inputs) == 1: groups = [(inputs, result)] else: - # 多输入而分组未知或数量不一致时保留输出, 使用空来源列表, 后续元数据标记 unresolved. + # Keep unresolved outputs without guessing source edges. groups = [([], result)] for source_values, group in groups: @@ -580,7 +556,6 @@ def _fine_transfer_values( ) return inputs, outputs, links - # 同步写入与增强写入共用配对规则: 用返回 ID 匹配唯一输入, 不按返回顺序猜测来源. def _persistence_values( self, subject: Any, @@ -595,7 +570,7 @@ def _persistence_values( inputs: list[TraceValue] = [] outputs: list[TraceValue] = [] links: list[TraceLink] = [] - # 索引保留每个 ID 对应的所有输入位置, 用于识别重复 ID 引起的歧义. + # Only a unique input ID establishes lineage, regardless of returned ID order. candidates: dict[str, list[int]] = defaultdict(list) source_value = self._enhanced_memory_value if enhanced else self._extracted_memory_value for source_index, (memory, record_cube, candidate_id) in enumerate(records): @@ -618,7 +593,6 @@ def _persistence_values( for index, result_id in enumerate(result_ids): memory_id = str(result_id) matching = candidates.get(memory_id, []) - # 返回 ID 可能乱序、跳过部分输入或没有已知来源; 只有唯一匹配才继承正文并建立连线. source_index = matching[0] if len(matching) == 1 else None memory, record_cube, _ = ( records[source_index] if source_index is not None else (None, cube_id, memory_id) @@ -642,15 +616,13 @@ def _persistence_values( ) return inputs, outputs, links - # 来源关系完整为 resolved, 部分可确认是 partial, 有输出但无连线是 unresolved. - # 没有输出也没有连线时视为 resolved, 因为不存在需要解释来源的输出. + # No outputs means there is no unresolved lineage. @staticmethod def _persistence_lineage(outputs: list[TraceValue], links: list[TraceLink]) -> str: if len(links) == len(outputs): return "resolved" return "partial" if links else "unresolved" - # 精细处理从完整 memories 提取 ID; 其他操作从 memory_ids 引用持久化节点. def _scheduler_persisted_inputs( self, operation_name: str, @@ -673,8 +645,6 @@ def _scheduler_persisted_inputs( for memory_id in memory_ids ] - # MemRead 成功/失败事件的共同入口; 根据操作类型构造输入输出与数据依赖. - # 后台任务的执行先后不自动产生边, 只有实际消费的记忆才连向输出或状态节点. def _on_scheduler_memory_operation( self, *, @@ -687,7 +657,7 @@ def _on_scheduler_memory_operation( status: str, ) -> None: source = str(_get(hook_context, "source", "")) - # 通用调度 Hook 也会承载其他 handler 的操作; 本插件这里只解释 MemRead 的业务契约. + # Other handlers share this Hook; only interpret MemRead operations. if not source.startswith(_MEM_READ_SOURCE_PREFIX): return @@ -715,6 +685,7 @@ def _on_scheduler_memory_operation( cube_id=cube_id, enhanced=True, ) + lineage = self._persistence_lineage(outputs, links) else: inputs = self._scheduler_persisted_inputs( operation_name, @@ -722,21 +693,16 @@ def _on_scheduler_memory_operation( cube_id=cube_id, ) - # 归档/删除/刷新或失败时通常没有记忆输出; 用状态节点承接每条实际输入, 保持操作可见. + # Operations without memory outputs still have an observable status. if not outputs: if status == "failed": category = "scheduler_operation_failed" comment = "The MemRead scheduler operation failed while processing this memory." - elif operation_name in { - "archive_merged_memories", - "remove_memories", - "remove_source_memories", - }: - category = ( - "memory_archival" - if operation_name == "archive_merged_memories" - else "memory_deletion" - ) + elif operation_name == "archive_merged_memories": + category = "memory_archival" + comment = f"Apply {operation_name} to a persisted memory unit." + elif operation_name in {"remove_memories", "remove_source_memories"}: + category = "memory_deletion" comment = f"Apply {operation_name} to a persisted memory unit." else: category = "scheduler_operation_result" @@ -770,12 +736,9 @@ def _on_scheduler_memory_operation( if status == "after" and any(value.category == "enhanced_memory" for value in outputs): metadata["lineage_status"] = "resolved" if links else "unresolved" elif operation_name == "add_enhanced_memories" and status == "after": - metadata["lineage_status"] = self._persistence_lineage( - [value for value in outputs if value.category == "persisted_memory"], - [link for link in links if link.category == "memory_persistence"], - ) + metadata["lineage_status"] = lineage if status == "failed": - # 异常消息可能包含业务正文或后端细节, 图中只记录异常类名, 不保存错误对象或消息. + # Record only the exception type; messages can expose business data. metadata["error_type"] = type(error).__name__ self._emit( self._event( diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py index a44d33e81..dc169360f 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py @@ -1,7 +1,4 @@ -"""添加与搜索共用的上下文解析、快照和事件构造。 - -这里只处理追踪数据, 不保存业务链的关联状态, 也不执行数据库操作。 -""" +"""Shared correlation, detached snapshots, and event construction.""" from __future__ import annotations @@ -11,34 +8,33 @@ from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue from memos_smartcomment.serialization import to_jsonable + +# isort: split from memos.log import get_logger logger = get_logger(__name__) -# 哨兵区分没有 trace_id 字段与显式 trace_id=None, 后者不能借用其他任务的线程上下文。 +# Distinguish a missing trace field from an explicitly empty trace. _MISSING_TRACE = object() -# 兼容字典与对象属性, 让回调可处理 MemOS 模型、HookContext 以及测试替身. def _get(value: Any, name: str, default: Any | None = None) -> Any: if isinstance(value, Mapping): return value.get(name, default) return getattr(value, name, default) -# 统一正文入口: 优先 memory 字段, 仅在其为 None 时退回 text 字段. def _memory_text(memory: Any) -> Any: text = _get(memory, "memory") return text if text is not None else _get(memory, "text") -# 决定事件归属哪张图: 显式 trace 优先, 缺少字段时才查线程上下文, 最后按用户/会话兜底. def _current_trace_id(subject: Any | None = None) -> str: trace_id = _get(subject, "trace_id", _MISSING_TRACE) if trace_id is not _MISSING_TRACE and trace_id and trace_id != "trace-id": return str(trace_id) - # 只有调用方未提供 trace 字段时才读取环境上下文; 避免调度批次中串用其他消息的 trace. + # An explicitly empty trace must not borrow ambient request context. if trace_id is _MISSING_TRACE: try: from memos.context.context import get_current_trace_id @@ -49,7 +45,6 @@ def _current_trace_id(subject: Any | None = None) -> str: except Exception: logger.debug("MemOS request context is unavailable", exc_info=True) - # 无 trace 时优先使用 MemReader 和 add 共有的 user/session; task_id 仅在无 user 时兜底. user_id = _get(subject, "user_id") if user_id is None: if task_id := _get(subject, "task_id"): @@ -59,7 +54,7 @@ def _current_trace_id(subject: Any | None = None) -> str: return f"standalone:{user_id}:{session_id}" -# 兼容添加、搜索和 HookContext 的不同 cube 字段名; 去重时保留原顺序. +# Accept the cube aliases used by add, search and scheduler contexts. def _cube_ids(subject: Any) -> tuple[str, ...]: raw = ( _get(subject, "writable_cube_ids") @@ -85,11 +80,10 @@ def __init__( self._submit = submit self._max_value_chars = max_value_chars - # 在当前 Hook 线程同步复制并脱敏, 防止排队期间原始业务对象被其他线程修改. + # Snapshot synchronously so the worker cannot observe later business mutations. def _snapshot(self, value: Any) -> Any: return to_jsonable(value, max_value_chars=self._max_value_chars) - # 统一节点构造入口: 正文和元数据都经过快照处理, 不把业务对象引用交给写入线程. def _value( self, *, @@ -113,7 +107,6 @@ def _value( metadata=self._snapshot(metadata or {}), ) - # 统一补齐 trace、用户、会话、cube 和 operation_id 等关联信息; 节点与连线由调用方提供. def _event( self, subject: Any, @@ -142,21 +135,19 @@ def _event( metadata=self._snapshot(event_metadata), ) - # 追踪提交失败只记录日志, 不让队列出口的异常改变原业务调用结果. + # Tracing failures must not change business results. def _emit(self, event: TraceEvent) -> None: try: self._submit(event) except Exception: logger.exception("Failed to enqueue smartcomment event: %s", event.operation) - # 从 HookContext 提取关联字段, 不把上下文对象本身画成节点. - # 这里的 user_name 是记忆存储使用的 cube 标识兜底, 与 user_id 是不同概念. + # Here user_name identifies the storage cube, not the user. @staticmethod def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, Any]: subject = { key: value for key in ( - "trace_id", "task_id", "user_id", "session_id", @@ -164,7 +155,7 @@ def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, An ) if (value := _get(context, key)) is not None } - # 保留显式 None trace 的语义, 让 _current_trace_id 不误用工作线程残留的 trace. + # Preserve None so worker-local trace context cannot leak into this operation. if context is not None: subject["trace_id"] = _get(context, "trace_id") cube_ids = _get(context, "cube_ids") @@ -174,7 +165,6 @@ def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, An subject.setdefault("cube_id", user_name) return subject - # 没有成功业务输出时, 用 failed 字符串作为图中终点; 异常对象不作为节点保存. def _operation_status_value(self, subject: Any, *, stage: str) -> TraceValue: operation_id = _get(subject, "operation_id") or _current_trace_id(subject) return self._value( @@ -188,10 +178,7 @@ def _operation_status_value(self, subject: Any, *, stage: str) -> TraceValue: metadata={"memos_stage": stage, "status": "failed"}, ) - # 按业务 identity 合并节点描述; 同身份保留最后一份值, 列表位置沿用首次出现顺序. + # Keep the last value for each identity in first-occurrence order. @staticmethod def _unique_values(values: list[TraceValue]) -> tuple[TraceValue, ...]: - d = {} - for value in values: - d[value.identity] = value - return tuple(d.values()) + return tuple({value.identity: value for value in values}.values()) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index 4101fc638..0f699fe86 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -1,8 +1,4 @@ -"""Search API 各阶段的候选快照、排序和过滤关系。 - -查询 -> raw -> threshold -> dedup -> rerank。 -先列出 Hook 回调, 再列出上下文解析、候选节点构造和跨阶段配对函数。 -""" +"""Capture Search candidates and their filtering and ranking lineage.""" from __future__ import annotations @@ -54,7 +50,6 @@ def clear(self) -> None: with self._search_stage_items_lock: self._search_stage_nodes.clear() - # 记录搜索的根输入 query; 请求包装对象不进入图. def on_search_before( self, *, @@ -76,7 +71,6 @@ def on_search_before( ) ) - # 召回结果作为 raw 阶段; 每个候选都由查询节点指向. def on_search_memory_results( self, *, @@ -99,7 +93,6 @@ def on_search_memory_results( }, ) - # 记录 MemOS 阈值过滤后的结果和所用阈值; 此回调本身不执行过滤. def on_search_results_after_threshold( self, *, @@ -125,7 +118,6 @@ def on_search_results_after_threshold( }, ) - # 记录去重后的候选; sim/mmr 表示启用去重, 关闭时仍保留该阶段节点. def on_search_results_after_dedup( self, *, @@ -151,7 +143,6 @@ def on_search_results_after_dedup( }, ) - # 记录 API 当前按 metadata.relativity 降序排序的结果, 插件不重新计算分数. def on_search_results_after_rerank( self, *, @@ -177,7 +168,6 @@ def on_search_results_after_rerank( }, ) - # 后处理失败时连接 query 与失败状态节点, 仅记录异常类型, 随后释放本次搜索缓存. def on_search_post_process_failed( self, *, @@ -216,7 +206,6 @@ def on_search_post_process_failed( ) self._clear_search_stages(subject) - # 搜索使用 Hook 的关联信息, 再用请求字段补充空缺及可读 cube 列表. def _search_subject( self, hook_context: HookContext | None, @@ -230,14 +219,12 @@ def _search_subject( subject["mem_cube_ids"] = cube_ids return subject - # 一条 trace 可能发生多次搜索; 优先用 operation_id 区分每次搜索, 缺失时才退回 trace. @staticmethod def _search_scope_id(subject: Any) -> str: """Identify one search operation within a possibly shared trace.""" operation_id = _get(subject, "operation_id") return str(operation_id) if operation_id is not None else _current_trace_id(subject) - # 以 trace + 搜索操作 + 阶段定位缓存, 避免同一 trace 内的多次搜索互相接错节点. def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: return ( _current_trace_id(subject), @@ -245,7 +232,6 @@ def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: stage, ) - # search.before 创建查询节点, 原始召回阶段按同一搜索身份引用它. def _search_query_value( self, subject: Any, @@ -265,7 +251,7 @@ def _search_query_value( metadata={"memos_stage": "search.before"}, ) - # 将 result_type -> cube 分桶 -> memories 的结构展平, 同时保留桶内名次和遍历位置. + # Flatten buckets while retaining bucket-local rank and traversal order. @staticmethod def _search_result_items(results: Any) -> list[dict[str, Any]]: if not isinstance(results, Mapping): @@ -281,12 +267,11 @@ def _search_result_items(results: Any) -> list[dict[str, Any]]: if not isinstance(memories, list | tuple): continue cube_id = _get(bucket, "cube_id") - # rank 从 1 开始且仅在本结果类型的当前 cube 桶内有效; global_position 是展平遍历顺序. rank_scope = f"{result_type}:{cube_id or bucket_index}" for rank, memory in enumerate(memories, start=1): global_position += 1 metadata = _get(memory, "metadata", {}) - # 按 API 常用字段优先级取分数; 0 也是有效分数, 所以后续用 is None 判断缺失. + # Zero is a valid score; only None falls through to the next field. score = _get(metadata, "relativity") if score is None: score = _get(metadata, "score") @@ -307,8 +292,6 @@ def _search_result_items(results: Any) -> list[dict[str, Any]]: ) return items - # 每一阶段都创建独立的搜索候选节点, 因而可展示分数与排名变化. - # 搜索节点使用 search_memory 身份, 不复用添加链路中的持久化记忆锚点. def _search_memory_value( self, subject: Any, @@ -317,17 +300,17 @@ def _search_memory_value( item: Mapping[str, Any], ) -> TraceValue: search_scope_id = self._search_scope_id(subject) - result_type = str(item.get("result_type", "memory")) - cube_id = item.get("cube_id") - bucket_index = item.get("bucket_index", 0) - rank = item.get("rank") - rank_scope = item.get("rank_scope") - global_position = item.get("global_position") - memory_id = item.get("memory_id") - # 节点身份还包含 stage 和 global_position, 因而同 ID 在不同阶段或重复出现时仍可单独展示. + result_type = item["result_type"] + cube_id = item["cube_id"] + bucket_index = item["bucket_index"] + rank = item["rank"] + rank_scope = item["rank_scope"] + global_position = item["global_position"] + memory_id = item["memory_id"] + # Stage and position distinguish repeated occurrences of the same memory. occurrence_id = memory_id if memory_id is not None else "anonymous" cube_scope = cube_id if cube_id is not None else f"bucket-{bucket_index}" - score = item.get("score") + score = item["score"] comment_parts = [ f"Search result memory at stage={stage}", f"rank={rank}", @@ -341,7 +324,7 @@ def _search_memory_value( name=f"{stage}_{result_type}_{rank}", value={ "memory_id": memory_id, - "memory": item.get("memory"), + "memory": item["memory"], "result_type": result_type, "cube_id": cube_id, "score": score, @@ -366,37 +349,34 @@ def _search_memory_value( }, ) - # 使用原始 item 计算指纹, 不用已截断的 node.value; 长正文截断不会影响跨阶段匹配. + # Match using the original item, never the truncated snapshot. return replace(node, match_key=self._search_memory_match_key(item)) - # 跨阶段匹配使用结果类型 + cube + 记忆 ID; 没有 ID 时退回未截断正文的指纹. - # 不把排名、分数或阶段纳入匹配键, 使同一候选在重排后仍能找到来源. + # Match by type, cube and ID, or by original text when no ID exists. @staticmethod def _search_memory_match_key(value: Mapping[str, Any]) -> str: - memory_id = value.get("memory_id") - identity = ( - ("id", str(memory_id)) if memory_id is not None else ("text", value.get("memory")) - ) - cube_id = value.get("cube_id") - cube_scope = cube_id if cube_id is not None else f"bucket-{value.get('bucket_index', 0)}" - raw = json.dumps([value.get("result_type"), cube_scope, identity], ensure_ascii=False) + memory_id = value["memory_id"] + identity = ("id", str(memory_id)) if memory_id is not None else ("text", value["memory"]) + cube_id = value["cube_id"] + cube_scope = cube_id if cube_id is not None else f"bucket-{value['bucket_index']}" + raw = json.dumps([value["result_type"], cube_scope, identity], ensure_ascii=False) return hashlib.sha256(raw.encode("utf-8")).hexdigest() - # 为每个匹配键建立先进先出的候选池, 逐个消费重复项, 避免一个来源匹配多个目标. - # 返回与当前项等长的来源列表 (可能含 None), 以及上一阶段未被匹配的移除项. + # Consume duplicate candidates FIFO so each source matches at most once. def _match_search_nodes( self, previous_nodes: tuple[TraceValue, ...], - current_items: tuple[dict[str, Any], ...], + current_nodes: tuple[TraceValue, ...], ) -> tuple[tuple[TraceValue | None, ...], tuple[TraceValue, ...]]: - pools: dict[str, deque[TraceValue]] = defaultdict(deque) + # Each node already carries its original candidate's match key. + pools: dict[str | None, deque[TraceValue]] = defaultdict(deque) for node in previous_nodes: - pools[node.match_key or self._search_memory_match_key(node.value)].append(node) + pools[node.match_key].append(node) matched: list[TraceValue | None] = [] matched_identities: set[str] = set() - for item in current_items: - candidates = pools[self._search_memory_match_key(item)] + for candidate in current_nodes: + candidates = pools[candidate.match_key] node = candidates.popleft() if candidates else None matched.append(node) if node is not None: @@ -405,7 +385,7 @@ def _match_search_nodes( removed = tuple(node for node in previous_nodes if node.identity not in matched_identities) return tuple(matched), removed - # 每个搜索阶段共用一个 filtered 状态节点, 承接这一阶段被移除候选的边. + # Removed candidates share one terminal node per stage. def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: search_scope_id = self._search_scope_id(subject) return self._value( @@ -419,7 +399,6 @@ def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue metadata={"stage": stage, "status": "filtered"}, ) - # 搜索结果阶段共用的处理流程: 展平结果、读取前阶段、配对连线、保存当前阶段并发出事件. def _on_search_results( self, *, @@ -435,7 +414,7 @@ def _on_search_results( if request is None: return subject = self._search_subject(hook_context, request) - current_items = tuple(self._search_result_items(results)) + current_items = self._search_result_items(results) with self._search_stage_items_lock: previous_nodes = ( self._search_stage_nodes.get( @@ -454,7 +433,7 @@ def _on_search_results( self._search_memory_value(subject, stage=stage, item=item) for item in current_items ) - # 首个结果阶段由 query 指向召回候选; 后续阶段必须按候选身份逐一配对. + # Only the first stage links candidates to the query. if previous_stage is None: query = self._search_query_value(subject, request, identity_only=True) inputs = (query,) @@ -471,7 +450,7 @@ def _on_search_results( else: matched_nodes, removed_nodes = self._match_search_nodes( previous_nodes, - current_items, + current_nodes, ) paired_links: list[TraceLink] = [] @@ -485,11 +464,11 @@ def _on_search_results( comment=f"Carry this Search candidate into stage={stage}.", ) ) - # 未找到来源的新候选也保留节点, 但不补造来源边; 未被匹配的旧候选则连向 filtered. - outputs = list(current_nodes) + # Keep unmatched candidates, but do not invent their provenance. + outputs = current_nodes if removed_nodes: filtered = self._search_filter_result_value(subject, stage=stage) - outputs.append(filtered) + outputs += (filtered,) paired_links.extend( TraceLink( source=replace(node, identity_only=True), @@ -501,7 +480,6 @@ def _on_search_results( ) links = tuple(paired_links) inputs = self._unique_values([link.source for link in links]) - outputs = tuple(outputs) with self._search_stage_items_lock: self._search_stage_nodes[self._search_stage_key(subject, stage)] = current_nodes @@ -522,11 +500,10 @@ def _on_search_results( metadata=event_metadata, ) ) - # rerank 是观测链路的正常终点; 异常终点由 on_search_post_process_failed 清理. + # Rerank and failure are terminal stages for this operation's cache. if stage == "rerank": self._clear_search_stages(subject) - # 只清理当前搜索 operation 的缓存, 保留同一 trace 中其他搜索的中间状态. def _clear_search_stages(self, subject: Any) -> None: trace_id = _current_trace_id(subject) search_scope_id = self._search_scope_id(subject) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py index cd3556915..62307b3ff 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py @@ -1,5 +1,3 @@ -# 集成入口: MemOS 从 memos.plugins entry point 发现此插件, 启用后调用 on_load. -# 调用链: MemOS Hook -> HookHandlers -> AsyncTraceAdapter -> SmartCommentRecorder -> JSON. from __future__ import annotations from collections.abc import Callable @@ -9,6 +7,8 @@ from memos_smartcomment.config import SmartCommentSettings from memos_smartcomment.handlers import HookHandlers + +# isort: split from memos.log import get_logger from memos.plugins.base import MemOSPlugin from memos.plugins.hook_defs import H @@ -18,36 +18,30 @@ AdapterFactory = Callable[[SmartCommentSettings], AsyncTraceAdapter] -# 以观察者身份订阅现有 Hook; 回调只提交追踪事件, 不提供替换业务结果的返回值. class SmartCommentPlugin(MemOSPlugin): """Trace existing MemOS semantic Hooks without changing business results.""" name = "smartcomment" version = "0.1.0" description = "Trace MemOS memory flows with smartcomment" - # 安装不等于启用; 需要通过 MEMOS_ENABLED_PLUGINS=smartcomment 显式开启. + # Opt in with MEMOS_ENABLED_PLUGINS=smartcomment; installation alone does not enable it. enabled_by_default = False - # adapter_factory 是可注入的创建函数, 便于测试时替换真实后台写入器. def __init__(self, adapter_factory: AdapterFactory | None = None) -> None: self._adapter_factory = adapter_factory or AsyncTraceAdapter self.adapter: AsyncTraceAdapter | None = None self.handlers: HookHandlers | None = None self.context: dict[str, Any] = {"shared": {}, "configs": {}} - # 读取配置、启动写入线程, 再把业务边界 Hook 绑定到对应回调. def on_load(self) -> None: settings = SmartCommentSettings.from_env() self.adapter = self._adapter_factory(settings) - # 回调在当前业务线程中生成快照; submit 只负责入队, 实际写图由后台线程执行. self.handlers = HookHandlers( self.adapter.submit, max_value_chars=settings.max_value_chars, ) self.adapter.start() - # 三组观测点: 添加/提取/持久化, MemRead 后台操作, Search 各个结果阶段. - # 成功与失败分别注册, 使失败状态也能与其输入记忆建立数据依赖. for hook, callback in ( (H.ADD_BEFORE, self.handlers.on_add_before), (H.ADD_AFTER, self.handlers.on_add_after), @@ -74,11 +68,9 @@ def on_load(self) -> None: settings.queue_size, ) - # 接收插件框架提供的共享组件上下文, 供插件生命周期持有. def init_components(self, context: dict[str, Any]) -> None: self.context = context - # 先释放 Hook 关联缓存, 再在等待预算内排空已接受事件, 最后记录处理统计. def on_shutdown(self) -> None: if self.handlers is not None: self.handlers.clear() diff --git a/apps/memtrace-memos-integration/memos_smartcomment/recorder.py b/apps/memtrace-memos-integration/memos_smartcomment/recorder.py index 8aeb3188b..c877e8966 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/recorder.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/recorder.py @@ -1,5 +1,4 @@ -# SmartComment 桥接层: 每个 recorder 管理一张 trace 图, 把 TraceValue/TraceLink 写入运行时. -# handlers 决定业务节点与依赖, 此处负责节点身份、操作作用域、快照恢复和导出. +# Translate semantic events into one SmartComment graph per trace. from __future__ import annotations import json @@ -21,12 +20,11 @@ logger = get_logger(__name__) -# 固定字典键顺序, 让相同结构的快照有稳定编码, 用于比较节点内容是否冲突. +# Use stable encoding when comparing immutable snapshots. def _encode(value: Any) -> str: return json.dumps(value, ensure_ascii=False, sort_keys=True) -# 与 _encode 配对, 供 SmartComment 将节点字符串还原为 JSON 值. def _decode(value: str) -> Any: return json.loads(value) @@ -34,7 +32,6 @@ def _decode(value: str) -> Any: class SmartCommentRecorder: """Own one smartcomment graph and append normalized MemOS events to it.""" - # 延迟导入 SmartComment 并创建图; trace、用户和项目身份取自首个事件及插件配置. def __init__(self, settings: SmartCommentSettings, first_event: TraceEvent) -> None: from smartcomment import ( comment_graph, @@ -57,7 +54,6 @@ def __init__(self, settings: SmartCommentSettings, first_event: TraceEvent) -> N strict=settings.strict, ) - # 将插件节点描述转换为 comment_variable 的值与参数, 以业务 identity 控制节点复用. @staticmethod def _as_comment_item(value: TraceValue) -> tuple[Any, dict[str, Any]]: options: dict[str, Any] = { @@ -72,8 +68,6 @@ def _as_comment_item(value: TraceValue) -> tuple[Any, dict[str, Any]]: options["class_name"] = value.class_name if value.comment is not None: options["comment"] = value.comment - # 持久化记忆采用不可变锚点: 始终按身份复用, 不因再次观测到不同正文就生成新版本. - # snapshot_status 另外区分仅含 ID 的 reference 与有正文的 complete. if value.category == "persisted_memory": # A database memory ID is an immutable graph anchor, not a versioned view. options["identity_only"] = True @@ -83,8 +77,7 @@ def _as_comment_item(value: TraceValue) -> tuple[Any, dict[str, Any]]: } return value.value, options - # 一次事件对应一个 SmartComment session 和一个 operation; 这不是 MemOS 对话会话. - # MemOS 的 session_id 作为元数据保留, 方便从执行图反查业务上下文. + # SmartComment sessions describe events; MemOS sessions remain metadata. def record(self, event: TraceEvent) -> None: session_metadata = { "memos_session_id": event.session_id, @@ -104,7 +97,7 @@ def record(self, event: TraceEvent) -> None: nodes: dict[int, Any] = {} completions: dict[str, TraceValue] = {} - # 按本次事件中的 TraceValue 对象缓存注册结果; 跨事件复用由业务 identity 决定. + # Cache object registrations within this event; identities handle cross-event reuse. def register(value: TraceValue) -> Any: key = id(value) if key not in nodes: @@ -117,7 +110,6 @@ def register(value: TraceValue) -> Any: ): if node.metadata.get("snapshot_status") == "reference": completions.setdefault(node.full_node_id, value) - # 完整快照出现冲突时只告警, 保留第一次完整内容; strict 模式也遵循这条规则. elif node.raw_value != _encode(value.value): logger.warning( "Ignoring a conflicting snapshot for immutable memory unit %s", @@ -143,8 +135,7 @@ def register(value: TraceValue) -> Any: ): for value in (*event.inputs, *event.outputs): register(value) - # 只按显式 TraceLink 建边, 不做 inputs x outputs 全连接. - # 端点即使未单独列入 inputs/outputs 也会注册; link 元数据优先于 operation 元数据. + # Register link endpoints even when omitted from inputs/outputs; link metadata wins. for link in event.links: self._comment_link( source=register(link.source), @@ -153,11 +144,9 @@ def register(value: TraceValue) -> Any: comment=link.comment, edge_metadata={**operation_metadata, **link.metadata}, ) - # 离开图作用域后再通过导出/导入补全占位, 避免活动作用域仍引用被替换的运行时图. if completions: self._complete_memory_snapshots(completions) - # 异步引用先到时只有 ID 占位; 完整写入快照到达后补一次正文, 保留节点 ID 和已有连线. def _complete_memory_snapshots(self, completions: dict[str, TraceValue]) -> None: """Fill an ID-only placeholder once, preserving its node ID and incident edges.""" exported = self._graph.export_graph() @@ -173,7 +162,6 @@ def _complete_memory_snapshots(self, completions: dict[str, TraceValue]) -> None # Use the public graph round-trip API after exiting the active graph scope. self.restore_graph(exported) - # 恢复前验证 trace、用户与项目一致; 导入后继续沿用当前配置的严格校验开关. def restore_graph(self, data: dict[str, Any]) -> None: from smartcomment.runtime import ExecNetwork @@ -184,7 +172,6 @@ def restore_graph(self, data: dict[str, Any]) -> None: graph.strict = self._graph.strict self._graph = graph - # 兼容旧版把同一持久化记忆拆成多版本的图: 以最早版本作锚点, 保留首个完整快照. @staticmethod def _normalize_memory_units(data: dict[str, Any]) -> dict[str, Any]: """Reconnect legacy versions to the earliest anchor and first complete snapshot.""" @@ -200,7 +187,7 @@ def _normalize_memory_units(data: dict[str, Any]) -> dict[str, Any]: "cube_id": metadata.get("cube_id"), } metadata["snapshot_status"] = "reference" if is_reference else "complete" - # SmartComment 导出节点的 name 承载 id_strategy 产生的身份, 用它聚合同一记忆的版本. + # Exported names carry the identity supplied by id_strategy. key = (node.get("class_name"), node["name"]) if key not in anchors: anchors[key] = {**node, "metadata": metadata} @@ -226,18 +213,19 @@ def _normalize_memory_units(data: dict[str, Any]) -> dict[str, Any]: "edges": [ { **edge, - **{ - # 旧版本的入边和出边都重定向到锚点, 合并节点时仍保留原始数据依赖. - field: aliases.get(edge[field], edge[field]) - for field in ("source_full_node_id", "target_full_node_id") - }, + "source_full_node_id": aliases.get( + edge["source_full_node_id"], edge["source_full_node_id"] + ), + "target_full_node_id": aliases.get( + edge["target_full_node_id"], edge["target_full_node_id"] + ), } for edge in graph_data["edges"] ], }, } - # 导出面向 MemOS 的执行图, 移除 SmartComment 内部 NONE 哨兵及相关边、操作和会话. + # Hide SmartComment's internal NONE sentinel and its associated graph elements. def export_graph(self) -> dict[str, Any]: exported = self._graph.export_graph() data = exported["data"] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/serialization.py b/apps/memtrace-memos-integration/memos_smartcomment/serialization.py index ec2bba2f7..9d221ee12 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/serialization.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/serialization.py @@ -1,5 +1,4 @@ -# 快照层: 将业务对象复制为可序列化的基础值, 在事件入队前按字段脱敏并限制大小. -# 普通消息正文只截断, 不扫描其中任意位置的凭据; 脱敏依据是结构化字段名. +# Redaction is field-based; free-form text is only truncated. from __future__ import annotations import dataclasses @@ -28,21 +27,18 @@ _ARGUMENT_KEYS = {"argument", "arguments"} -# 统一字段大小写和连接符后做子串匹配; 含 token 等敏感词的字段会整体脱敏. def _is_secret_key(key: str) -> bool: normalized = key.lower().replace("-", "_") return any(part in normalized for part in _SECRET_PARTS) -# 按字符数保留前缀并追加省略号; 限制作用于每个字符串, 不是整张图的总字节数. def _truncate(value: str, max_value_chars: int) -> str: if len(value) <= max_value_chars: return value return value[:max_value_chars] + "…" -# 递归处理字典、集合、dataclass、Pydantic 模型和普通对象. -# max_items 限制每层容器的输出项数; _seen 检测当前递归路径上的循环引用. +# Limits apply per container; cycle detection follows the active recursion path. def to_jsonable( value: Any, *, @@ -54,7 +50,6 @@ def to_jsonable( if value is None or isinstance(value, bool | int): return value - # JSON 不支持标准的 NaN/Infinity 数字表示, 因此把非有限浮点数转为字符串. if isinstance(value, float): return value if math.isfinite(value) else str(value) if isinstance(value, str): @@ -71,7 +66,7 @@ def to_jsonable( seen.add(value_id) try: - # 先把模型展开为字段映射, 再统一递归处理, 让嵌套字段也经过脱敏. + # Expand models first so nested fields receive the same redaction. if dataclasses.is_dataclass(value) and not isinstance(value, type): value = {field.name: getattr(value, field.name) for field in dataclasses.fields(value)} elif hasattr(value, "model_dump"): @@ -89,12 +84,9 @@ def to_jsonable( key = str(raw_key) if _is_secret_key(key): result[key] = "" - # 向量只保留长度提示, 避免高维数组占用追踪队列和图文件. elif key.lower() in _VECTOR_KEYS: length = len(item) if hasattr(item, "__len__") else "unknown" result[key] = f"" - # 工具调用的 arguments 可能是内嵌 JSON 字符串; 必须先解析和脱敏, 再截断. - # 无法解析时省略整段参数, 避免保留无法检查敏感字段的原文. elif key.lower() in _ARGUMENT_KEYS and isinstance(item, str): # OpenAI tool calls carry a JSON string, which must be decoded # before key-based redaction. Incomplete arguments are not safe @@ -139,7 +131,6 @@ def to_jsonable( result.append(f"<{len(items) - max_items} more items>") return result - # 未知对象优先读取属性字典; 没有属性字典时使用截断后的 repr 作为兜底表示. if hasattr(value, "__dict__"): return to_jsonable( vars(value), @@ -149,5 +140,5 @@ def to_jsonable( ) return _truncate(repr(value), max_value_chars) finally: - # 退出当前分支就释放标记, 因此兄弟分支共享同一个对象不会被误判为循环. + # Shared objects in sibling branches are not cycles. seen.discard(value_id) diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index 022872a00..2e343bb07 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -586,7 +586,7 @@ def _fine_transfer_result_grouping(self, mem_reader: Any) -> str: from memos.mem_reader.simple_struct import SimpleStructMemReader # Match the bound implementation so inherited methods work, while overrides - # with a different grouping contract remain unknown unless explicitly declared. + # with a different grouping contract remain unknown. transfer = getattr(mem_reader.fine_transfer_simple_mem, "__func__", None) if transfer is MultiModalStructMemReader.fine_transfer_simple_mem: return "batch" diff --git a/tests/api/test_search_hook_context.py b/tests/api/test_search_hook_context.py new file mode 100644 index 000000000..a5d39b3e8 --- /dev/null +++ b/tests/api/test_search_hook_context.py @@ -0,0 +1,178 @@ +import pytest + +from memos.api.handlers.base_handler import HandlerDependencies +from memos.api.handlers.search_handler import SearchHandler +from memos.api.product_models import APISearchRequest +from memos.plugins.hook_context import HookContext +from memos.plugins.hook_defs import H +from memos.plugins.hooks import register_hook + + +pytestmark = pytest.mark.usefixtures("clean_hooks") + + +def _empty_search_results() -> dict: + return { + "text_mem": [], + "act_mem": [], + "para_mem": [], + "pref_mem": [], + "pref_note": "", + "tool_mem": [], + "skill_mem": [], + } + + +class _CubeView: + def __init__(self, results: dict): + self.results = results + self.request = None + + def search_memories(self, request): + self.request = request + return self.results + + +def _search_handler() -> tuple[SearchHandler, _CubeView]: + handler = SearchHandler( + HandlerDependencies( + naive_mem_cube=object(), + mem_scheduler=object(), + searcher=object(), + deepsearch_agent=object(), + ) + ) + cube_view = _CubeView(_empty_search_results()) + handler._build_cube_view = lambda _search_req: cube_view + return handler, cube_view + + +def test_search_success_hooks_run_in_order_and_share_context(monkeypatch): + events = [] + result_hook_calls = [] + handler, cube_view = _search_handler() + search_req = APISearchRequest( + user_id="user", + query="original query", + session_id="session-1", + readable_cube_ids=["cube-1", "cube-2", "cube-1"], + ) + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", + lambda: "trace-1", + ) + + def replace_request(*, hook_context, request): + events.append((H.SEARCH_BEFORE, hook_context)) + return request.model_copy(update={"query": "hooked query"}) + + def record_results(hook_name): + def callback(*, hook_context, **kwargs): + events.append((hook_name, hook_context)) + result_hook_calls.append(kwargs) + + return callback + + def replace_response(*, hook_context, result, **_kwargs): + events.append((H.SEARCH_AFTER, hook_context)) + return result.model_copy(update={"message": "hooked response"}) + + result_hooks = [ + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + H.SEARCH_CONTEXT_RENDER, + ] + register_hook(H.SEARCH_BEFORE, replace_request) + for hook_name in result_hooks: + register_hook(hook_name, record_results(hook_name)) + register_hook(H.SEARCH_AFTER, replace_response) + + response = handler.handle_search_memories(search_req) + + assert [name for name, _context in events] == [ + H.SEARCH_BEFORE, + *result_hooks, + H.SEARCH_AFTER, + ] + contexts = [context for _name, context in events] + assert all(context is contexts[0] for context in contexts) + context = contexts[0] + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "user" + assert context.session_id == "session-1" + assert context.cube_ids == ("cube-1", "cube-2") + assert context.source == "api.search" + assert context.attributes == {"mode": "fast"} + assert context.operation_id + assert cube_view.request.query == "hooked query" + assert all(call["handler"] is handler for call in result_hook_calls) + assert all(call["search_req"] is cube_view.request for call in result_hook_calls) + assert response.message == "hooked response" + + +@pytest.mark.parametrize( + "hook_name", + [ + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + H.SEARCH_CONTEXT_RENDER, + ], +) +def test_search_result_hooks_replace_memories_in_response(hook_name): + handler, _cube_view = _search_handler() + memory = { + "id": "plugin-memory", + "memory": "Remember tea", + "metadata": {"relativity": 1.0, "memory_type": "LongTermMemory"}, + } + + def replace_memories(*, results, **_kwargs): + return {**results, "text_mem": [{"cube_id": "cube-1", "memories": [memory]}]} + + register_hook(hook_name, replace_memories) + + response = handler.handle_search_memories( + APISearchRequest(user_id="user", query="drink", dedup="no") + ) + + assert response.data["text_mem"][0]["memories"] == [memory] + + +def test_search_post_processing_failure_triggers_only_failure_hook(monkeypatch): + error = RuntimeError("post processing failed") + events = [] + handler, _cube_view = _search_handler() + search_req = APISearchRequest(user_id="user", query="query") + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", + lambda: "trace-1", + ) + + def fail_post_processing(_results, _relativity): + raise error + + def record_failure(**kwargs): + events.append((H.SEARCH_POST_PROCESS_FAILED, kwargs)) + + handler._apply_relativity_threshold = fail_post_processing + register_hook(H.SEARCH_POST_PROCESS_FAILED, record_failure) + register_hook(H.SEARCH_CONTEXT_RENDER, lambda **_kwargs: events.append(("render", {}))) + register_hook(H.SEARCH_AFTER, lambda **_kwargs: events.append(("after", {}))) + + with pytest.raises(RuntimeError, match="post processing failed") as exc_info: + handler.handle_search_memories(search_req) + + assert exc_info.value is error + assert [name for name, _kwargs in events] == [H.SEARCH_POST_PROCESS_FAILED] + failure = events[0][1] + assert failure["error"] is error + assert failure["handler"] is handler + assert failure["search_req"].query == "query" + context = failure["hook_context"] + assert context.trace_id == "trace-1" + assert context.user_id == "user" diff --git a/tests/api/test_search_pipeline_hooks.py b/tests/api/test_search_pipeline_hooks.py index 329a03350..bfdad25a0 100644 --- a/tests/api/test_search_pipeline_hooks.py +++ b/tests/api/test_search_pipeline_hooks.py @@ -1,20 +1,6 @@ -import pytest - -from memos.api.handlers.base_handler import HandlerDependencies from memos.api.handlers.formatters_handler import rerank_knowledge_mem -from memos.api.handlers.search_handler import SearchHandler from memos.api.product_models import APISearchRequest -from memos.multi_mem_cube.single_cube import SingleCubeView -from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H, get_hook_spec -from memos.plugins.hooks import _hooks, register_hook - - -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() def _memory(memory_id: str, memory: str, memory_type: str = "LongTermMemory") -> dict: @@ -41,39 +27,6 @@ def _file_memory(memory_id: str, memory: str, source_content: str) -> dict: } -def _empty_search_results() -> dict: - return { - "text_mem": [], - "act_mem": [], - "para_mem": [], - "pref_mem": [], - "pref_note": "", - "tool_mem": [], - "skill_mem": [], - } - - -def _search_handler(results: dict | None = None) -> tuple[SearchHandler, SingleCubeView]: - handler = SearchHandler( - HandlerDependencies( - naive_mem_cube=object(), - mem_scheduler=object(), - searcher=object(), - deepsearch_agent=object(), - ) - ) - cube_view = SingleCubeView.__new__(SingleCubeView) - search_results = results if results is not None else _empty_search_results() - cube_view.search_memories = lambda _request: search_results - # Preserve the existing _build_cube_view(search_req) call contract. - handler._build_cube_view = lambda _search_req: cube_view - return handler, cube_view - - -def _raise(error: Exception): - raise error - - def test_search_request_passes_context_format_through_to_plugins(): req = APISearchRequest( user_id="user", @@ -85,136 +38,16 @@ def test_search_request_passes_context_format_through_to_plugins(): def test_search_pipeline_hook_specs_are_registered(): - for hook_name in (H.SEARCH_RESULTS_AFTER_RERANK, H.SEARCH_CONTEXT_RENDER): - spec = get_hook_spec(hook_name) - assert spec is not None - assert spec.pipe_key == "results" - assert spec.params == ["hook_context", "handler", "search_req", "results"] - - -@pytest.mark.parametrize( - ("hook_name", "params", "pipe_key"), - [ - (H.SEARCH_BEFORE, ["hook_context", "request"], "request"), - (H.SEARCH_AFTER, ["hook_context", "request", "result"], "result"), - ( - H.SEARCH_MEMORY_RESULTS, - ["hook_context", "handler", "search_req", "results"], - "results", - ), - ( - H.SEARCH_RESULTS_AFTER_THRESHOLD, - ["hook_context", "handler", "search_req", "results"], - "results", - ), - ( - H.SEARCH_RESULTS_AFTER_DEDUP, - ["hook_context", "handler", "search_req", "results"], - "results", - ), - ( - H.SEARCH_POST_PROCESS_FAILED, - ["hook_context", "handler", "search_req", "error"], - None, - ), - ], -) -def test_search_hook_specs_match_trigger_contracts(hook_name, params, pipe_key): - spec = get_hook_spec(hook_name) - - assert spec is not None - assert spec.params == params - assert spec.pipe_key == pipe_key - - -def test_search_success_hooks_run_in_order_and_share_one_context(monkeypatch): - events: list[tuple[str, dict]] = [] - handler, cube_view = _search_handler() - search_req = APISearchRequest( - user_id="user", - query="query", - session_id="session-1", - readable_cube_ids=["cube-1", "cube-2", "cube-1"], - ) - monkeypatch.setattr( - "memos.api.handlers.search_handler.get_current_trace_id", - lambda: "trace-1", - ) - - def record(hook_name): - def callback(**kwargs): - events.append((hook_name, kwargs)) - - return callback - - ordered_hooks = [ - H.SEARCH_BEFORE, - H.SEARCH_MEMORY_RESULTS, - H.SEARCH_RESULTS_AFTER_THRESHOLD, - H.SEARCH_RESULTS_AFTER_DEDUP, - H.SEARCH_RESULTS_AFTER_RERANK, - H.SEARCH_CONTEXT_RENDER, - H.SEARCH_AFTER, - ] - for hook_name in ordered_hooks: - register_hook(hook_name, record(hook_name)) - - handler.handle_search_memories(search_req) - - assert [name for name, _kwargs in events] == ordered_hooks - contexts = [kwargs["hook_context"] for _name, kwargs in events] - assert all(context is contexts[0] for context in contexts) - context = contexts[0] - assert isinstance(context, HookContext) - assert context.trace_id == "trace-1" - assert context.user_id == "user" - assert context.session_id == "session-1" - assert context.cube_ids == ("cube-1", "cube-2") - assert context.source == "api.search" - assert context.attributes == {"mode": "fast"} - assert context.operation_id is not None - assert not hasattr(cube_view, "search_hook_context") - + after_rerank = get_hook_spec(H.SEARCH_RESULTS_AFTER_RERANK) + render = get_hook_spec(H.SEARCH_CONTEXT_RENDER) -def test_search_context_render_hook_can_replace_results(): - handler, _cube_view = _search_handler() - search_req = APISearchRequest(user_id="user", query="query") - - def render(*, results, **_kwargs): - return {**results, "rendered_by_plugin": True} - - register_hook(H.SEARCH_CONTEXT_RENDER, render) - - response = handler.handle_search_memories(search_req) - - assert response.data["rendered_by_plugin"] is True - - -def test_search_post_processing_failure_triggers_only_post_process_hook(monkeypatch): - error = RuntimeError("post processing failed") - post_process_failures = [] - handler, _cube_view = _search_handler() - handler._apply_relativity_threshold = lambda _results, _relativity: _raise(error) - search_req = APISearchRequest(user_id="user", query="query") - monkeypatch.setattr( - "memos.api.handlers.search_handler.get_current_trace_id", - lambda: "trace-1", - ) - - register_hook( - H.SEARCH_POST_PROCESS_FAILED, - lambda **kwargs: post_process_failures.append(kwargs), - ) - with pytest.raises(RuntimeError, match="post processing failed"): - handler.handle_search_memories(search_req) + assert after_rerank is not None + assert after_rerank.pipe_key == "results" + assert after_rerank.params == ["hook_context", "handler", "search_req", "results"] - assert len(post_process_failures) == 1 - assert post_process_failures[0]["error"] is error - context = post_process_failures[0]["hook_context"] - assert isinstance(context, HookContext) - assert context.trace_id == "trace-1" - assert context.user_id == "user" - assert context.operation_id is not None + assert render is not None + assert render.pipe_key == "results" + assert render.params == ["hook_context", "handler", "search_req", "results"] def test_rerank_knowledge_mem_preserves_conversation_sources_by_default(): diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..509111b2b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.fixture +def clean_hooks(): + """Keep Hook registrations isolated for tests that opt into this fixture.""" + from memos.plugins.hooks import _hooks + + _hooks.clear() + yield + _hooks.clear() diff --git a/tests/mem_reader/test_hook_pipeline.py b/tests/mem_reader/test_hook_pipeline.py index 986fcffda..d912f0ab2 100644 --- a/tests/mem_reader/test_hook_pipeline.py +++ b/tests/mem_reader/test_hook_pipeline.py @@ -1,60 +1,37 @@ -"""MemReader success/failure Hook contracts and result piping.""" - -from __future__ import annotations +"""MemReader Hook contracts at the extraction boundary.""" from types import MethodType -from uuid import UUID import pytest from memos.mem_reader.simple_struct import SimpleStructMemReader -from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H -from memos.plugins.hooks import _hooks, register_hook +from memos.plugins.hooks import register_hook -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() +pytestmark = pytest.mark.usefixtures("clean_hooks") -def test_extract_after_hook_can_replace_reader_result(): +def test_extract_after_hook_receives_operation_and_can_replace_result(monkeypatch): reader = object.__new__(SimpleStructMemReader) original_result = [["original"]] plugin_result = [["plugin"]] - - def fake_read_memory(self, *args, **kwargs): - return original_result - - reader._read_memory = MethodType(fake_read_memory, reader) - register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **_kwargs: plugin_result) - - result = reader.get_memory( - [{"role": "user", "content": "remember this"}], - type="chat", - info={"user_id": "alice", "session_id": "session-1"}, - ) - - assert result == plugin_result - - -def test_extract_after_hook_receives_context_and_business_values(monkeypatch): - reader = object.__new__(SimpleStructMemReader) - result = [["memory"]] scene_data = [{"role": "user", "content": "remember this"}] info = {"user_id": "alice", "session_id": "session-1"} captured = {} - def fake_read_memory(self, *args, **kwargs): - return result + def fake_read_memory(self, *_args, **_kwargs): + return original_result + + def replace_result(**kwargs): + captured.update(kwargs) + return plugin_result reader._read_memory = MethodType(fake_read_memory, reader) monkeypatch.setattr("memos.mem_reader.simple_struct.get_current_trace_id", lambda: "trace-1") - register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **kwargs: captured.update(kwargs)) + register_hook(H.MEM_READER_EXTRACT_AFTER, replace_result) - actual = reader.get_memory( + result = reader.get_memory( scene_data, type="chat", info=info, @@ -63,17 +40,13 @@ def fake_read_memory(self, *args, **kwargs): chat_history=[], ) - assert actual is result context = captured.pop("hook_context") - assert isinstance(context, HookContext) + assert result == plugin_result assert context.trace_id == "trace-1" assert context.user_id == "alice" assert context.session_id == "session-1" assert context.cube_ids == ("cube-1",) assert context.source == "mem_reader.extract" - assert context.attributes == {"mode": "fine", "type": "chat"} - assert context.operation_id is not None - assert UUID(context.operation_id).hex == context.operation_id assert captured == { "mem_reader": reader, "scene_data": scene_data, @@ -82,40 +55,39 @@ def fake_read_memory(self, *args, **kwargs): "mode": "fine", "user_name": "cube-1", "kwargs": {"chat_history": []}, - "result": result, + "result": original_result, } -def test_extract_failure_hook_preserves_original_error_and_skips_after(monkeypatch): +def test_extract_failure_hook_reports_and_preserves_original_error(monkeypatch): reader = object.__new__(SimpleStructMemReader) error = ValueError("reader failed") - failed_events = [] - after_events = [] + events = [] - def fake_read_memory(self, *args, **kwargs): + def fake_read_memory(self, *_args, **_kwargs): raise error - def on_failed(**kwargs): - failed_events.append(kwargs) - raise RuntimeError("plugin failed") + def record(stage): + def callback(**kwargs): + events.append((stage, kwargs)) + + return callback reader._read_memory = MethodType(fake_read_memory, reader) monkeypatch.setattr("memos.mem_reader.simple_struct.get_current_trace_id", lambda: "trace-1") - register_hook(H.MEM_READER_EXTRACT_FAILED, on_failed) - register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **kwargs: after_events.append(kwargs)) + register_hook(H.MEM_READER_EXTRACT_FAILED, record("failed")) + register_hook(H.MEM_READER_EXTRACT_AFTER, record("after")) with pytest.raises(ValueError, match="reader failed") as exc_info: reader.get_memory( [{"role": "user", "content": "remember this"}], type="chat", info={"user_id": "alice", "session_id": "session-1"}, - mode="fast", user_name="cube-1", ) assert exc_info.value is error - assert len(failed_events) == 1 - assert failed_events[0]["error"] is error - assert failed_events[0]["hook_context"].source == "mem_reader.extract" - assert failed_events[0]["hook_context"].operation_id is not None - assert after_events == [] + assert [stage for stage, _kwargs in events] == ["failed"] + failure = events[0][1] + assert failure["error"] is error + assert failure["hook_context"].source == "mem_reader.extract" diff --git a/tests/mem_scheduler/test_mem_read_grouping.py b/tests/mem_scheduler/test_mem_read_grouping.py index 06561c060..c6d0e003d 100644 --- a/tests/mem_scheduler/test_mem_read_grouping.py +++ b/tests/mem_scheduler/test_mem_read_grouping.py @@ -1,6 +1,5 @@ """Fine-transfer Hooks describe the actual Reader result grouping.""" -from collections import defaultdict from types import SimpleNamespace from unittest.mock import Mock @@ -23,8 +22,7 @@ def fine_transfer_simple_mem(self, *args, **kwargs): @pytest.fixture -def captured_operations(monkeypatch): - monkeypatch.setattr("memos.plugins.hooks._hooks", defaultdict(list)) +def captured_operations(clean_hooks): captured = [] register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, lambda **kwargs: captured.append(kwargs)) return captured @@ -38,18 +36,16 @@ def memory(text): @pytest.mark.parametrize( - "reader_class,declared_grouping,expected_grouping", + "reader_class,expected_grouping", [ - (SimpleStructMemReader, None, "per_input"), - (StrategyStructMemReader, None, "per_input"), - (MultiModalStructMemReader, None, "batch"), - (CustomTransferReader, None, "unknown"), - (CustomTransferReader, "batch", "batch"), - (SimpleStructMemReader, "unknown", "unknown"), + (SimpleStructMemReader, "per_input"), + (StrategyStructMemReader, "per_input"), + (MultiModalStructMemReader, "batch"), + (CustomTransferReader, "unknown"), ], ) def test_scheduler_hook_reports_reader_grouping_without_changing_results( - monkeypatch, captured_operations, reader_class, declared_grouping, expected_grouping + monkeypatch, captured_operations, reader_class, expected_grouping ): sources = [memory("first"), memory("missing"), memory("last")] enhanced = [memory("first enhanced"), memory("last enhanced")] @@ -58,8 +54,6 @@ def test_scheduler_hook_reports_reader_grouping_without_changing_results( reader.graph_db = None reader.memory_version_switch = "off" reader.transfer_result = [enhanced] - if declared_grouping is not None: - reader.fine_transfer_result_grouping = declared_grouping def process(source, custom_tags, **kwargs): if source is sources[1]: @@ -108,10 +102,9 @@ def process(source, custom_tags, **kwargs): text_mem.memory_manager.remove_and_refresh_memory.assert_called_once_with(user_name="test-cube") -def test_scheduler_failed_hook_keeps_grouping_and_original_error(monkeypatch, captured_operations): +def test_scheduler_failed_hook_keeps_grouping_and_original_error(captured_operations): error = ValueError("synthetic transfer failure") reader = CustomTransferReader.__new__(CustomTransferReader) - reader.fine_transfer_result_grouping = "batch" def fail(*args, **kwargs): raise error @@ -131,7 +124,7 @@ def fail(*args, **kwargs): ) assert len(failed) == 1 assert failed[0]["error"] is error - assert failed[0]["operation_input"]["result_grouping"] == "batch" + assert failed[0]["operation_input"]["result_grouping"] == "unknown" assert all( event["hook_context"].source != "scheduler.mem_read.fine_transfer_simple_mem" for event in captured_operations diff --git a/tests/mem_scheduler/test_operation_observer.py b/tests/mem_scheduler/test_operation_observer.py index b8513e5d5..6b4beaa5e 100644 --- a/tests/mem_scheduler/test_operation_observer.py +++ b/tests/mem_scheduler/test_operation_observer.py @@ -1,33 +1,126 @@ -"""Scheduler operation Hook contracts at existing exception boundaries.""" +"""Scheduler Hook contracts and their MemRead integration points.""" -from __future__ import annotations - -from uuid import UUID +from unittest.mock import MagicMock import pytest +from memos.mem_scheduler.schemas.message_schemas import ScheduleMessageItem +from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import ( + MemReadMessageHandler, +) from memos.mem_scheduler.task_schedule_modules.operation_observer import ( SchedulerOperationObserver, ) -from memos.plugins.hook_context import HookContext +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata from memos.plugins.hook_defs import H -from memos.plugins.hooks import _hooks, register_hook +from memos.plugins.hooks import register_hook + + +pytestmark = pytest.mark.usefixtures("clean_hooks") + +RAW_MEMORY_ID = "00000000-0000-0000-0000-000000000001" +ENHANCED_MEMORY_ID = "00000000-0000-0000-0000-000000000002" +SCHEDULER_ITEM_ID = "00000000-0000-0000-0000-000000000003" +OPERATION_SOURCE = { + "fine": "fine_transfer_simple_mem", + "add": "add_enhanced_memories", + "archive": "archive_merged_memories", + "delete": "remove_memories", + "soft_delete": "remove_memories", +} + + +def _memory(memory_id, memory_type, *, merged_from=None): + return TextualMemoryItem( + id=memory_id, + memory=f"{memory_type} content", + metadata=TreeNodeTextualMemoryMetadata( + user_id="alice", + session_id="session-1", + memory_type=memory_type, + sources=[], + info={"merged_from": merged_from} if merged_from else {}, + ), + ) -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() +def _mem_read_dependencies(memory_version_switch): + raw_memory = _memory(RAW_MEMORY_ID, "WorkingMemory") + enhanced_memory = _memory( + ENHANCED_MEMORY_ID, + "LongTermMemory", + merged_from=RAW_MEMORY_ID, + ) + reader = MagicMock() + reader.fine_transfer_simple_mem.return_value = [[enhanced_memory]] + reader.save_rawfile = False + reader.memory_version_switch = memory_version_switch + + text_memory = MagicMock() + text_memory.get.return_value = raw_memory + text_memory.add.return_value = [ENHANCED_MEMORY_ID] + text_memory.memory_manager.reorganizer.is_reorganize = False + + scheduler_context = MagicMock() + scheduler_context.get_mem_reader.return_value = reader + scheduler_context.services.create_event_log.return_value = MagicMock() + + message = ScheduleMessageItem( + item_id=SCHEDULER_ITEM_ID, + user_id="alice", + trace_id="trace-1", + mem_cube_id="cube-1", + session_id="session-1", + label="mem_read", + content=f'["{RAW_MEMORY_ID}"]', + user_name="cube-1", + task_id="task-1", + ) + return MemReadMessageHandler(scheduler_context), reader, text_memory, message -def test_scheduler_observer_emits_context_and_business_values_as_hook_kwargs(): - captured = {} - register_hook( - H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **kwargs: captured.update(kwargs), + +def test_mem_read_handler_uses_replaced_transfer_results_and_stored_ids(monkeypatch): + handler, _reader, text_memory, message = _mem_read_dependencies("off") + replacement = _memory("00000000-0000-0000-0000-000000000004", "LongTermMemory") + stored_id = "00000000-0000-0000-0000-000000000005" + text_memory.memory_manager.reorganizer.is_reorganize = True + monkeypatch.setattr( + "memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler.is_playground_api", + lambda: False, ) + def replace_result(*, operation, result, **_kwargs): + if operation == "fine": + return [[replacement]] + if operation == "add": + return [stored_id] + return result + + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, replace_result) + handler._process_memories_with_reader( + mem_ids=[RAW_MEMORY_ID], + user_id="alice", + mem_cube_id="cube-1", + text_mem=text_memory, + user_name="cube-1", + operation_context=message, + ) + + text_memory.add.assert_called_once_with([replacement], user_name="cube-1") + submitted = handler.scheduler_context.services.submit_messages.call_args.args[0] + assert submitted[0].content == f'["{stored_id}"]' + + +def test_scheduler_observer_pipes_result_and_exposes_operation_contract(): + captured = {} + + def replace_result(**kwargs): + captured.update(kwargs) + return ["plugin-result"] + + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, replace_result) + with SchedulerOperationObserver( source="scheduler.mem_read.add_enhanced_memories", operation="add", @@ -37,42 +130,30 @@ def test_scheduler_observer_emits_context_and_business_values_as_hook_kwargs(): ) as observer: observer.result(["stored-1"]) - assert isinstance(captured["hook_context"], HookContext) - assert captured["hook_context"].trace_id == "trace-1" + assert observer.value == ["plugin-result"] assert captured["operation"] == "add" assert captured["target"] == "textual_memory" assert captured["operation_input"] == {"memory_ids": ["memory-1"]} assert captured["result"] == ["stored-1"] - assert "event" not in captured + assert captured["hook_context"].source == "scheduler.mem_read.add_enhanced_memories" -def test_scheduler_observer_failure_hook_preserves_original_error_and_skips_after(): +def test_scheduler_observer_reports_failure_and_preserves_error(): error = ValueError("scheduler operation failed") - failed_events = [] - after_events = [] - - def on_failed(**kwargs): - failed_events.append(kwargs) - raise RuntimeError("plugin failed") - - register_hook(H.SCHEDULER_MEMORY_OPERATION_FAILED, on_failed) + events = [] + register_hook( + H.SCHEDULER_MEMORY_OPERATION_FAILED, + lambda **kwargs: events.append(("failed", kwargs)), + ) register_hook( H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **kwargs: after_events.append(kwargs), + lambda **kwargs: events.append(("after", kwargs)), ) observer = SchedulerOperationObserver( source="scheduler.mem_read.fine_transfer_simple_mem", operation="fine", target="textual_memory", - operation_context={ - "trace_id": "trace-1", - "user_id": "alice", - "session_id": "session-1", - "task_id": "task-1", - "mem_cube_id": "cube-1", - "item_id": "item-1", - "label": "mem_read", - }, + operation_context={"trace_id": "trace-1", "mem_cube_id": "cube-1"}, operation_input={"memory_ids": ["memory-1"]}, capture_failure=True, ) @@ -81,70 +162,87 @@ def on_failed(**kwargs): raise error assert exc_info.value is error - assert len(failed_events) == 1 - event = failed_events[0] - context = event["hook_context"] - assert isinstance(context, HookContext) - assert context.trace_id == "trace-1" - assert context.user_id == "alice" - assert context.session_id == "session-1" - assert context.task_id == "task-1" - assert context.cube_ids == ("cube-1",) - assert context.source == "scheduler.mem_read.fine_transfer_simple_mem" - assert context.attributes == {"scheduler_item_id": "item-1", "label": "mem_read"} - assert context.operation_id is not None - assert UUID(context.operation_id).hex == context.operation_id - assert event["operation"] == "fine" - assert event["target"] == "textual_memory" - assert event["operation_input"] == {"memory_ids": ["memory-1"]} - assert event["error"] is error - assert after_events == [] - assert observer.value is None - - -def test_scheduler_observer_emits_explicit_failure_only_once(): - failed_events = [] - after_events = [] - error = ValueError("scheduler operation failed") - - register_hook( - H.SCHEDULER_MEMORY_OPERATION_FAILED, - lambda **kwargs: failed_events.append(kwargs), - ) - register_hook( - H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **kwargs: after_events.append(kwargs), + assert [stage for stage, _kwargs in events] == ["failed"] + failure = events[0][1] + assert failure["error"] is error + assert failure["hook_context"].cube_ids == ("cube-1",) + + +@pytest.mark.parametrize( + ("memory_version_switch", "expected_operations"), + [ + ("off", ["fine", "add", "archive", "delete"]), + ("on", ["fine", "add", "soft_delete"]), + ], +) +def test_mem_read_handler_exposes_each_memory_operation( + memory_version_switch, + expected_operations, + monkeypatch, +): + handler, reader, text_memory, message = _mem_read_dependencies(memory_version_switch) + events = [] + monkeypatch.setattr( + "memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler.is_playground_api", + lambda: False, ) - with SchedulerOperationObserver( - source="scheduler.mem_read.remove_memories", - operation="delete", - target="textual_memory", - operation_context={"trace_id": "trace-1"}, - operation_input={"memory_ids": ["memory-1"]}, - ) as observer: - observer.failed(error) - observer.failed(error) - observer.result(None) - - assert len(failed_events) == 1 - assert failed_events[0]["error"] is error - assert after_events == [] + def record_operation(**kwargs): + events.append(kwargs) + return kwargs["result"] + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, record_operation) -def test_scheduler_observer_exposes_piped_result_after_successful_exit(): - register_hook( - H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **_kwargs: ["plugin-result"], + handler._process_memories_with_reader( + mem_ids=[RAW_MEMORY_ID], + user_id="alice", + mem_cube_id="cube-1", + text_mem=text_memory, + user_name="cube-1", + task_id="task-1", + operation_context=message, ) - with SchedulerOperationObserver( - source="scheduler.mem_read.add_enhanced_memories", - operation="add", - target="textual_memory", - operation_context={"trace_id": "trace-1"}, - operation_input={}, - ) as observer: - observer.result(["original-result"]) - - assert observer.value == ["plugin-result"] + assert [event["operation"] for event in events] == expected_operations + assert [event["hook_context"].source for event in events] == [ + f"scheduler.mem_read.{OPERATION_SOURCE[operation]}" for operation in expected_operations + ] + for event in events: + context = event["hook_context"] + assert event["target"] == "textual_memory" + assert context.trace_id == "trace-1" + assert context.user_id == "alice" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube-1",) + assert context.attributes["scheduler_item_id"] == SCHEDULER_ITEM_ID + assert context.attributes["label"] == "mem_read" + + inputs = {event["operation"]: event["operation_input"] for event in events} + assert inputs["fine"]["memories"][0].id == RAW_MEMORY_ID + assert inputs["fine"]["type"] == "chat" + assert inputs["add"]["memories"][0].id == ENHANCED_MEMORY_ID + assert inputs["add"]["user_name"] == "cube-1" + + if memory_version_switch == "off": + assert inputs["archive"] == {"memory_ids": [RAW_MEMORY_ID]} + assert inputs["delete"] == {"memory_ids": [RAW_MEMORY_ID]} + reader.graph_db.update_node.assert_called_once_with( + RAW_MEMORY_ID, + {"status": "archived"}, + user_name="cube-1", + ) + text_memory.delete.assert_called_once_with([RAW_MEMORY_ID], user_name="cube-1") + text_memory.soft_delete.assert_not_called() + else: + assert inputs["soft_delete"] == { + "memory_ids": [RAW_MEMORY_ID], + "preserved_memory_ids": [ENHANCED_MEMORY_ID], + } + reader.graph_db.update_node.assert_not_called() + text_memory.delete.assert_not_called() + text_memory.soft_delete.assert_called_once_with( + [RAW_MEMORY_ID], + "cube-1", + [ENHANCED_MEMORY_ID], + ) diff --git a/tests/multi_mem_cube/test_single_cube_hooks.py b/tests/multi_mem_cube/test_single_cube_hooks.py index b384157ba..f639bec8e 100644 --- a/tests/multi_mem_cube/test_single_cube_hooks.py +++ b/tests/multi_mem_cube/test_single_cube_hooks.py @@ -3,23 +3,17 @@ from __future__ import annotations import logging -import uuid from importlib import import_module from unittest.mock import MagicMock import pytest -from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H -from memos.plugins.hooks import _hooks, register_hook +from memos.plugins.hooks import register_hook -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() +pytestmark = pytest.mark.usefixtures("clean_hooks") def _make_add_request(**overrides): @@ -48,7 +42,7 @@ def single_cube_view(): from memos.multi_mem_cube.single_cube import SingleCubeView memory = TextualMemoryItem( - id=str(uuid.uuid4()), + id="00000000-0000-0000-0000-000000000001", memory="hello world", metadata=TreeNodeTextualMemoryMetadata( user_id="u1", @@ -79,15 +73,38 @@ def single_cube_view(): return view, memory -def test_text_memory_after_hook_can_replace_memory_ids(single_cube_view): - view, _memory = single_cube_view - register_hook(H.TEXT_MEMORY_ADD_AFTER, lambda **_kwargs: ["plugin-memory-id"]) +def test_text_memory_after_hook_observes_write_and_replaces_memory_ids(single_cube_view): + view, memory = single_cube_view + captured = {} + + def replace_memory_ids(**kwargs): + captured.update(kwargs) + return ["plugin-memory-id"] - results = view.add_memories(_make_add_request(async_mode="async")) + register_hook(H.TEXT_MEMORY_ADD_AFTER, replace_memory_ids) + + results = view.add_memories( + _make_add_request( + async_mode="async", + session_id="session-1", + task_id="task-1", + ) + ) assert results[0]["memory_id"] == "plugin-memory-id" + context = captured.pop("hook_context") + assert context.user_id == "test_user" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube_test",) + assert context.source == "cube_view.text_mem.add" + assert captured == { + "text_memory": view.naive_mem_cube.text_mem, + "memories": [memory], + "kwargs": {"user_name": "cube_test"}, + "result": [memory.id], + } submitted_message = view.mem_scheduler.submit_messages.call_args.kwargs["messages"][0] - assert submitted_message.mem_cube_id == "cube_test" assert submitted_message.content == '["plugin-memory-id"]' @@ -97,20 +114,21 @@ def test_text_memory_failure_hook_preserves_original_error_and_skips_after( ): view, memory = single_cube_view error = ValueError("text memory add failed") - failed_calls = [] - after_calls = [] + events = [] view.naive_mem_cube.text_mem.add.side_effect = error monkeypatch.setattr( "memos.multi_mem_cube.single_cube.get_current_trace_id", lambda: "trace-1", ) - def on_failed(**kwargs): - failed_calls.append(kwargs) - raise RuntimeError("plugin failed") + def record(stage): + def callback(**kwargs): + events.append((stage, kwargs)) - register_hook(H.TEXT_MEMORY_ADD_FAILED, on_failed) - register_hook(H.TEXT_MEMORY_ADD_AFTER, lambda **kwargs: after_calls.append(kwargs)) + return callback + + register_hook(H.TEXT_MEMORY_ADD_FAILED, record("failed")) + register_hook(H.TEXT_MEMORY_ADD_AFTER, record("after")) with pytest.raises(ValueError, match="text memory add failed") as exc_info: view.add_memories( @@ -122,19 +140,13 @@ def on_failed(**kwargs): ) assert exc_info.value is error - assert after_calls == [] - assert len(failed_calls) == 1 - call = failed_calls[0] + assert [stage for stage, _kwargs in events] == ["failed"] + call = events[0][1] context = call["hook_context"] - assert isinstance(context, HookContext) assert context.trace_id == "trace-1" assert context.user_id == "test_user" - assert context.session_id == "session-1" - assert context.task_id == "task-1" assert context.cube_ids == ("cube_test",) assert context.source == "cube_view.text_mem.add" - assert context.attributes == {"sync_mode": "async", "extract_mode": "fast"} - assert context.operation_id is not None assert call["text_memory"] is view.naive_mem_cube.text_mem assert call["memories"] == [memory] assert call["kwargs"] == {"user_name": "cube_test"} diff --git a/tests/plugins/test_hook_context.py b/tests/plugins/test_hook_context.py index c1fdad0bd..ad06e2f98 100644 --- a/tests/plugins/test_hook_context.py +++ b/tests/plugins/test_hook_context.py @@ -1,41 +1,44 @@ -"""HookContext values, callback compatibility, and @hookable propagation.""" - -from __future__ import annotations +"""Public HookContext contracts and context-aware callback behavior.""" import asyncio -from collections.abc import Mapping -from typing import Any, get_type_hints -from uuid import UUID - import pytest -from memos.plugins.hook_context import HookContext, build_hook_context +from memos.plugins.hook_context import build_hook_context from memos.plugins.hook_defs import H, define_hook, get_hook_spec -from memos.plugins.hooks import ( - _hooks, - hookable, - register_hook, - trigger_hook, - trigger_single_hook, +from memos.plugins.hooks import hookable, register_hook, trigger_hook, trigger_single_hook + + +pytestmark = pytest.mark.usefixtures("clean_hooks") + + +@pytest.mark.parametrize( + ("hook_name", "pipe_key"), + [ + (H.SEARCH_BEFORE, "request"), + (H.SEARCH_AFTER, "result"), + (H.SEARCH_MEMORY_RESULTS, "results"), + (H.SEARCH_RESULTS_AFTER_THRESHOLD, "results"), + (H.SEARCH_RESULTS_AFTER_DEDUP, "results"), + (H.SEARCH_RESULTS_AFTER_RERANK, "results"), + (H.SEARCH_CONTEXT_RENDER, "results"), + (H.SEARCH_POST_PROCESS_FAILED, None), + (H.TEXT_MEMORY_ADD_AFTER, "result"), + (H.TEXT_MEMORY_ADD_FAILED, None), + (H.SCHEDULER_MEMORY_OPERATION_AFTER, "result"), + (H.SCHEDULER_MEMORY_OPERATION_FAILED, None), + (H.MEM_READER_EXTRACT_AFTER, "result"), + (H.MEM_READER_EXTRACT_FAILED, None), + ], ) +def test_integration_hook_specs_are_registered(hook_name, pipe_key): + spec = get_hook_spec(hook_name) + assert spec is not None + assert spec.pipe_key == pipe_key -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() - - -def test_hook_context_type_hints_are_runtime_resolvable(): - hints = get_type_hints(HookContext) - assert hints["attributes"] == Mapping[str, Any] - assert hints["operation_id"] == str | None - - -def test_build_hook_context_populates_correlation_metadata(): +def test_build_hook_context_copies_correlation_metadata(): cube_ids = ["cube-1", "cube-2"] attributes = {"mode": "fine"} @@ -46,95 +49,26 @@ def test_build_hook_context_populates_correlation_metadata(): session_id="session-1", task_id="task-1", cube_ids=cube_ids, - operation_id="operation-1", attributes=attributes, ) cube_ids.append("cube-3") - attributes["mode"] = "mutated" + attributes["mode"] = "fast" - assert context == HookContext( - trace_id="trace-1", - user_id="user-1", - session_id="session-1", - task_id="task-1", - cube_ids=("cube-1", "cube-2"), - operation_id="operation-1", - source="mem_reader.extract", - attributes={"mode": "fine"}, - ) + assert context.trace_id == "trace-1" + assert context.user_id == "user-1" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube-1", "cube-2") + assert context.source == "mem_reader.extract" + assert context.attributes == {"mode": "fine"} + assert context.operation_id -def test_build_hook_context_generates_unique_operation_ids(): - first = build_hook_context(source="text_memory.add") - second = build_hook_context(source="text_memory.add") - - assert first.operation_id is not None - assert second.operation_id is not None - assert UUID(first.operation_id).hex == first.operation_id - assert UUID(second.operation_id).hex == second.operation_id - assert first.operation_id != second.operation_id - - -def test_context_aware_hook_specs_match_trigger_contracts(): - expected = { - H.TEXT_MEMORY_ADD_AFTER: ( - ["hook_context", "text_memory", "memories", "kwargs", "result"], - "result", - ), - H.TEXT_MEMORY_ADD_FAILED: ( - ["hook_context", "text_memory", "memories", "kwargs", "error"], - None, - ), - H.SCHEDULER_MEMORY_OPERATION_AFTER: ( - ["hook_context", "operation", "target", "operation_input", "result"], - "result", - ), - H.SCHEDULER_MEMORY_OPERATION_FAILED: ( - ["hook_context", "operation", "target", "operation_input", "error"], - None, - ), - H.MEM_READER_EXTRACT_AFTER: ( - [ - "hook_context", - "mem_reader", - "scene_data", - "type", - "info", - "mode", - "user_name", - "kwargs", - "result", - ], - "result", - ), - H.MEM_READER_EXTRACT_FAILED: ( - [ - "hook_context", - "mem_reader", - "scene_data", - "type", - "info", - "mode", - "user_name", - "kwargs", - "error", - ], - None, - ), - } - - for hook_name, (params, pipe_key) in expected.items(): - spec = get_hook_spec(hook_name) - assert spec is not None - assert spec.params == params - assert spec.pipe_key == pipe_key - - -def test_hook_context_is_omitted_only_for_legacy_exact_signature_callbacks(): - hook_name = "test.hook-context.compat" +def test_hook_context_is_additive_for_new_and_legacy_callbacks(): + hook_name = "test.hook-context.compatibility" define_hook( hook_name, - description="hook context compatibility test", + description="HookContext compatibility", params=["hook_context", "result"], pipe_key="result", ) @@ -149,164 +83,106 @@ def context_callback(*, hook_context, result): calls.append(("context", hook_context, result)) return result + 1 - def kwargs_callback(*, result, **kwargs): - calls.append(("kwargs", kwargs["hook_context"], result)) - return result + 1 - register_hook(hook_name, legacy_callback) register_hook(hook_name, context_callback) - register_hook(hook_name, kwargs_callback) result = trigger_hook(hook_name, hook_context=context, result=1) - assert result == 4 - assert calls == [ - ("legacy", 1), - ("context", context, 2), - ("kwargs", context, 3), - ] - - -def test_hook_callback_type_error_is_not_retried(): - hook_name = "test.hook-context.type-error" - define_hook( - hook_name, - description="hook callback type error test", - params=["hook_context", "value"], - ) - calls = [] - - def callback(*, hook_context, value): - calls.append((hook_context, value)) - raise TypeError("raised inside callback") - - register_hook(hook_name, callback) - - trigger_hook(hook_name, hook_context="context", value="value") - - assert calls == [("context", "value")] + assert result == 3 + assert calls == [("legacy", 1), ("context", context, 2)] -def test_trigger_single_hook_supports_legacy_exact_signature(): +def test_single_provider_hook_supports_legacy_callback(): hook_name = "test.single.hook-context" define_hook( hook_name, - description="single hook context compatibility test", + description="Single-provider HookContext compatibility", params=["hook_context", "value"], ) + calls = [] - def handler(*, value): + def callback(*, value): + calls.append(value) return value + 1 - register_hook(hook_name, handler) + register_hook(hook_name, callback) - result = trigger_single_hook(hook_name, hook_context=object(), value=1) + assert trigger_single_hook(hook_name, hook_context=object(), value=1) == 2 + assert calls == [1] - assert result == 2 +def test_hookable_shares_and_reuses_context_for_sync_operation(): + built_context = object() + supplied_context = object() + builder_calls = [] + observed = [] -def test_hookable_shares_context_with_sync_operation_and_legacy_callback(): - context = object() - calls = [] + def build_context(_handler, request): + builder_calls.append(request) + return built_context class Handler: - @hookable( - "context_sync", - context_builder=lambda _self, _request: context, - ) + @hookable("context_sync", context_builder=build_context) def run(self, request, *, hook_context=None): - calls.append(("operation", hook_context, request)) - return f"processed:{request}" - - def legacy_before(*, request): - calls.append(("legacy_before", request)) + observed.append(("operation", hook_context)) + return request - def context_before(*, hook_context, request): - calls.append(("context_before", hook_context, request)) + def before(*, hook_context, **_kwargs): + observed.append(("before", hook_context)) - def context_after(*, hook_context, request, result): - calls.append(("context_after", hook_context, request, result)) + def after(*, hook_context, **_kwargs): + observed.append(("after", hook_context)) - register_hook("context_sync.before", legacy_before) - register_hook("context_sync.before", context_before) - register_hook("context_sync.after", context_after) + register_hook("context_sync.before", before) + register_hook("context_sync.after", after) - result = Handler().run("request") + Handler().run("built") + Handler().run("supplied", hook_context=supplied_context) - assert result == "processed:request" - assert calls == [ - ("legacy_before", "request"), - ("context_before", context, "request"), - ("operation", context, "request"), - ("context_after", context, "request", "processed:request"), - ] - before_spec = get_hook_spec("context_sync.before") - after_spec = get_hook_spec("context_sync.after") - assert before_spec is not None - assert after_spec is not None - assert before_spec.params == ["hook_context", "request"] - assert after_spec.params == [ - "hook_context", - "request", - "result", + assert builder_calls == ["built"] + assert observed == [ + ("before", built_context), + ("operation", built_context), + ("after", built_context), + ("before", supplied_context), + ("operation", supplied_context), + ("after", supplied_context), ] -def test_hookable_reuses_explicit_context_without_calling_builder(): +def test_hookable_shares_and_reuses_context_for_async_operation(): + built_context = object() supplied_context = object() builder_calls = [] - observed_contexts = [] + observed = [] - def build_context(_self, request): + def build_context(_handler, request): builder_calls.append(request) - return object() + return built_context class Handler: - @hookable("explicit_context", context_builder=build_context) - def run(self, request, *, hook_context=None): - observed_contexts.append(hook_context) - return request - - def observe_context(*, hook_context, **_kwargs): - observed_contexts.append(hook_context) - - register_hook("explicit_context.before", observe_context) - register_hook("explicit_context.after", observe_context) - - result = Handler().run("request", hook_context=supplied_context) - - assert result == "request" - assert builder_calls == [] - assert observed_contexts == [supplied_context] * 3 - - -def test_hookable_shares_context_with_async_operation(): - context = object() - calls = [] - - class Handler: - @hookable( - "context_async", - context_builder=lambda _self, _request: context, - ) + @hookable("context_async", context_builder=build_context) async def run(self, request, *, hook_context=None): - calls.append(("operation", hook_context, request)) - return "async_result" - - def on_before(*, hook_context, request): - calls.append(("before", hook_context, request)) - - def on_after(*, hook_context, request, result): - calls.append(("after", hook_context, request, result)) - - register_hook("context_async.before", on_before) - register_hook("context_async.after", on_after) - - result = asyncio.run(Handler().run("request")) + observed.append(("operation", hook_context)) + return request - assert result == "async_result" - assert calls == [ - ("before", context, "request"), - ("operation", context, "request"), - ("after", context, "request", "async_result"), + def observe(stage): + def callback(*, hook_context, **_kwargs): + observed.append((stage, hook_context)) + + return callback + + register_hook("context_async.before", observe("before")) + register_hook("context_async.after", observe("after")) + + assert asyncio.run(Handler().run("built")) == "built" + assert asyncio.run(Handler().run("supplied", hook_context=supplied_context)) == "supplied" + assert builder_calls == ["built"] + assert observed == [ + ("before", built_context), + ("operation", built_context), + ("after", built_context), + ("before", supplied_context), + ("operation", supplied_context), + ("after", supplied_context), ] From 1918f9f9c10c422d7567e74e6cdcdce950916733 Mon Sep 17 00:00:00 2001 From: Chen Jiang Date: Mon, 28 Sep 2026 16:43:33 +0800 Subject: [PATCH 04/22] refactor(smartcomment): clarify trace semantics and layout Clarify node, event, and link descriptions and group related handlers. Remove unused TraceValue.name and legacy snapshot migration while preserving current-format recovery and existing lineage rules. Validation: plugin and Hook/Reader suite 82 passed; related MemOS suite 266 passed, 1 skipped. Full make test was blocked by an Ollama model-list network request. Scoped pre-commit checks passed. --- .../README.changes.md | 20 +- apps/memtrace-memos-integration/README.md | 35 +- .../memos_smartcomment/CONVENTIONS.md | 59 +++- .../memos_smartcomment/adapter.py | 53 +-- .../memos_smartcomment/events.py | 6 +- .../handlers/add_handler.py | 316 ++++++++++------- .../memos_smartcomment/handlers/base.py | 16 +- .../handlers/search_handler.py | 321 ++++++++++-------- .../memos_smartcomment/recorder.py | 70 +--- 9 files changed, 518 insertions(+), 378 deletions(-) diff --git a/apps/memtrace-memos-integration/README.changes.md b/apps/memtrace-memos-integration/README.changes.md index a6fd65db4..9c556ab7d 100644 --- a/apps/memtrace-memos-integration/README.changes.md +++ b/apps/memtrace-memos-integration/README.changes.md @@ -1,6 +1,6 @@ # SmartComment 集成优化说明 -基线:`feat/smartcomment-integration` 分支的 `e80b7ced`。本文仅记录在该版本之上的修改,不重复介绍基线已有的 Hook 和插件功能。 +基线:`feat/smartcomment-integration` 分支的 `e80b7ced`。第 1–6 节记录已提交的首轮优化 `305e3ea7`;第 7 节记录随后按 TODO 清单进行的整理。本文不重复介绍基线已有的 Hook 和插件功能。 ## 1. 修改范围 @@ -77,3 +77,21 @@ ## 6. 提交范围 提交内容为上述插件清理、Hook 测试优化、文档及本说明。插件目录下本地维护的 `tests/`、`tmp/` 和日志继续被 Git 忽略,不纳入提交。通用 Hook 与独立插件仍按两个上游 PR 的定位准备,不在本轮重新划分基线已有提交。 + +## 7. Trace 说明与代码布局整理(基于 `305e3ea7`) + +| TODO | 本轮处理 | +| --- | --- | +| 1. 字段命名与说明 | 保留 category、class_name、operation 和 identity 取值;新增 `TraceEvent.comment`,让操作/session 展示业务说明;补齐显式连线说明,澄清写入返回、搜索过滤、桶内排名、归档与删除的观测边界。 | +| 2. 历史兼容 | 删除 `recorder._normalize_memory_units()` 的旧图格式迁移。当前格式恢复、ID 占位补全、首个完整快照及缓存淘汰后的续写保留。 | +| 3. 冗余字段 | 删除未被读取或导出的 `TraceValue.name` 及构造参数;导出图的 `name` 仍由 identity 生成。 | +| 4. 可读性 | 为规范化 Search 候选增加一个内部 `_SearchItem` TypedDict。保留多个真实入口共用的记忆展开函数,不改来源匹配算法。 | +| 5. 代码布局 | Add 的调度处理紧接调度回调;Search 的共用流程紧接阶段回调,缓存清理归入关联状态;Adapter 集中生命周期方法。加简短功能分组注释,不新建模块。 | + +修改集中在 `events.py`、`handlers/base.py`、`handlers/add_handler.py`、`handlers/search_handler.py`、`recorder.py`、`adapter.py`,同步更新 README 和 CONVENTIONS。MemOS 核心、仓库原有测试、Hook 测试、MemTrace、依赖及锁文件均未改动。 + +本轮不是完整输出逐字等价:图中的 comment 有意更新,`TraceValue` 构造不再接受 name,不再自动迁移历史图;来源关系、节点身份、分类、业务值和现有调用点的处理规则保持不变。持久化节点已有完整快照时不会只为更新说明而覆盖历史 comment。历史 split-version/无 snapshot_status 图应放在独立目录,不继续追加。 + +验证:修改前插件回归 **31 passed**;修改后 **33 passed**,包括真实 SmartComment 图导出/读回、当前快照恢复及占位补全、说明传递、缓存恢复和失败恢复。相关 MemOS 离线回归 **266 passed, 1 skipped**(原有 skip),数据库/LLM 边界仍使用替身。插件测试仍只在本地维护。未重新部署完整 MemOS;此前全量测试的 Ollama 环境限制不变。 + +Hook/Reader 定点回归 **49 passed**(与上述离线集合重叠);改动文件的 pre-commit、两套 Ruff 配置检查/格式检查及 `git diff --check` 通过。Adapter 所有方法的语法树与基线一致,仅顺序和注释变化。 diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md index 46c9514eb..4321054a6 100644 --- a/apps/memtrace-memos-integration/README.md +++ b/apps/memtrace-memos-integration/README.md @@ -101,7 +101,9 @@ Hook callbacks live in `memos_smartcomment/handlers/`: - `__init__.py`: `HookHandlers` combines both flows and clears both caches at shutdown. The existing `from memos_smartcomment.handlers import HookHandlers` import is unchanged. -Each handler lists Hook callbacks before its internal helpers. `AddHandler` and +Each handler lists Hook callbacks before its shared processing flow, followed by +correlation state, node construction, and lineage helpers. Short section comments mark +these responsibilities. `AddHandler` and `SearchHandler` can also be instantiated independently with an event submission callable. ### Captured memory flow @@ -139,8 +141,10 @@ A persisted memory unit is one immutable graph anchor per cube and memory ID. It complete snapshot is retained; different later observations log a warning without replacing it or creating a new version, including in strict mode. If an asynchronous reference arrives before the write event, its ID-only placeholder is completed once without changing the node -ID or existing edges. Restoring older snapshots also joins split versions of persisted units -to their earliest anchor and retains the first complete snapshot and all incident edges. +ID or existing edges. Current-format snapshots restore those anchors and edges directly, +including after cache eviction. Historical formats with split persisted-memory versions +or missing `snapshot_status` metadata are no longer migrated; use a separate output +directory instead of appending to those artifacts. Fine-transfer Hooks declare `operation_input.result_grouping`: `per_input` means one group per input in input order, including empty groups for failures; `batch` @@ -210,6 +214,28 @@ are not connected merely because one ran after another; the graph contains only data-flow edges. When an operation has no data output, its `scheduler_result` string is used as the target of edges from the memory values it consumed. +### Trace field conventions + +Keep machine identifiers separate from reader-facing descriptions: + +- `identity` controls node reuse; SmartComment exports it as the node `name`. + The unused internal `TraceValue.name` field has been removed. +- Node `category` names a business role; `class_name` is a type/identity namespace. + Existing names are retained because consumers and node IDs can depend on them. +- `TraceEvent.operation` is the stable dotted operation name; its optional `comment` + explains the observation in the exported operation and session. Each built-in link + has its own description of the source-to-target relationship. +- Write comments describe returned IDs, not independently confirmed database writes. + Search comments distinguish bucket-local rank from traversal order, and removal + from search results from deletion in storage. Scheduler comments identify the backend + operation, including `delete` versus `soft_delete`, without claiming every input was removed. + +These comments are displayed by graph viewers and MemTrace notebooks; they are not +parsing keys. The complete field tables and reading order are in +[`CONVENTIONS.md`](memos_smartcomment/CONVENTIONS.md). + +### Correlation and snapshot isolation + Each add receives a fresh UUID for its message batch, including repeated adds that reuse the same Python list. A bounded registry correlates the live request with MemReader callbacks for each target cube. The registry uses weak references for API requests and @@ -254,8 +280,7 @@ event = TraceEvent( ) ``` -Callers that previously relied on omitted `links` or `links=None` for automatic -input/output edges must now provide explicit links. Match endpoints using each +Always provide explicit links for known dependencies. Match endpoints using each operation's data contract; unknown lineage keeps its nodes without a guessed edge. ## Verify the integration diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md index 4e8f1f889..6f4374e6b 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -10,23 +10,22 @@ | 对象 | 表示什么 | 关键字段 | | --- | --- | --- | -| `TraceValue` | 一份业务值快照,最终注册为图节点 | `name`、`value`、`identity`、`class_name`、`category`、`identity_only`、`comment`、`metadata`、`match_key` | -| `TraceEvent` | 一次 Hook 观测,最终建立一个 session 和一个 operation | `operation`、`category`、`trace_id`、用户/会话/cube 关联信息、`inputs`、`outputs`、`links`、`metadata`、`event_id`、`created_at` | +| `TraceValue` | 一份业务值快照,最终注册为图节点 | `value`、`identity`、`class_name`、`category`、`identity_only`、`comment`、`metadata`、`match_key` | +| `TraceEvent` | 一次 Hook 观测,最终建立一个 session 和一个 operation | `operation`、`category`、`comment`、`trace_id`、用户/会话/cube 关联信息、`inputs`、`outputs`、`links`、`metadata`、`event_id`、`created_at` | | `TraceLink` | 来源值到目标值的显式数据依赖 | `source`、`target`、`category`、`comment`、`metadata` | `inputs` 和 `outputs` 只注册节点,**不会自动生成输入与输出之间的边**。只有 `links` 生成边;未单独列入输入/输出的 link 端点也会注册。操作发生的时间先后不构成连线依据。 ### 1.1 `identity`、`name`、`full_node_id` 的区别 -当前 `TraceValue` 的字段叫 **`identity`,没有 `identity_id` 字段**。本地 SmartComment 的节点导出也使用 `name`、`full_name`、`node_id`、`full_node_id`。如果讨论中说「identity_id」,应先明确是业务身份还是带版本的图节点 ID。 +`TraceValue.identity` 是业务身份;导出图使用 `name`、`full_name`、`node_id`、`full_node_id` 表示身份及其版本。未被消费的内部 `TraceValue.name` 已删除,导出节点的 `name` 字段不变。 Recorder 用 `id_strategy` 返回 `TraceValue.identity`,因此映射如下: | 字段 | 当前含义/构成 | | --- | --- | -| `TraceValue.name` | handler 内的业务名称,如 `messages`、`query`、`raw_text_mem_1`;当前 recorder 不将它传给 SmartComment | | `TraceValue.identity` | handler 构造的业务身份字符串 | -| 导出节点 `name` | 等于 `TraceValue.identity`,不是 `TraceValue.name` | +| 导出节点 `name` | 等于 `TraceValue.identity` | | 导出节点 `full_name` | `{class_name}:{identity}`;没有 `class_name` 时为 `{identity}` | | 导出节点 `node_id` | `{identity}@{version}` | | 导出节点 `full_node_id` | `{class_name}:{identity}@{version}`;边的 source/target 使用此字段 | @@ -69,23 +68,39 @@ cube 列表取第一个非空字段:`writable_cube_ids → readable_cube_ids | 普通节点,身份相同但 value 不同,`identity_only=False` | 非 strict 模式创建新版本;strict 模式报一致性错误 | | `category="persisted_memory"` | recorder 强制按身份复用,使用下面的不可变锚点规则 | -持久化记忆由 `(cube_id, memory_id)` 标识,保留第一次完整正文快照。仅有 ID 时先创建占位;完整快照到达后补全一次,保留原节点 ID 和连线。此后内容冲突只告警,不覆盖或升版,strict 模式也相同。恢复旧图时,同一持久化身份的多个版本合并到最早锚点,并保留首个完整快照及关联边。 +持久化记忆由 `(cube_id, memory_id)` 标识,保留第一次完整正文快照。仅有 ID 时先创建占位;完整快照到达后补全一次,保留原节点 ID 和连线。此后内容冲突只告警,不覆盖或升版,strict 模式也相同。当前格式快照直接恢复,保留已有身份和边;不再迁移缺少 `snapshot_status` 或把持久化锚点拆成多个版本的历史图。此类历史产物应使用独立目录保存,不继续追加。 普通节点复用时,不会因为新一次观测的 metadata/comment 不同就自动更新它们;逐次操作信息应查看对应 operation/edge。持久化占位补全是 recorder 的专门处理。 +### 1.4 分类、操作名与说明文字 + +| 字段 | 用途与命名规则 | +| --- | --- | +| 节点 `category` | 值在流程中的业务角色,使用 snake_case 名词,如 `extracted_memory`、`persisted_memory`。相同正文在不同阶段可有不同角色。 | +| 节点 `class_name` | SmartComment 类型/身份命名空间,不是阶段名;`memory` 共用于提取、增强和持久化记忆,`str` 用于查询和状态。改名会改变 `full_node_id`。 | +| `TraceEvent.operation` | 稳定的操作标识,以 `memos.` 开头,点号分隔流程和动作;Scheduler 保留调用点及 `after/failed` 后缀。与业务关联 ID `operation_id` 不同。 | +| 事件/边 `category` | 操作或依赖关系的类别,如 `memory_persistence`、`memory_filtered`,不要求与节点类别相同。 | +| 节点 `comment` | 说明捕获了什么值及其观测边界;候选排名注明桶内范围,状态节点不冒充业务输出。 | +| 事件 `comment` | 说明本次操作的业务意图和实际观测结果,写入导出 operation/session 的 `comment`。自定义事件未提供时保留按 operation 生成的默认说明。 | +| 连线 `comment` | 解释为什么从来源值连向目标值;所有内置流程显式提供,不用操作名代替依赖含义。 | + +说明使用简短英文句子,不作为身份、分类或程序分支依据。`persisted_memory` 表示写入返回 ID 对应的图锚点,不是独立数据库确认;`filtered` 表示候选不再出现在当前结果,不是删除存储中的记忆;`after`/`completed` 表示方法返回,`failed` 表示异常到达 Hook 边界,不表示回滚。 + +本轮保留现有 category/class_name/operation 取值。MemTrace 通用图展示和 notebook 会显示 comment;部分 QA 工具另按 `class_name="query"`、`category="message"` 或 `"message & query"` 识别查询,不能据此把本插件的 Search 观测图当作完整 QA 图。本次不修改这些消费方或扩展 QA 适配。 + ## 2. `TraceValue` 节点总表 下面的 `{cube}` 在 Add 链中表示 `cube_id or 'unknown'`;`{trace}` 为解析后的 trace。`{search_scope}` 优先使用非 `None` 的 `operation_id`,否则使用 trace。`{status}` 在调度链中为 `after` 或 `failed`。 ### 2.1 AddHandler:输入、提取、持久化与后台增强 -| `TraceValue.name` | `category` | `class_name` | `identity` | `value` | +| 业务含义 | `category` | `class_name` | `identity` | `value` | | --- | --- | --- | --- | --- | -| `messages` | `message_batch` | `message_batch` | `message_batch:{trace}:{uuid4.hex}` | messages 的快照 | -| `extracted_memory` | `extracted_memory` | `memory` | `extracted_memory:{trace}:{cube}:{memory_id}` | 记忆正文 | -| `persisted_memory` | `persisted_memory` | `memory` | `memory:{cube}:{memory_id}` | 完整时为正文;引用/来源未知时为 `{"memory_id": ..., "cube_id": ...}` | -| `enhanced_memory` | `enhanced_memory` | `memory` | `enhanced_memory:{trace}:{cube}:{memory_id}` | 生成时为正文;后续持久化引用时为 ID/cube 字典 | -| `scheduler_result` | `scheduler_result` | `str` | `scheduler_result:{operation_id or 'unknown'}:{status}` | `"{operation_name} completed"` 或 `"{operation_name} failed"` | +| 消息批次 | `message_batch` | `message_batch` | `message_batch:{trace}:{uuid4.hex}` | messages 的快照 | +| 提取候选 | `extracted_memory` | `memory` | `extracted_memory:{trace}:{cube}:{memory_id}` | 记忆正文 | +| 写入锚点 / 存储引用 | `persisted_memory` | `memory` | `memory:{cube}:{memory_id}` | 完整时为正文;引用/来源未知时为 `{"memory_id": ..., "cube_id": ...}` | +| 增强记忆 | `enhanced_memory` | `memory` | `enhanced_memory:{trace}:{cube}:{memory_id}` | 生成时为正文;后续持久化引用时为 ID/cube 字典 | +| 调度观测状态 | `scheduler_result` | `str` | `scheduler_result:{operation_id or 'unknown'}:{status}` | `"{operation_name} completed"` 或 `"{operation_name} failed"` | 正文统一优先取 `memory` 字段,仅当其为 `None` 时取 `text`;空字符串是有效正文。Add 遍历记录时要求同时有 ID 和正文,ID 优先取 `memory_id`,再取 `id`;不会递归遍历 `metadata/info/internal_info` 包装字段。 @@ -108,11 +123,11 @@ Recorder 对导出的 `persisted_memory.metadata` 额外添加 `snapshot_status= ### 2.2 SearchHandler:查询与各阶段候选 -| `TraceValue.name` | `category` | `class_name` | `identity` | `value` | +| 业务含义 | `category` | `class_name` | `identity` | `value` | | --- | --- | --- | --- | --- | -| `query` | `search_query` | `str` | `search_query:{search_scope}` | 请求 query;无字段时默认空字符串 | -| `{stage}_{result_type}_{rank}` | `search_memory` | `search_memory` | `search_memory:{search_scope}:{stage}:{result_type}:{cube_scope}:{occurrence_id}:{global_position}` | `memory_id`、`memory`、`result_type`、`cube_id`、`score` 五字段字典 | -| `{stage}_filtered` | `search_filter_result` | `str` | `search_filter_result:{search_scope}:{stage}` | 固定字符串 `"filtered"` | +| 搜索查询 | `search_query` | `str` | `search_query:{search_scope}` | 请求 query;无字段时默认空字符串 | +| 阶段候选 | `search_memory` | `search_memory` | `search_memory:{search_scope}:{stage}:{result_type}:{cube_scope}:{occurrence_id}:{global_position}` | `memory_id`、`memory`、`result_type`、`cube_id`、`score` 五字段字典 | +| 候选移除状态 | `search_filter_result` | `str` | `search_filter_result:{search_scope}:{stage}` | 固定字符串 `"filtered"` | | 搜索身份/位置字段 | 规则 | | --- | --- | @@ -166,7 +181,7 @@ SHA256(UTF8(JSON([result_type, cube_scope, ["id", str(memory_id)]]))) | 字段 | 取值 | | --- | --- | -| `name / category` | 均为 `operation_status` | +| `category` | `operation_status` | | `class_name / value` | `str` / `"failed"` | | `identity` | `operation_status:{operation_id or trace_id}:{stage}` | | `metadata` | `memos_stage=stage`、`status="failed"` | @@ -347,6 +362,8 @@ session_metadata = { `user_id` 由 graph 持有并进入 SmartComment 对象的顶层字段,不在以上公共 metadata 字典中。link 未指定 category/comment 时沿用 operation;`event.category` 同时作为 operation 和 session 的 category,**不会把节点 category 改成操作 category**。 +内置 handler 为事件及每条边分别提供说明;自定义 `TraceLink.comment=None` 仍继承操作说明。说明文字不放进 metadata,也不参与节点身份或跨阶段匹配。 + 目前只有持久化配对边显式添加 link metadata,其他 handler 边使用空字典并继承 operation metadata。 ## 6. 快照、导出与边界 @@ -371,6 +388,14 @@ session_metadata = { ## 7. 查代码与扩展时的检查点 +阅读顺序: + +- `add_handler.py`:同步 Add 回调 → MemRead 回调及调度流程 → 消息关联状态 → 节点构造 → 来源配对。 +- `search_handler.py`:按阶段排列的回调 → 共用阶段处理流程 → operation 关联及清理 → 候选快照 → 跨阶段匹配。 +- `base.py`:上下文字段适配 → 快照/事件构造与提交 → Hook 关联 → 公用失败节点与去重。 +- `adapter.py`:请求侧 start/submit/flush/close → 工作线程的快照读写 → 缓存管理与处理循环。 +- `recorder.py`:值注册与显式连边 → 占位补全 → 当前格式恢复与导出。 + | 问题 | 主要实现 | | --- | --- | | trace/cube 从哪里来 | [base.py](handlers/base.py):`_current_trace_id`、`_cube_ids`、`_hook_subject` | diff --git a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py index 952c7461d..594cbe3ac 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py @@ -89,6 +89,7 @@ def __init__( self._state_lock = threading.Lock() self._stats = AdapterStats() + # Request-facing lifecycle: bounded, non-blocking submission and explicit drain/close. @property def stats(self) -> AdapterStats: with self._state_lock: @@ -125,6 +126,32 @@ def submit(self, event: TraceEvent) -> bool: logger.warning("Dropping smartcomment event because the queue is full: %s", event.operation) return False + # A successful flush means tasks finished; stats.failed reports failed writes. + def flush(self, timeout: float = 5.0) -> bool: + deadline = time.monotonic() + timeout + with self._queue.all_tasks_done: + while self._queue.unfinished_tasks: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._queue.all_tasks_done.wait(remaining) + return True + + # Drain accepted events; a timeout warns without interrupting an active write. + def close(self, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + with self._state_lock: + self._closed = True + # Accepted events must drain even if start() was never called. + if not self._started and not self._queue.empty(): + self._start_worker() + thread = self._thread + if thread is not None: + thread.join(timeout=max(0, deadline - time.monotonic())) + if thread.is_alive(): + logger.warning("smartcomment writer did not stop within %.1f seconds", timeout) + + # Worker-owned persistence: restore durable snapshots and atomically replace them. def output_path(self, trace_id: str, user_id: str | None = None) -> Path: user_part = _safe_path_component(user_id, "anonymous") trace_part = _safe_path_component(trace_id, "trace") @@ -152,6 +179,7 @@ def _export(self, event: TraceEvent, recorder: Recorder) -> int: data = exported.get("data", {}) return sum(len(data.get(key, ())) for key in ("nodes", "edges", "operations", "sessions")) + # Worker-owned cache: keep only durable graphs and evict within configured limits. # Evict the least recently used graph until all limits are satisfied. def _trim_cache(self) -> None: now = time.monotonic() @@ -209,28 +237,3 @@ def _run(self) -> None: finally: self._recorders.clear() self._cached_graph_items = 0 - - # A successful flush means tasks finished; stats.failed reports failed writes. - def flush(self, timeout: float = 5.0) -> bool: - deadline = time.monotonic() + timeout - with self._queue.all_tasks_done: - while self._queue.unfinished_tasks: - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._queue.all_tasks_done.wait(remaining) - return True - - # Drain accepted events; a timeout warns without interrupting an active write. - def close(self, timeout: float = 5.0) -> None: - deadline = time.monotonic() + timeout - with self._state_lock: - self._closed = True - # Accepted events must drain even if start() was never called. - if not self._started and not self._queue.empty(): - self._start_worker() - thread = self._thread - if thread is not None: - thread.join(timeout=max(0, deadline - time.monotonic())) - if thread.is_alive(): - logger.warning("smartcomment writer did not stop within %.1f seconds", timeout) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/events.py b/apps/memtrace-memos-integration/memos_smartcomment/events.py index c058da516..78cf981a1 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/events.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/events.py @@ -11,10 +11,10 @@ class TraceValue: """One semantic variable consumed or produced by a MemOS operation.""" - name: str value: Any - # Identity controls node reuse; name is a descriptive label. + # Identity becomes the exported node name and controls reuse within a class_name. identity: str + # Category is the business role; class_name is the node's type/identity namespace. category: str = "variable" class_name: str | None = None # Reference an existing node, or create a placeholder if it has not arrived yet. @@ -55,3 +55,5 @@ class TraceEvent: # event_id identifies this observation; operation_id is correlation metadata. event_id: str = field(default_factory=lambda: uuid4().hex) created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat()) + # Human-readable operation meaning; omitted comments use the recorder's default. + comment: str | None = None diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py index c133b2598..688780cf1 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -28,7 +28,7 @@ _MEM_READ_SOURCE_PREFIX = "scheduler.mem_read." -# Only infer a cube when ownership is unambiguous. +# Input adapters: preserve record order and only infer unambiguous cube ownership. def _single_cube_id(subject: Any) -> str | None: cube_ids = _cube_ids(subject) return cube_ids[0] if len(cube_ids) == 1 else None @@ -85,6 +85,7 @@ def clear(self) -> None: with self._message_batches_lock: self._message_batches.clear() + # Synchronous add Hooks: messages -> extracted candidates -> observed write results. def on_add_before(self, *, request: Any, **_kwargs: Any) -> None: if bool(_get(request, "is_feedback", False)): return @@ -105,6 +106,7 @@ def on_add_before(self, *, request: Any, **_kwargs: Any) -> None: request, operation="memos.add.message_input", category="memory_add_request", + comment="Capture the message batch submitted for memory extraction.", outputs=(message,), metadata=metadata, ) @@ -161,10 +163,16 @@ def on_mem_reader_extract_after( subject, operation="memos.mem_reader.extract", category="memory_extraction", + comment="Extract memory candidates from this message batch; no write is implied.", inputs=(message,), outputs=extracted, links=tuple( - TraceLink(source=message, target=memory, category="memory_extraction") + TraceLink( + source=message, + target=memory, + category="memory_extraction", + comment="Extract this memory candidate from the source message batch.", + ) for memory in extracted ), metadata=metadata, @@ -196,9 +204,17 @@ def on_mem_reader_extract_failed( subject, operation="memos.mem_reader.extract.failed", category="memory_extraction", + comment="Observe an exception at the MemReader extraction boundary.", inputs=(message,), outputs=(status,), - links=(TraceLink(source=message, target=status, category="operation_failed"),), + links=( + TraceLink( + source=message, + target=status, + category="operation_failed", + comment="Extraction from this message batch raised an observed exception.", + ), + ), metadata={ "memos_stage": "mem_reader.extract.failed", "status": "failed", @@ -234,6 +250,10 @@ def on_text_memory_add_after( subject, operation="memos.text_memory.persist", category="memory_persistence", + comment=( + "Observe returned memory IDs and match them to unique write inputs; " + "database persistence is not independently verified." + ), inputs=self._unique_values(inputs), outputs=self._unique_values(outputs), links=tuple(links), @@ -269,10 +289,22 @@ def on_text_memory_add_failed( subject, operation="memos.text_memory.persist.failed", category="memory_persistence", + comment=( + "Observe an exception at the memory-write boundary; " + "earlier writes may already have taken effect." + ), inputs=tuple(inputs), outputs=(status,), links=tuple( - TraceLink(source=value, target=status, category="operation_failed") + TraceLink( + source=value, + target=status, + category="operation_failed", + comment=( + "Writing this input was part of an operation that raised an " + "observed exception; this does not imply rollback." + ), + ) for value in inputs ), metadata={ @@ -284,6 +316,7 @@ def on_text_memory_add_failed( ) ) + # MemRead Hooks: dispatch observed background operations without repeating their work. def on_scheduler_memory_operation_after( self, *, @@ -320,6 +353,140 @@ def on_scheduler_memory_operation_failed( status="failed", ) + def _on_scheduler_memory_operation( + self, + *, + hook_context: HookContext, + operation: Any, + target: Any, + operation_input: Any, + result: Any | None = None, + error: BaseException | None = None, + status: str, + ) -> None: + source = str(_get(hook_context, "source", "")) + # Other handlers share this Hook; only interpret MemRead operations. + if not source.startswith(_MEM_READ_SOURCE_PREFIX): + return + + operation_name = source.removeprefix(_MEM_READ_SOURCE_PREFIX) + description = { + "fine_transfer_simple_mem": "Refine source memories into enhanced memories", + "add_enhanced_memories": "Write enhanced memories and observe returned memory IDs", + "archive_merged_memories": "Mark merged source memories as archived", + "remove_memories": "Request deletion of source memories", + "remove_source_memories": "Request deletion of source memories", + }.get(operation_name, f"Run MemRead operation {operation_name}") + outcome = "the method returned" if status == "after" else "an exception reached the Hook" + operation_comment = ( + f"{description} (backend operation: {operation}); {outcome}. " + "Database effects are not independently verified." + ) + if not isinstance(operation_input, Mapping): + operation_input = {} + subject = self._hook_subject(hook_context, user_name=operation_input.get("user_name")) + cube_id = _single_cube_id(subject) + + inputs: list[TraceValue] = [] + outputs: list[TraceValue] = [] + links: list[TraceLink] = [] + if status == "after" and operation_name == "fine_transfer_simple_mem": + inputs, outputs, links = self._fine_transfer_values( + subject, + operation_input, + result, + cube_id=cube_id, + ) + elif operation_name == "add_enhanced_memories": + inputs, outputs, links = self._persistence_values( + subject, + operation_input.get("memories"), + result if status == "after" else (), + cube_id=cube_id, + enhanced=True, + ) + lineage = self._persistence_lineage(outputs, links) + else: + inputs = self._scheduler_persisted_inputs( + operation_name, + operation_input, + cube_id=cube_id, + ) + + # Operations without memory outputs still have an observable status. + if not outputs: + if status == "failed": + category = "scheduler_operation_failed" + comment = ( + "This memory was an input to a MemRead operation that raised an " + "observed exception; this does not imply rollback." + ) + elif operation_name == "archive_merged_memories": + category = "memory_archival" + comment = ( + "The archive method returned for this source memory; " + "the plugin does not independently verify its stored status." + ) + elif operation_name in {"remove_memories", "remove_source_memories"}: + category = "memory_deletion" + comment = ( + f"The {operation} method returned with this memory among its inputs; " + "this does not confirm removal, and soft-delete may preserve selected IDs." + ) + else: + category = "scheduler_operation_result" + comment = ( + f"The {operation_name} method returned without a memory output " + "for this observed input group." + ) + result_value = self._scheduler_result_value( + hook_context, + operation_name=operation_name, + status=status, + comment=operation_comment, + ) + outputs.append(result_value) + links.extend( + TraceLink( + source=value, + target=result_value, + category=category, + comment=comment, + ) + for value in inputs + ) + + metadata = { + "memos_stage": f"{source}.{status}", + "handler": "mem_read", + "operation_name": operation_name, + "operation": operation, + "target": target, + "status": status, + } + if operation_name == "fine_transfer_simple_mem": + metadata["result_grouping"] = operation_input.get("result_grouping", "unknown") + if status == "after" and any(value.category == "enhanced_memory" for value in outputs): + metadata["lineage_status"] = "resolved" if links else "unresolved" + elif operation_name == "add_enhanced_memories" and status == "after": + metadata["lineage_status"] = lineage + if status == "failed": + # Record only the exception type; messages can expose business data. + metadata["error_type"] = type(error).__name__ + self._emit( + self._event( + subject, + operation=f"memos.{source}.{status}", + category="scheduler_memory_operation", + comment=operation_comment, + inputs=self._unique_values(inputs), + outputs=self._unique_values(outputs), + links=tuple(links), + metadata=metadata, + ) + ) + + # Message correlation: creation registers a batch; references consume one cube's claim. @staticmethod def _message_batch_key(subject: Any, messages: Any) -> tuple[str, str | None, int]: return _current_trace_id(subject), _get(subject, "user_id"), id(messages) @@ -369,6 +536,7 @@ def _messages_from_scene_data(scene_data: Any) -> Any: return messages return scene_data + # Node construction: identity selects the anchor; comment explains the captured value. def _message_value( self, subject: Any, @@ -378,13 +546,12 @@ def _message_value( metadata: dict[str, Any] | None = None, ) -> TraceValue: return self._value( - name="messages", value=messages, identity=self._message_identity(subject, messages, reference=identity_only), category="message_batch", class_name="message_batch", identity_only=identity_only, - comment=("Message batch received at MemOS stage add.before."), + comment="Message batch supplied to MemOS for memory extraction.", metadata=metadata, ) @@ -400,13 +567,12 @@ def _extracted_memory_value( ) -> TraceValue: trace_id = _current_trace_id(subject) return self._value( - name="extracted_memory", value=_memory_text(memory), identity=f"extracted_memory:{trace_id}:{cube_id or 'unknown'}:{memory_id}", category="extracted_memory", class_name="memory", identity_only=identity_only, - comment=("Memory candidate produced at MemOS stage mem_reader.extract.after."), + comment="Memory candidate from MemReader extraction; not a write confirmation.", metadata=metadata, ) @@ -420,7 +586,6 @@ def _persisted_memory_value( stage: str = "text_memory.add.after", ) -> TraceValue: return self._value( - name="persisted_memory", value=( _memory_text(memory) if memory is not None @@ -431,7 +596,8 @@ def _persisted_memory_value( class_name="memory", identity_only=memory is None, comment=( - f"Textual memory successfully persisted at MemOS stage {stage}." + f"Memory snapshot matched to an ID returned at {stage}; " + "database persistence is not independently verified." if memory is not None else f"Write returned this memory ID at {stage}; its source and content are unresolved." ), @@ -449,13 +615,15 @@ def _persisted_memory_reference( memory_id: str, ) -> TraceValue: return self._value( - name="persisted_memory", value={"memory_id": memory_id, "cube_id": cube_id}, identity=f"memory:{cube_id or 'unknown'}:{memory_id}", category="persisted_memory", class_name="memory", identity_only=True, - comment="Reference to a memory unit previously persisted by MemOS.", + comment=( + "Memory ID supplied to a MemRead operation; " + "this reference alone does not confirm its content or database state." + ), metadata={"cube_id": cube_id, "memory_id": memory_id}, ) @@ -472,7 +640,6 @@ def _enhanced_memory_value( {"memory_id": memory_id, "cube_id": cube_id} if identity_only else _memory_text(memory) ) return self._value( - name="enhanced_memory", value=value, identity=( f"enhanced_memory:{_current_trace_id(subject)}:{cube_id or 'unknown'}:{memory_id}" @@ -490,17 +657,17 @@ def _scheduler_result_value( *, operation_name: str, status: str, + comment: str, ) -> TraceValue: operation_id = _get(hook_context, "operation_id") or "unknown" result = f"{operation_name} {'completed' if status == 'after' else 'failed'}" return self._value( - name="scheduler_result", value=result, identity=f"scheduler_result:{operation_id}:{status}", category="scheduler_result", class_name="str", identity_only=False, - comment=f"String result for MemRead scheduler operation {operation_name}.", + comment=comment, metadata={ "operation_id": operation_id, "operation_name": operation_name, @@ -508,6 +675,7 @@ def _scheduler_result_value( }, ) + # Lineage construction: connect only grouping- or ID-backed sources, never guess by position. def _fine_transfer_values( self, subject: Any, @@ -550,7 +718,10 @@ def _fine_transfer_values( source=source, target=target, category="memory_refinement", - comment="Refine persisted memory in the MemRead scheduler.", + comment=( + "This source memory contributed to the fine-transfer group " + "that produced the target enhanced memory." + ), ) for source in source_values ) @@ -610,7 +781,11 @@ def _persistence_values( source=inputs[source_index], target=target, category="memory_persistence", - comment=f"Persist the memory with the matching ID at {stage}.", + comment=( + f"Match this input memory to the ID returned at {stage}; " + "the edge records write-result lineage, not independent " + "database confirmation." + ), metadata={"pair_index": index, "input_index": source_index}, ) ) @@ -644,110 +819,3 @@ def _scheduler_persisted_inputs( self._persisted_memory_reference(cube_id=cube_id, memory_id=str(memory_id)) for memory_id in memory_ids ] - - def _on_scheduler_memory_operation( - self, - *, - hook_context: HookContext, - operation: Any, - target: Any, - operation_input: Any, - result: Any | None = None, - error: BaseException | None = None, - status: str, - ) -> None: - source = str(_get(hook_context, "source", "")) - # Other handlers share this Hook; only interpret MemRead operations. - if not source.startswith(_MEM_READ_SOURCE_PREFIX): - return - - operation_name = source.removeprefix(_MEM_READ_SOURCE_PREFIX) - if not isinstance(operation_input, Mapping): - operation_input = {} - subject = self._hook_subject(hook_context, user_name=operation_input.get("user_name")) - cube_id = _single_cube_id(subject) - - inputs: list[TraceValue] = [] - outputs: list[TraceValue] = [] - links: list[TraceLink] = [] - if status == "after" and operation_name == "fine_transfer_simple_mem": - inputs, outputs, links = self._fine_transfer_values( - subject, - operation_input, - result, - cube_id=cube_id, - ) - elif operation_name == "add_enhanced_memories": - inputs, outputs, links = self._persistence_values( - subject, - operation_input.get("memories"), - result if status == "after" else (), - cube_id=cube_id, - enhanced=True, - ) - lineage = self._persistence_lineage(outputs, links) - else: - inputs = self._scheduler_persisted_inputs( - operation_name, - operation_input, - cube_id=cube_id, - ) - - # Operations without memory outputs still have an observable status. - if not outputs: - if status == "failed": - category = "scheduler_operation_failed" - comment = "The MemRead scheduler operation failed while processing this memory." - elif operation_name == "archive_merged_memories": - category = "memory_archival" - comment = f"Apply {operation_name} to a persisted memory unit." - elif operation_name in {"remove_memories", "remove_source_memories"}: - category = "memory_deletion" - comment = f"Apply {operation_name} to a persisted memory unit." - else: - category = "scheduler_operation_result" - comment = f"Produce the result of {operation_name} from its memory input." - result_value = self._scheduler_result_value( - hook_context, - operation_name=operation_name, - status=status, - ) - outputs.append(result_value) - links.extend( - TraceLink( - source=value, - target=result_value, - category=category, - comment=comment, - ) - for value in inputs - ) - - metadata = { - "memos_stage": f"{source}.{status}", - "handler": "mem_read", - "operation_name": operation_name, - "operation": operation, - "target": target, - "status": status, - } - if operation_name == "fine_transfer_simple_mem": - metadata["result_grouping"] = operation_input.get("result_grouping", "unknown") - if status == "after" and any(value.category == "enhanced_memory" for value in outputs): - metadata["lineage_status"] = "resolved" if links else "unresolved" - elif operation_name == "add_enhanced_memories" and status == "after": - metadata["lineage_status"] = lineage - if status == "failed": - # Record only the exception type; messages can expose business data. - metadata["error_type"] = type(error).__name__ - self._emit( - self._event( - subject, - operation=f"memos.{source}.{status}", - category="scheduler_memory_operation", - inputs=self._unique_values(inputs), - outputs=self._unique_values(outputs), - links=tuple(links), - metadata=metadata, - ) - ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py index dc169360f..c1dd14205 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py @@ -18,6 +18,7 @@ _MISSING_TRACE = object() +# Context adapters: resolve current Hook/request shapes without borrowing an empty trace. def _get(value: Any, name: str, default: Any | None = None) -> Any: if isinstance(value, Mapping): return value.get(name, default) @@ -80,14 +81,13 @@ def __init__( self._submit = submit self._max_value_chars = max_value_chars - # Snapshot synchronously so the worker cannot observe later business mutations. + # Detached values and events: capture business data before handing it to the worker. def _snapshot(self, value: Any) -> Any: return to_jsonable(value, max_value_chars=self._max_value_chars) def _value( self, *, - name: str, value: Any, identity: str, category: str, @@ -97,7 +97,6 @@ def _value( metadata: dict[str, Any] | None = None, ) -> TraceValue: return TraceValue( - name=name, value=self._snapshot(value), identity=identity, category=category, @@ -113,6 +112,7 @@ def _event( *, operation: str, category: str, + comment: str, inputs: tuple[TraceValue, ...] = (), outputs: tuple[TraceValue, ...] = (), links: tuple[TraceLink, ...] = (), @@ -124,6 +124,7 @@ def _event( return TraceEvent( operation=operation, category=category, + comment=comment, trace_id=_current_trace_id(subject), session_id=_get(subject, "session_id"), task_id=_get(subject, "task_id"), @@ -142,7 +143,7 @@ def _emit(self, event: TraceEvent) -> None: except Exception: logger.exception("Failed to enqueue smartcomment event: %s", event.operation) - # Here user_name identifies the storage cube, not the user. + # Hook correlation: user_name identifies the storage cube, not the user. @staticmethod def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, Any]: subject = { @@ -165,16 +166,19 @@ def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, An subject.setdefault("cube_id", user_name) return subject + # Shared terminal nodes and identity-preserving deduplication. def _operation_status_value(self, subject: Any, *, stage: str) -> TraceValue: operation_id = _get(subject, "operation_id") or _current_trace_id(subject) return self._value( - name="operation_status", value="failed", identity=f"operation_status:{operation_id}:{stage}", category="operation_status", class_name="str", identity_only=False, - comment=f"MemOS operation failed at stage {stage}.", + comment=( + f"An exception reached the {stage} Hook boundary; " + "this status does not imply rollback of earlier side effects." + ), metadata={"memos_stage": stage, "status": "failed"}, ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index 0f699fe86..72ecd26ec 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping from dataclasses import replace from threading import Lock -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypedDict from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue from memos_smartcomment.handlers.base import ( @@ -32,6 +32,20 @@ } +class _SearchItem(TypedDict): + """One candidate flattened from a result-type/cube bucket, before snapshotting.""" + + result_type: str + bucket_index: int + cube_id: str | None + memory_id: Any + memory: Any + rank: int + rank_scope: str + global_position: int + score: Any + + class SearchHandler(BaseHandler): """Capture Search API stages with operation-scoped candidate correlation.""" @@ -50,6 +64,7 @@ def clear(self) -> None: with self._search_stage_items_lock: self._search_stage_nodes.clear() + # Search Hooks in pipeline order; the failure Hook closes the same operation scope. def on_search_before( self, *, @@ -66,6 +81,7 @@ def on_search_before( subject, operation="memos.search.request", category="search_request", + comment="Capture the query submitted to the MemOS Search API.", outputs=(query,), metadata={"memos_stage": "search.before"}, ) @@ -85,6 +101,7 @@ def on_search_memory_results( hook_context=hook_context, operation="memos.search.retrieve", category="memory_retrieval", + comment="Observe the memory candidates returned for this query before post-processing.", stage="raw", previous_stage=None, metadata={ @@ -108,6 +125,10 @@ def on_search_results_after_threshold( hook_context=hook_context, operation="memos.search.threshold_filter", category="memory_filtering", + comment=( + "Observe candidates after the threshold stage; " + "metadata.applied indicates whether filtering was enabled." + ), stage="threshold", previous_stage="raw", metadata={ @@ -133,6 +154,10 @@ def on_search_results_after_dedup( hook_context=hook_context, operation="memos.search.deduplicate", category="memory_deduplication", + comment=( + "Observe candidates after the deduplication stage; " + "metadata.applied indicates whether deduplication was enabled." + ), stage="dedup", previous_stage="threshold", metadata={ @@ -157,6 +182,7 @@ def on_search_results_after_rerank( hook_context=hook_context, operation="memos.search.rerank", category="memory_reranking", + comment="Observe candidate order and content after the Search API rerank stage.", stage="rerank", previous_stage="dedup", metadata={ @@ -186,6 +212,7 @@ def on_search_post_process_failed( subject, operation=f"memos.{stage}", category="search_failure", + comment="Observe an exception during Search API post-processing for this query.", inputs=(query,), outputs=(status,), links=( @@ -193,7 +220,7 @@ def on_search_post_process_failed( source=query, target=status, category="operation_failed", - comment=f"Search processing failed at MemOS stage {stage}.", + comment="Post-processing this query raised an observed exception.", ), ), metadata={ @@ -206,6 +233,121 @@ def on_search_post_process_failed( ) self._clear_search_stages(subject) + # Stage processing: build nodes, connect known sources, then publish and clean up. + def _on_search_results( + self, + *, + request: Any, + results: Any, + hook_context: HookContext | None, + operation: str, + category: str, + comment: str, + stage: str, + previous_stage: str | None, + metadata: dict[str, Any], + ) -> None: + if request is None: + return + subject = self._search_subject(hook_context, request) + current_items = self._search_result_items(results) + with self._search_stage_items_lock: + previous_nodes = ( + self._search_stage_nodes.get( + self._search_stage_key(subject, previous_stage), + (), + ) + if previous_stage is not None + else () + ) + + inputs: tuple[TraceValue, ...] + outputs: tuple[TraceValue, ...] + links: tuple[TraceLink, ...] + removed_nodes: tuple[TraceValue, ...] = () + current_nodes = tuple( + self._search_memory_value(subject, stage=stage, item=item) for item in current_items + ) + + # Only the first stage links candidates to the query. + if previous_stage is None: + query = self._search_query_value(subject, request, identity_only=True) + inputs = (query,) + outputs = current_nodes + links = tuple( + TraceLink( + source=query, + target=node, + category="memory_retrieval", + comment="This query returned the target memory candidate in raw search results.", + ) + for node in current_nodes + ) + else: + matched_nodes, removed_nodes = self._match_search_nodes( + previous_nodes, + current_nodes, + ) + + paired_links: list[TraceLink] = [] + for source, candidate in zip(matched_nodes, current_nodes, strict=True): + if source is not None: + paired_links.append( + TraceLink( + source=replace(source, identity_only=True), + target=candidate, + category=_SEARCH_LINK_CATEGORIES[stage], + comment=( + f"Match the same candidate before and after search stage={stage}; " + "the target records this stage's content, score, and rank." + ), + ) + ) + # Keep unmatched candidates, but do not invent their provenance. + outputs = current_nodes + if removed_nodes: + filtered = self._search_filter_result_value(subject, stage=stage) + outputs += (filtered,) + paired_links.extend( + TraceLink( + source=replace(node, identity_only=True), + target=filtered, + category="memory_filtered", + comment=( + f"This candidate is absent after search stage={stage}; " + "no deletion from memory storage is implied." + ), + ) + for node in removed_nodes + ) + links = tuple(paired_links) + inputs = self._unique_values([link.source for link in links]) + + with self._search_stage_items_lock: + self._search_stage_nodes[self._search_stage_key(subject, stage)] = current_nodes + + event_metadata = { + **metadata, + "link_strategy": "explicit_pairwise", + "filtered_count": len(removed_nodes), + } + self._emit( + self._event( + subject, + operation=operation, + category=category, + comment=comment, + inputs=inputs, + outputs=outputs, + links=links, + metadata=event_metadata, + ) + ) + # Rerank and failure are terminal stages for this operation's cache. + if stage == "rerank": + self._clear_search_stages(subject) + + # Operation correlation: one trace may contain multiple independent searches. def _search_subject( self, hook_context: HookContext | None, @@ -232,6 +374,16 @@ def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: stage, ) + def _clear_search_stages(self, subject: Any) -> None: + trace_id = _current_trace_id(subject) + search_scope_id = self._search_scope_id(subject) + with self._search_stage_items_lock: + for key in [ + key for key in self._search_stage_nodes if key[:2] == (trace_id, search_scope_id) + ]: + del self._search_stage_nodes[key] + + # Candidate snapshots: keep stage-local rank separate from cross-stage identity. def _search_query_value( self, subject: Any, @@ -241,23 +393,22 @@ def _search_query_value( ) -> TraceValue: search_scope_id = self._search_scope_id(subject) return self._value( - name="query", value=_get(request, "query", ""), identity=f"search_query:{search_scope_id}", category="search_query", class_name="str", identity_only=identity_only, - comment="User Query submitted to the MemOS Search API.", + comment="Query submitted to the MemOS Search API for memory retrieval.", metadata={"memos_stage": "search.before"}, ) # Flatten buckets while retaining bucket-local rank and traversal order. @staticmethod - def _search_result_items(results: Any) -> list[dict[str, Any]]: + def _search_result_items(results: Any) -> list[_SearchItem]: if not isinstance(results, Mapping): return [] - items: list[dict[str, Any]] = [] + items: list[_SearchItem] = [] global_position = 0 for result_type, buckets in results.items(): if not isinstance(buckets, list | tuple): @@ -297,7 +448,7 @@ def _search_memory_value( subject: Any, *, stage: str, - item: Mapping[str, Any], + item: _SearchItem, ) -> TraceValue: search_scope_id = self._search_scope_id(subject) result_type = item["result_type"] @@ -312,16 +463,14 @@ def _search_memory_value( cube_scope = cube_id if cube_id is not None else f"bucket-{bucket_index}" score = item["score"] comment_parts = [ - f"Search result memory at stage={stage}", - f"rank={rank}", - f"rank_scope={rank_scope}", - f"global_position={global_position}", + f"Memory candidate observed at search stage={stage}", + f"rank={rank} within result-type/cube bucket {rank_scope} (1-based)", + f"global_position={global_position} is traversal order, not a cross-cube rank", ] if score is not None: comment_parts.append(f"score={score}") node = self._value( - name=f"{stage}_{result_type}_{rank}", value={ "memory_id": memory_id, "memory": item["memory"], @@ -352,9 +501,25 @@ def _search_memory_value( # Match using the original item, never the truncated snapshot. return replace(node, match_key=self._search_memory_match_key(item)) - # Match by type, cube and ID, or by original text when no ID exists. + # Removed candidates share one terminal node per stage. + def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: + search_scope_id = self._search_scope_id(subject) + return self._value( + value="filtered", + identity=f"search_filter_result:{search_scope_id}:{stage}", + category="search_filter_result", + class_name="str", + identity_only=False, + comment=( + f"Previous-stage candidates no longer present after search stage={stage}; " + "this status does not mean deletion from memory storage." + ), + metadata={"stage": stage, "status": "filtered"}, + ) + + # Cross-stage lineage: match by type/cube/ID or original text, consuming duplicates FIFO. @staticmethod - def _search_memory_match_key(value: Mapping[str, Any]) -> str: + def _search_memory_match_key(value: _SearchItem) -> str: memory_id = value["memory_id"] identity = ("id", str(memory_id)) if memory_id is not None else ("text", value["memory"]) cube_id = value["cube_id"] @@ -384,131 +549,3 @@ def _match_search_nodes( removed = tuple(node for node in previous_nodes if node.identity not in matched_identities) return tuple(matched), removed - - # Removed candidates share one terminal node per stage. - def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: - search_scope_id = self._search_scope_id(subject) - return self._value( - name=f"{stage}_filtered", - value="filtered", - identity=f"search_filter_result:{search_scope_id}:{stage}", - category="search_filter_result", - class_name="str", - identity_only=False, - comment=f"Memory was removed from Search API results at stage={stage}.", - metadata={"stage": stage, "status": "filtered"}, - ) - - def _on_search_results( - self, - *, - request: Any, - results: Any, - hook_context: HookContext | None, - operation: str, - category: str, - stage: str, - previous_stage: str | None, - metadata: dict[str, Any], - ) -> None: - if request is None: - return - subject = self._search_subject(hook_context, request) - current_items = self._search_result_items(results) - with self._search_stage_items_lock: - previous_nodes = ( - self._search_stage_nodes.get( - self._search_stage_key(subject, previous_stage), - (), - ) - if previous_stage is not None - else () - ) - - inputs: tuple[TraceValue, ...] - outputs: tuple[TraceValue, ...] - links: tuple[TraceLink, ...] - removed_nodes: tuple[TraceValue, ...] = () - current_nodes = tuple( - self._search_memory_value(subject, stage=stage, item=item) for item in current_items - ) - - # Only the first stage links candidates to the query. - if previous_stage is None: - query = self._search_query_value(subject, request, identity_only=True) - inputs = (query,) - outputs = current_nodes - links = tuple( - TraceLink( - source=query, - target=node, - category="memory_retrieval", - comment="Search query retrieved this memory candidate.", - ) - for node in current_nodes - ) - else: - matched_nodes, removed_nodes = self._match_search_nodes( - previous_nodes, - current_nodes, - ) - - paired_links: list[TraceLink] = [] - for source, candidate in zip(matched_nodes, current_nodes, strict=True): - if source is not None: - paired_links.append( - TraceLink( - source=replace(source, identity_only=True), - target=candidate, - category=_SEARCH_LINK_CATEGORIES[stage], - comment=f"Carry this Search candidate into stage={stage}.", - ) - ) - # Keep unmatched candidates, but do not invent their provenance. - outputs = current_nodes - if removed_nodes: - filtered = self._search_filter_result_value(subject, stage=stage) - outputs += (filtered,) - paired_links.extend( - TraceLink( - source=replace(node, identity_only=True), - target=filtered, - category="memory_filtered", - comment=f"This memory was filtered out at stage={stage}.", - ) - for node in removed_nodes - ) - links = tuple(paired_links) - inputs = self._unique_values([link.source for link in links]) - - with self._search_stage_items_lock: - self._search_stage_nodes[self._search_stage_key(subject, stage)] = current_nodes - - event_metadata = { - **metadata, - "link_strategy": "explicit_pairwise", - "filtered_count": len(removed_nodes), - } - self._emit( - self._event( - subject, - operation=operation, - category=category, - inputs=inputs, - outputs=outputs, - links=links, - metadata=event_metadata, - ) - ) - # Rerank and failure are terminal stages for this operation's cache. - if stage == "rerank": - self._clear_search_stages(subject) - - def _clear_search_stages(self, subject: Any) -> None: - trace_id = _current_trace_id(subject) - search_scope_id = self._search_scope_id(subject) - with self._search_stage_items_lock: - for key in [ - key for key in self._search_stage_nodes if key[:2] == (trace_id, search_scope_id) - ]: - del self._search_stage_nodes[key] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/recorder.py b/apps/memtrace-memos-integration/memos_smartcomment/recorder.py index c877e8966..4a5e85211 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/recorder.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/recorder.py @@ -54,6 +54,7 @@ def __init__(self, settings: SmartCommentSettings, first_event: TraceEvent) -> N strict=settings.strict, ) + # Event recording: register explicit values and dependencies, never an input/output product. @staticmethod def _as_comment_item(value: TraceValue) -> tuple[Any, dict[str, Any]]: options: dict[str, Any] = { @@ -123,13 +124,21 @@ def register(value: TraceValue) -> Any: session_id=f"event-{event.event_id}", session_name=event.operation, category=event.category, - comment=f"MemOS event for {event.operation}.", + comment=( + event.comment + if event.comment is not None + else f"MemOS event for {event.operation}." + ), metadata=session_metadata, ), self._comment_op_scope( op_name=event.operation, category=event.category, - comment=f"MemOS semantic operation: {event.operation}.", + comment=( + event.comment + if event.comment is not None + else f"MemOS semantic operation: {event.operation}." + ), metadata=operation_metadata, ), ): @@ -147,6 +156,7 @@ def register(value: TraceValue) -> Any: if completions: self._complete_memory_snapshots(completions) + # Current-format snapshots: complete references once, restore, and export for persistence. def _complete_memory_snapshots(self, completions: dict[str, TraceValue]) -> None: """Fill an ID-only placeholder once, preserving its node ID and incident edges.""" exported = self._graph.export_graph() @@ -163,68 +173,16 @@ def _complete_memory_snapshots(self, completions: dict[str, TraceValue]) -> None self.restore_graph(exported) def restore_graph(self, data: dict[str, Any]) -> None: + """Restore a current-format snapshot without changing its identities or edges.""" from smartcomment.runtime import ExecNetwork for field in ("graph_id", "user_id", "project_id"): if data.get(field) != getattr(self._graph, field): raise MemOSError(f"Snapshot {field} does not match the requested graph") - graph = ExecNetwork.import_graph(self._normalize_memory_units(data)) + graph = ExecNetwork.import_graph(data) graph.strict = self._graph.strict self._graph = graph - @staticmethod - def _normalize_memory_units(data: dict[str, Any]) -> dict[str, Any]: - """Reconnect legacy versions to the earliest anchor and first complete snapshot.""" - graph_data = data["data"] - anchors: dict[tuple[str | None, str], dict[str, Any]] = {} - aliases: dict[str, str] = {} - units = [node for node in graph_data["nodes"] if node["category"] == "persisted_memory"] - for node in sorted(units, key=lambda node: node["version"]): - metadata = dict(node.get("metadata", {})) - if "snapshot_status" not in metadata: - is_reference = "memory_id" in metadata and _decode(node["value"]) == { - "memory_id": metadata["memory_id"], - "cube_id": metadata.get("cube_id"), - } - metadata["snapshot_status"] = "reference" if is_reference else "complete" - # Exported names carry the identity supplied by id_strategy. - key = (node.get("class_name"), node["name"]) - if key not in anchors: - anchors[key] = {**node, "metadata": metadata} - anchor = anchors[key] - aliases[node["full_node_id"]] = anchor["full_node_id"] - if ( - anchor["metadata"]["snapshot_status"] == "reference" - and metadata["snapshot_status"] == "complete" - ): - anchor.update(value=node["value"], comment=node.get("comment"), metadata=metadata) - - nodes = [] - for node in graph_data["nodes"]: - if node["category"] != "persisted_memory": - nodes.append(node) - elif aliases[node["full_node_id"]] == node["full_node_id"]: - nodes.append(anchors[(node.get("class_name"), node["name"])]) - return { - **data, - "data": { - **graph_data, - "nodes": nodes, - "edges": [ - { - **edge, - "source_full_node_id": aliases.get( - edge["source_full_node_id"], edge["source_full_node_id"] - ), - "target_full_node_id": aliases.get( - edge["target_full_node_id"], edge["target_full_node_id"] - ), - } - for edge in graph_data["edges"] - ], - }, - } - # Hide SmartComment's internal NONE sentinel and its associated graph elements. def export_graph(self) -> dict[str, Any]: exported = self._graph.export_graph() From 2385eb36113120693e8dbe406d1ce7e5132adebb Mon Sep 17 00:00:00 2001 From: qzs Date: Tue, 6 Oct 2026 16:11:12 +0800 Subject: [PATCH 05/22] refactor(memtrace): clarify graph labels and trace comments --- .../handlers/add_handler.py | 91 +++++++------------ .../memos_smartcomment/handlers/base.py | 9 +- .../handlers/search_handler.py | 19 ++-- 3 files changed, 43 insertions(+), 76 deletions(-) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py index 688780cf1..f62a45c28 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -163,7 +163,7 @@ def on_mem_reader_extract_after( subject, operation="memos.mem_reader.extract", category="memory_extraction", - comment="Extract memory candidates from this message batch; no write is implied.", + comment="Extract memory candidates from this message batch.", inputs=(message,), outputs=extracted, links=tuple( @@ -198,7 +198,7 @@ def on_mem_reader_extract_failed( return subject = self._hook_subject(hook_context, user_name=user_name) message = self._message_value(subject, messages, identity_only=True) - status = self._operation_status_value(subject, stage="mem_reader.extract.failed") + status = self._operation_status_value(subject, error, stage="mem_reader.extract.failed") self._emit( self._event( subject, @@ -250,10 +250,7 @@ def on_text_memory_add_after( subject, operation="memos.text_memory.persist", category="memory_persistence", - comment=( - "Observe returned memory IDs and match them to unique write inputs; " - "database persistence is not independently verified." - ), + comment=("Observe returned memory IDs and match them to unique write inputs."), inputs=self._unique_values(inputs), outputs=self._unique_values(outputs), links=tuple(links), @@ -283,16 +280,13 @@ def on_text_memory_add_failed( ) if not inputs: return - status = self._operation_status_value(subject, stage="text_memory.add.failed") + status = self._operation_status_value(subject, error, stage="text_memory.add.failed") self._emit( self._event( subject, operation="memos.text_memory.persist.failed", category="memory_persistence", - comment=( - "Observe an exception at the memory-write boundary; " - "earlier writes may already have taken effect." - ), + comment=(f"Observe an exception at the memory-write boundary {error!s}."), inputs=tuple(inputs), outputs=(status,), links=tuple( @@ -300,10 +294,7 @@ def on_text_memory_add_failed( source=value, target=status, category="operation_failed", - comment=( - "Writing this input was part of an operation that raised an " - "observed exception; this does not imply rollback." - ), + comment=("Raise an observed exception."), ) for value in inputs ), @@ -374,14 +365,11 @@ def _on_scheduler_memory_operation( "fine_transfer_simple_mem": "Refine source memories into enhanced memories", "add_enhanced_memories": "Write enhanced memories and observe returned memory IDs", "archive_merged_memories": "Mark merged source memories as archived", - "remove_memories": "Request deletion of source memories", - "remove_source_memories": "Request deletion of source memories", - }.get(operation_name, f"Run MemRead operation {operation_name}") + "remove_memories": "deletion of source memories", + "remove_source_memories": "deletion of source memories", + }.get(operation_name, f"Run MemRead Scheduler operation {operation_name}") outcome = "the method returned" if status == "after" else "an exception reached the Hook" - operation_comment = ( - f"{description} (backend operation: {operation}); {outcome}. " - "Database effects are not independently verified." - ) + operation_comment = f"{description} (backend operation: {operation}); {outcome}." if not isinstance(operation_input, Mapping): operation_input = {} subject = self._hook_subject(hook_context, user_name=operation_input.get("user_name")) @@ -417,33 +405,29 @@ def _on_scheduler_memory_operation( if not outputs: if status == "failed": category = "scheduler_operation_failed" - comment = ( - "This memory was an input to a MemRead operation that raised an " - "observed exception; this does not imply rollback." - ) + link_comment = "Raise an observed exception." + value_comment = f"An observed exception {error!s} occurred." elif operation_name == "archive_merged_memories": category = "memory_archival" - comment = ( - "The archive method returned for this source memory; " - "the plugin does not independently verify its stored status." - ) + link_comment = "Archive the memory unit." + value_comment = "The memory unit has been archived successfully." elif operation_name in {"remove_memories", "remove_source_memories"}: category = "memory_deletion" - comment = ( - f"The {operation} method returned with this memory among its inputs; " - "this does not confirm removal, and soft-delete may preserve selected IDs." + link_comment = f"Remove the memory unit through the {operation} operation." + value_comment = ( + f"The memory unit has been removed through the {operation} operation." ) else: category = "scheduler_operation_result" - comment = ( - f"The {operation_name} method returned without a memory output " - "for this observed input group." + link_comment = f"The {operation_name} method is not traced. It does not produce a memory output for this observed input group." + value_comment = ( + f"The memory unit has been processed through the {operation} operation." ) result_value = self._scheduler_result_value( hook_context, operation_name=operation_name, status=status, - comment=operation_comment, + comment=value_comment, ) outputs.append(result_value) links.extend( @@ -451,7 +435,7 @@ def _on_scheduler_memory_operation( source=value, target=result_value, category=category, - comment=comment, + comment=link_comment, ) for value in inputs ) @@ -548,10 +532,10 @@ def _message_value( return self._value( value=messages, identity=self._message_identity(subject, messages, reference=identity_only), - category="message_batch", - class_name="message_batch", + category="input_message_batch", + class_name="input_message_batch", identity_only=identity_only, - comment="Message batch supplied to MemOS for memory extraction.", + comment="The user's original input message batch, without any modifications.", metadata=metadata, ) @@ -572,11 +556,11 @@ def _extracted_memory_value( category="extracted_memory", class_name="memory", identity_only=identity_only, - comment="Memory candidate from MemReader extraction; not a write confirmation.", + comment="Memory candidate after MemReader extraction, not yet persisted.", metadata=metadata, ) - # Share persisted anchors within a graph, including references arriving before the write. + # Share persisted anchors within a graph, including references arriving before the write. def _persisted_memory_value( self, memory: Any, @@ -596,10 +580,9 @@ def _persisted_memory_value( class_name="memory", identity_only=memory is None, comment=( - f"Memory snapshot matched to an ID returned at {stage}; " - "database persistence is not independently verified." + f"Persisted memory unit matched to an ID returned at {stage}." if memory is not None - else f"Write returned this memory ID at {stage}; its source and content are unresolved." + else f"Write returned this memory ID at {stage}. Its source and content are unresolved." ), metadata={ "memos_stage": stage, @@ -620,10 +603,7 @@ def _persisted_memory_reference( category="persisted_memory", class_name="memory", identity_only=True, - comment=( - "Memory ID supplied to a MemRead operation; " - "this reference alone does not confirm its content or database state." - ), + comment=("Referred persisted memory unit."), metadata={"cube_id": cube_id, "memory_id": memory_id}, ) @@ -647,7 +627,7 @@ def _enhanced_memory_value( category="enhanced_memory", class_name="memory", identity_only=identity_only, - comment="Memory produced by the MemRead scheduler's fine-transfer stage.", + comment="Enhanced memory candidate produced by the MemRead Scheduler's fine-transfer stage.", metadata={"cube_id": cube_id, "memory_id": memory_id}, ) @@ -719,8 +699,7 @@ def _fine_transfer_values( target=target, category="memory_refinement", comment=( - "This source memory contributed to the fine-transfer group " - "that produced the target enhanced memory." + "Refine the memory unit through the fine-transfer stage of the MemRead Scheduler." ), ) for source in source_values @@ -781,11 +760,7 @@ def _persistence_values( source=inputs[source_index], target=target, category="memory_persistence", - comment=( - f"Match this input memory to the ID returned at {stage}; " - "the edge records write-result lineage, not independent " - "database confirmation." - ), + comment=("Persist the memory candidate to form a memory unit."), metadata={"pair_index": index, "input_index": source_index}, ) ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py index c1dd14205..9e52992f9 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/base.py @@ -167,7 +167,9 @@ def _hook_subject(context: Any, *, user_name: Any | None = None) -> dict[str, An return subject # Shared terminal nodes and identity-preserving deduplication. - def _operation_status_value(self, subject: Any, *, stage: str) -> TraceValue: + def _operation_status_value( + self, subject: Any, error: BaseException, *, stage: str + ) -> TraceValue: operation_id = _get(subject, "operation_id") or _current_trace_id(subject) return self._value( value="failed", @@ -175,10 +177,7 @@ def _operation_status_value(self, subject: Any, *, stage: str) -> TraceValue: category="operation_status", class_name="str", identity_only=False, - comment=( - f"An exception reached the {stage} Hook boundary; " - "this status does not imply rollback of earlier side effects." - ), + comment=(f"An exception reached the {stage} Hook boundary: {error!s}."), metadata={"memos_stage": stage, "status": "failed"}, ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index 72ecd26ec..2d440e8d9 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -182,7 +182,7 @@ def on_search_results_after_rerank( hook_context=hook_context, operation="memos.search.rerank", category="memory_reranking", - comment="Observe candidate order and content after the Search API rerank stage.", + comment="Observe candidate order and content after rerank stage.", stage="rerank", previous_stage="dedup", metadata={ @@ -206,7 +206,7 @@ def on_search_post_process_failed( stage = "search.post_process.failed" subject = self._search_subject(hook_context, search_req) query = self._search_query_value(subject, search_req, identity_only=True) - status = self._operation_status_value(subject, stage=stage) + status = self._operation_status_value(subject, error, stage=stage) self._emit( self._event( subject, @@ -279,7 +279,7 @@ def _on_search_results( source=query, target=node, category="memory_retrieval", - comment="This query returned the target memory candidate in raw search results.", + comment="Return memory candidates in raw search results.", ) for node in current_nodes ) @@ -298,8 +298,7 @@ def _on_search_results( target=candidate, category=_SEARCH_LINK_CATEGORIES[stage], comment=( - f"Match the same candidate before and after search stage={stage}; " - "the target records this stage's content, score, and rank." + f"Match the same candidate before and after search stage={stage}." ), ) ) @@ -313,10 +312,7 @@ def _on_search_results( source=replace(node, identity_only=True), target=filtered, category="memory_filtered", - comment=( - f"This candidate is absent after search stage={stage}; " - "no deletion from memory storage is implied." - ), + comment=(f"This candidate is absent after search stage={stage}."), ) for node in removed_nodes ) @@ -510,10 +506,7 @@ def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue category="search_filter_result", class_name="str", identity_only=False, - comment=( - f"Previous-stage candidates no longer present after search stage={stage}; " - "this status does not mean deletion from memory storage." - ), + comment=(f"Previous-stage candidates no longer present after search stage={stage}."), metadata={"stage": stage, "status": "filtered"}, ) From 638c89fc6de8cffa05d50f3e8656d9fd7c67a892 Mon Sep 17 00:00:00 2001 From: qzs Date: Wed, 14 Oct 2026 14:22:34 +0800 Subject: [PATCH 06/22] refactor(memtrace): adjust the search handler's node and link comment. --- .../memos_smartcomment/config.py | 10 +- .../handlers/add_handler.py | 519 +++++++++--------- .../handlers/search_handler.py | 248 +++++---- .../memos_smartcomment/plugin.py | 1 + 4 files changed, 402 insertions(+), 376 deletions(-) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/config.py b/apps/memtrace-memos-integration/memos_smartcomment/config.py index c1ac398cd..ac4fb3901 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/config.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/config.py @@ -30,7 +30,7 @@ class SmartCommentSettings: output_dir: Path = Path(".memos/memtrace") queue_size: int = 1024 - max_value_chars: int = 20_000 + max_value_chars: float = 20_000 # Enable SmartComment checks without changing immutable persisted-memory semantics. strict: bool = False project_id: str = "memos" @@ -44,9 +44,11 @@ def from_env(cls) -> SmartCommentSettings: return cls( output_dir=Path(os.getenv("MEMOS_SMARTCOMMENT_OUTPUT_DIR", ".memos/memtrace")), queue_size=_positive_int("MEMOS_SMARTCOMMENT_QUEUE_SIZE", 1024), - max_value_chars=_positive_int( - "MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS", - 20_000, + max_value_chars=float( + _positive_int( + "MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS", + 20_000, + ) ), strict=_as_bool("MEMOS_SMARTCOMMENT_STRICT"), project_id=os.getenv("MEMOS_SMARTCOMMENT_PROJECT_ID", "memos"), diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py index f62a45c28..f1ad4ac1b 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -28,13 +28,14 @@ _MEM_READ_SOURCE_PREFIX = "scheduler.mem_read." -# Input adapters: preserve record order and only infer unambiguous cube ownership. +# Memory records carry their storage cube through nested result groups. +# A single target cube supplies the default ownership; record-level cube +# fields identify memories returned from extraction, writes, and refinement. def _single_cube_id(subject: Any) -> str | None: cube_ids = _cube_ids(subject) return cube_ids[0] if len(cube_ids) == 1 else None -# Walk ID-bearing memory records, carrying cube ownership through wrappers. def _walk_memory_records( value: Any, cube_id: str | None = None ) -> Iterator[tuple[Any, str | None, str]]: @@ -85,7 +86,11 @@ def clear(self) -> None: with self._message_batches_lock: self._message_batches.clear() - # Synchronous add Hooks: messages -> extracted candidates -> observed write results. + # Message intake and batch correlation + # Each add request introduces a message batch shared by its target cubes. + # Extraction in each cube consumes a reference to that batch, so the resulting + # memories point back to the same input. The batch leaves the pending cache + # when all cubes consume it or the add request completes. def on_add_before(self, *, request: Any, **_kwargs: Any) -> None: if bool(_get(request, "is_feedback", False)): return @@ -118,6 +123,67 @@ def on_add_after(self, *, request: Any, **_kwargs: Any) -> None: with self._message_batches_lock: self._message_batches.pop(key, None) + def _message_value( + self, + subject: Any, + messages: Any, + *, + identity_only: bool, + metadata: dict[str, Any] | None = None, + ) -> TraceValue: + return self._value( + value=messages, + identity=self._message_identity(subject, messages, reference=identity_only), + category="input_message_batch", + class_name="input_message_batch", + identity_only=identity_only, + comment="The user's original input message batch, without any modifications.", + metadata=metadata, + ) + + @staticmethod + def _message_batch_key(subject: Any, messages: Any) -> tuple[str, str | None, int]: + return _current_trace_id(subject), _get(subject, "user_id"), id(messages) + + def _message_identity(self, subject: Any, messages: Any, *, reference: bool) -> str: + key = self._message_batch_key(subject, messages) + with self._message_batches_lock: + if reference: + batch = self._message_batches.get(key) + # Match the batch through the request that owns its message object. + if batch is not None and _get(batch.owner(), "messages") is messages: + cubes = _cube_ids(subject) + # Keep the batch until every target cube has consumed its reference. + batch.pending_cubes.difference_update(cubes) + if not cubes or not batch.pending_cubes: + del self._message_batches[key] + return batch.identity + self._message_batches.pop(key, None) + # Assign a new identity to each add and each standalone extraction. + identity = f"message_batch:{key[0]}:{uuid4().hex}" + if not reference: + owner: Callable[[], Any] + try: + # Let the request owner control its lifetime. + owner = weakref.ref(subject) + except TypeError: + # Retain dict and namespace owners for the pending batch lifetime. + def owner() -> Any: + return subject + + cubes = set(_cube_ids(subject)) or {str(_get(subject, "user_id", "unknown"))} + self._message_batches[key] = _MessageBatch(owner, identity, cubes) + self._message_batches.move_to_end(key) + # Evict the oldest batches when the pending cache reaches capacity. + while len(self._message_batches) > self._max_pending_adds: + self._message_batches.popitem(last=False) + return identity + + # Chat memory extraction + # MemReader turns a chat message batch into memory candidates for a storage + # cube. Each candidate links to its source batch and retains its identity + # for the write stage. An extraction exception links the batch to a failure + # status in the same trace. def on_mem_reader_extract_after( self, *, @@ -227,7 +293,41 @@ def on_mem_reader_extract_failed( ) ) - # A successful Hook observes the method boundary, not an independent database check. + @staticmethod + def _messages_from_scene_data(scene_data: Any) -> Any: + if isinstance(scene_data, list | tuple) and len(scene_data) == 1: + messages = scene_data[0] + if isinstance(messages, list | tuple | str): + return messages + return scene_data + + def _extracted_memory_value( + self, + subject: Any, + memory: Any, + *, + cube_id: str | None, + memory_id: str, + identity_only: bool, + metadata: dict[str, Any] | None = None, + ) -> TraceValue: + trace_id = _current_trace_id(subject) + return self._value( + value=_memory_text(memory), + identity=f"extracted_memory:{trace_id}:{cube_id or 'unknown'}:{memory_id}", + category="extracted_memory", + class_name="memory", + identity_only=identity_only, + comment="Memory candidate after MemReader extraction, not yet persisted.", + metadata=metadata, + ) + + # Memory persistence + # Writes turn extracted or enhanced candidates into stored memory units. + # Each returned ID is paired with its unique input candidate to establish + # the persistence link. Stored units use cube and memory IDs as a shared + # identity across writes and later scheduler operations. Write exceptions + # link the submitted candidates to a failure status. def on_text_memory_add_after( self, *, @@ -263,7 +363,6 @@ def on_text_memory_add_after( ) ) - # Failure does not imply rollback of writes already committed. def on_text_memory_add_failed( self, *, @@ -307,7 +406,123 @@ def on_text_memory_add_failed( ) ) - # MemRead Hooks: dispatch observed background operations without repeating their work. + def _persistence_values( + self, + subject: Any, + memories: Any, + result: Any, + *, + cube_id: str | None, + enhanced: bool = False, + ) -> tuple[list[TraceValue], list[TraceValue], list[TraceLink]]: + records = list(_walk_memory_records(memories, cube_id)) + result_ids = list(result) if isinstance(result, list | tuple) else [] + inputs: list[TraceValue] = [] + outputs: list[TraceValue] = [] + links: list[TraceLink] = [] + # Pair returned IDs with unique input candidates. + candidates: dict[str, list[int]] = defaultdict(list) + source_value = self._enhanced_memory_value if enhanced else self._extracted_memory_value + for source_index, (memory, record_cube, candidate_id) in enumerate(records): + resolved_cube = record_cube or cube_id + source = source_value( + subject, + memory, + cube_id=resolved_cube, + memory_id=candidate_id, + identity_only=True, + ) + candidates[candidate_id].append(source_index) + inputs.append(source) + + stage = ( + "scheduler.mem_read.add_enhanced_memories.after" + if enhanced + else "text_memory.add.after" + ) + for index, result_id in enumerate(result_ids): + memory_id = str(result_id) + matching = candidates.get(memory_id, []) + source_index = matching[0] if len(matching) == 1 else None + memory, record_cube, _ = ( + records[source_index] if source_index is not None else (None, cube_id, memory_id) + ) + target = self._persisted_memory_value( + memory, + cube_id=record_cube or cube_id, + memory_id=memory_id, + stage=stage, + ) + outputs.append(target) + if source_index is not None: + links.append( + TraceLink( + source=inputs[source_index], + target=target, + category="memory_persistence", + comment=("Persist the memory candidate to form a memory unit."), + metadata={"pair_index": index, "input_index": source_index}, + ) + ) + return inputs, outputs, links + + @staticmethod + def _persistence_lineage(outputs: list[TraceValue], links: list[TraceLink]) -> str: + if len(links) == len(outputs): + return "resolved" + return "partial" if links else "unresolved" + + def _persisted_memory_value( + self, + memory: Any, + *, + cube_id: str | None, + memory_id: str, + stage: str = "text_memory.add.after", + ) -> TraceValue: + return self._value( + value=( + _memory_text(memory) + if memory is not None + else {"memory_id": memory_id, "cube_id": cube_id} + ), + identity=f"memory:{cube_id or 'unknown'}:{memory_id}", + category="persisted_memory", + class_name="memory", + identity_only=memory is None, + comment=( + f"Persisted memory unit matched to an ID returned at {stage}." + if memory is not None + else f"Write returned this memory ID at {stage}. Its source and content are unresolved." + ), + metadata={ + "memos_stage": stage, + "cube_id": cube_id, + "memory_id": memory_id, + }, + ) + + def _persisted_memory_reference( + self, + *, + cube_id: str | None, + memory_id: str, + ) -> TraceValue: + return self._value( + value={"memory_id": memory_id, "cube_id": cube_id}, + identity=f"memory:{cube_id or 'unknown'}:{memory_id}", + category="persisted_memory", + class_name="memory", + identity_only=True, + comment=("Referred persisted memory unit."), + metadata={"cube_id": cube_id, "memory_id": memory_id}, + ) + + # Background memory maintenance + # MemRead refines stored memories, writes enhanced candidates, and archives + # or removes source memories. The scheduler operation selects the relevant + # memory flow and records its inputs and outputs. Archival, deletion, and + # failures link the affected memories to an operation status. def on_scheduler_memory_operation_after( self, *, @@ -356,7 +571,7 @@ def _on_scheduler_memory_operation( status: str, ) -> None: source = str(_get(hook_context, "source", "")) - # Other handlers share this Hook; only interpret MemRead operations. + # MemRead sources identify the background maintenance flow. if not source.startswith(_MEM_READ_SOURCE_PREFIX): return @@ -401,7 +616,7 @@ def _on_scheduler_memory_operation( cube_id=cube_id, ) - # Operations without memory outputs still have an observable status. + # Represent completion and failure with a status when outputs are empty. if not outputs: if status == "failed": category = "scheduler_operation_failed" @@ -455,7 +670,7 @@ def _on_scheduler_memory_operation( elif operation_name == "add_enhanced_memories" and status == "after": metadata["lineage_status"] = lineage if status == "failed": - # Record only the exception type; messages can expose business data. + # Classify the failure by exception type. metadata["error_type"] = type(error).__name__ self._emit( self._event( @@ -470,166 +685,27 @@ def _on_scheduler_memory_operation( ) ) - # Message correlation: creation registers a batch; references consume one cube's claim. - @staticmethod - def _message_batch_key(subject: Any, messages: Any) -> tuple[str, str | None, int]: - return _current_trace_id(subject), _get(subject, "user_id"), id(messages) - - def _message_identity(self, subject: Any, messages: Any, *, reference: bool) -> str: - key = self._message_batch_key(subject, messages) - with self._message_batches_lock: - if reference: - batch = self._message_batches.get(key) - # Validate the live owner, so an abandoned request cannot match a - # subsequently allocated list at the same address. - if batch is not None and _get(batch.owner(), "messages") is messages: - cubes = _cube_ids(subject) - # Keep the batch until every target cube has consumed its reference. - batch.pending_cubes.difference_update(cubes) - if not cubes or not batch.pending_cubes: - del self._message_batches[key] - return batch.identity - self._message_batches.pop(key, None) - # Each add gets a fresh identity; unmatched references never borrow another request's. - identity = f"message_batch:{key[0]}:{uuid4().hex}" - if not reference: - owner: Callable[[], Any] - try: - # Do not keep abandoned requests alive. - owner = weakref.ref(subject) - except TypeError: - # Plain dict/namespace callers cannot be weakly referenced. - # Their entries still have the same bounded, per-add lifetime. - def owner() -> Any: - return subject - - cubes = set(_cube_ids(subject)) or {str(_get(subject, "user_id", "unknown"))} - self._message_batches[key] = _MessageBatch(owner, identity, cubes) - self._message_batches.move_to_end(key) - # Bound pending batches even if no completion Hook arrives. - while len(self._message_batches) > self._max_pending_adds: - self._message_batches.popitem(last=False) - return identity - - # Unwrap a single scene without copying the messages used for correlation. - @staticmethod - def _messages_from_scene_data(scene_data: Any) -> Any: - if isinstance(scene_data, list | tuple) and len(scene_data) == 1: - messages = scene_data[0] - if isinstance(messages, list | tuple | str): - return messages - return scene_data - - # Node construction: identity selects the anchor; comment explains the captured value. - def _message_value( - self, - subject: Any, - messages: Any, - *, - identity_only: bool, - metadata: dict[str, Any] | None = None, - ) -> TraceValue: - return self._value( - value=messages, - identity=self._message_identity(subject, messages, reference=identity_only), - category="input_message_batch", - class_name="input_message_batch", - identity_only=identity_only, - comment="The user's original input message batch, without any modifications.", - metadata=metadata, - ) - - def _extracted_memory_value( - self, - subject: Any, - memory: Any, - *, - cube_id: str | None, - memory_id: str, - identity_only: bool, - metadata: dict[str, Any] | None = None, - ) -> TraceValue: - trace_id = _current_trace_id(subject) - return self._value( - value=_memory_text(memory), - identity=f"extracted_memory:{trace_id}:{cube_id or 'unknown'}:{memory_id}", - category="extracted_memory", - class_name="memory", - identity_only=identity_only, - comment="Memory candidate after MemReader extraction, not yet persisted.", - metadata=metadata, - ) - - # Share persisted anchors within a graph, including references arriving before the write. - def _persisted_memory_value( - self, - memory: Any, - *, - cube_id: str | None, - memory_id: str, - stage: str = "text_memory.add.after", - ) -> TraceValue: - return self._value( - value=( - _memory_text(memory) - if memory is not None - else {"memory_id": memory_id, "cube_id": cube_id} - ), - identity=f"memory:{cube_id or 'unknown'}:{memory_id}", - category="persisted_memory", - class_name="memory", - identity_only=memory is None, - comment=( - f"Persisted memory unit matched to an ID returned at {stage}." - if memory is not None - else f"Write returned this memory ID at {stage}. Its source and content are unresolved." - ), - metadata={ - "memos_stage": stage, - "cube_id": cube_id, - "memory_id": memory_id, - }, - ) - - def _persisted_memory_reference( - self, - *, - cube_id: str | None, - memory_id: str, - ) -> TraceValue: - return self._value( - value={"memory_id": memory_id, "cube_id": cube_id}, - identity=f"memory:{cube_id or 'unknown'}:{memory_id}", - category="persisted_memory", - class_name="memory", - identity_only=True, - comment=("Referred persisted memory unit."), - metadata={"cube_id": cube_id, "memory_id": memory_id}, - ) - - def _enhanced_memory_value( + def _scheduler_persisted_inputs( self, - subject: Any, - memory: Any, + operation_name: str, + operation_input: Mapping[str, Any], *, cube_id: str | None, - memory_id: str, - identity_only: bool, - ) -> TraceValue: - value = ( - {"memory_id": memory_id, "cube_id": cube_id} if identity_only else _memory_text(memory) - ) - return self._value( - value=value, - identity=( - f"enhanced_memory:{_current_trace_id(subject)}:{cube_id or 'unknown'}:{memory_id}" - ), - category="enhanced_memory", - class_name="memory", - identity_only=identity_only, - comment="Enhanced memory candidate produced by the MemRead Scheduler's fine-transfer stage.", - metadata={"cube_id": cube_id, "memory_id": memory_id}, - ) + ) -> list[TraceValue]: + if operation_name == "fine_transfer_simple_mem": + records = _walk_memory_records(operation_input.get("memories"), cube_id) + return [ + self._persisted_memory_reference( + cube_id=record_cube or cube_id, + memory_id=memory_id, + ) + for _memory, record_cube, memory_id in records + ] + memory_ids = operation_input.get("memory_ids") or [] + return [ + self._persisted_memory_reference(cube_id=cube_id, memory_id=str(memory_id)) + for memory_id in memory_ids + ] def _scheduler_result_value( self, @@ -655,7 +731,12 @@ def _scheduler_result_value( }, ) - # Lineage construction: connect only grouping- or ID-backed sources, never guess by position. + # Memory refinement + # Fine-transfer produces enhanced candidates from stored source memories. + # Per-input result groups link each source to its own candidates; batch + # results link the source group to every candidate. A single source owns + # all of its results. Enhanced identities carry these candidates into the + # subsequent persistence step. def _fine_transfer_values( self, subject: Any, @@ -677,10 +758,11 @@ def _fine_transfer_values( ([source], group) for source, group in zip(inputs, result_groups, strict=True) ] # Batch grouping (or one source) gives every output all known inputs. + # TODO: Detailed link analysis for batch grouping elif grouping == "batch" or len(inputs) == 1: groups = [(inputs, result)] else: - # Keep unresolved outputs without guessing source edges. + # Unresolved groups produce standalone enhanced candidates. groups = [([], result)] for source_values, group in groups: @@ -706,91 +788,26 @@ def _fine_transfer_values( ) return inputs, outputs, links - def _persistence_values( + def _enhanced_memory_value( self, subject: Any, - memories: Any, - result: Any, + memory: Any, *, cube_id: str | None, - enhanced: bool = False, - ) -> tuple[list[TraceValue], list[TraceValue], list[TraceLink]]: - records = list(_walk_memory_records(memories, cube_id)) - result_ids = list(result) if isinstance(result, list | tuple) else [] - inputs: list[TraceValue] = [] - outputs: list[TraceValue] = [] - links: list[TraceLink] = [] - # Only a unique input ID establishes lineage, regardless of returned ID order. - candidates: dict[str, list[int]] = defaultdict(list) - source_value = self._enhanced_memory_value if enhanced else self._extracted_memory_value - for source_index, (memory, record_cube, candidate_id) in enumerate(records): - resolved_cube = record_cube or cube_id - source = source_value( - subject, - memory, - cube_id=resolved_cube, - memory_id=candidate_id, - identity_only=True, - ) - candidates[candidate_id].append(source_index) - inputs.append(source) - - stage = ( - "scheduler.mem_read.add_enhanced_memories.after" - if enhanced - else "text_memory.add.after" + memory_id: str, + identity_only: bool, + ) -> TraceValue: + value = ( + {"memory_id": memory_id, "cube_id": cube_id} if identity_only else _memory_text(memory) + ) + return self._value( + value=value, + identity=( + f"enhanced_memory:{_current_trace_id(subject)}:{cube_id or 'unknown'}:{memory_id}" + ), + category="enhanced_memory", + class_name="memory", + identity_only=identity_only, + comment="Enhanced memory candidate produced by the MemRead Scheduler's fine-transfer stage.", + metadata={"cube_id": cube_id, "memory_id": memory_id}, ) - for index, result_id in enumerate(result_ids): - memory_id = str(result_id) - matching = candidates.get(memory_id, []) - source_index = matching[0] if len(matching) == 1 else None - memory, record_cube, _ = ( - records[source_index] if source_index is not None else (None, cube_id, memory_id) - ) - target = self._persisted_memory_value( - memory, - cube_id=record_cube or cube_id, - memory_id=memory_id, - stage=stage, - ) - outputs.append(target) - if source_index is not None: - links.append( - TraceLink( - source=inputs[source_index], - target=target, - category="memory_persistence", - comment=("Persist the memory candidate to form a memory unit."), - metadata={"pair_index": index, "input_index": source_index}, - ) - ) - return inputs, outputs, links - - # No outputs means there is no unresolved lineage. - @staticmethod - def _persistence_lineage(outputs: list[TraceValue], links: list[TraceLink]) -> str: - if len(links) == len(outputs): - return "resolved" - return "partial" if links else "unresolved" - - def _scheduler_persisted_inputs( - self, - operation_name: str, - operation_input: Mapping[str, Any], - *, - cube_id: str | None, - ) -> list[TraceValue]: - if operation_name == "fine_transfer_simple_mem": - records = _walk_memory_records(operation_input.get("memories"), cube_id) - return [ - self._persisted_memory_reference( - cube_id=record_cube or cube_id, - memory_id=memory_id, - ) - for _memory, record_cube, memory_id in records - ] - memory_ids = operation_input.get("memory_ids") or [] - return [ - self._persisted_memory_reference(cube_id=cube_id, memory_id=str(memory_id)) - for memory_id in memory_ids - ] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index 2d440e8d9..aa98901dd 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -64,7 +64,10 @@ def clear(self) -> None: with self._search_stage_items_lock: self._search_stage_nodes.clear() - # Search Hooks in pipeline order; the failure Hook closes the same operation scope. + # Query intake and search identity + # A search starts with a query and the readable cubes from its request. + # Hook context and request fields place that query in a user trace and + # operation scope. Every result stage refers to the same query identity. def on_search_before( self, *, @@ -87,6 +90,48 @@ def on_search_before( ) ) + def _search_subject( + self, + hook_context: HookContext | None, + request: Any, + ) -> dict[str, Any]: + subject = self._hook_subject(hook_context) + for key in ("trace_id", "user_id", "session_id", "task_id"): + if subject.get(key) is None and (value := _get(request, key)) is not None: + subject[key] = value + if cube_ids := _cube_ids(request): + subject["mem_cube_ids"] = cube_ids + return subject + + @staticmethod + def _search_scope_id(subject: Any) -> str: + """Identify one search operation within a possibly shared trace.""" + operation_id = _get(subject, "operation_id") + return str(operation_id) if operation_id is not None else _current_trace_id(subject) + + def _search_query_value( + self, + subject: Any, + request: Any, + *, + identity_only: bool, + ) -> TraceValue: + search_scope_id = self._search_scope_id(subject) + return self._value( + value=_get(request, "query", ""), + identity=f"search_query:{search_scope_id}", + category="search_query", + class_name="str", + identity_only=identity_only, + comment="Query submitted to the MemOS Search API for memory retrieval.", + metadata={"memos_stage": "search.before"}, + ) + + # Retrieved candidates + # Retrieval returns candidates grouped by memory type and storage cube. + # Each occurrence receives a snapshot with its content, score, and rank + # within that bucket. Stage and traversal position distinguish repeated + # occurrences, while the original candidate supplies its matching key. def on_search_memory_results( self, *, @@ -194,46 +239,6 @@ def on_search_results_after_rerank( }, ) - def on_search_post_process_failed( - self, - *, - handler: Any, - search_req: Any, - error: BaseException, - hook_context: HookContext | None = None, - **_kwargs: Any, - ) -> None: - stage = "search.post_process.failed" - subject = self._search_subject(hook_context, search_req) - query = self._search_query_value(subject, search_req, identity_only=True) - status = self._operation_status_value(subject, error, stage=stage) - self._emit( - self._event( - subject, - operation=f"memos.{stage}", - category="search_failure", - comment="Observe an exception during Search API post-processing for this query.", - inputs=(query,), - outputs=(status,), - links=( - TraceLink( - source=query, - target=status, - category="operation_failed", - comment="Post-processing this query raised an observed exception.", - ), - ), - metadata={ - "memos_stage": stage, - "status": "failed", - "error_type": error.__class__.__name__, - "handler": handler.__class__.__name__, - }, - ) - ) - self._clear_search_stages(subject) - - # Stage processing: build nodes, connect known sources, then publish and clean up. def _on_search_results( self, *, @@ -269,7 +274,7 @@ def _on_search_results( self._search_memory_value(subject, stage=stage, item=item) for item in current_items ) - # Only the first stage links candidates to the query. + # Retrieval establishes the query-to-candidate links. if previous_stage is None: query = self._search_query_value(subject, request, identity_only=True) inputs = (query,) @@ -297,12 +302,10 @@ def _on_search_results( source=replace(source, identity_only=True), target=candidate, category=_SEARCH_LINK_CATEGORIES[stage], - comment=( - f"Match the same candidate before and after search stage={stage}." - ), + comment=(f"Perform post-processing via the stage {stage}."), ) ) - # Keep unmatched candidates, but do not invent their provenance. + # Publish every current candidate, including newly appearing candidates. outputs = current_nodes if removed_nodes: filtered = self._search_filter_result_value(subject, stage=stage) @@ -312,7 +315,7 @@ def _on_search_results( source=replace(node, identity_only=True), target=filtered, category="memory_filtered", - comment=(f"This candidate is absent after search stage={stage}."), + comment=(f"Filter out this memory candidate via the stage {stage}."), ) for node in removed_nodes ) @@ -343,62 +346,6 @@ def _on_search_results( if stage == "rerank": self._clear_search_stages(subject) - # Operation correlation: one trace may contain multiple independent searches. - def _search_subject( - self, - hook_context: HookContext | None, - request: Any, - ) -> dict[str, Any]: - subject = self._hook_subject(hook_context) - for key in ("trace_id", "user_id", "session_id", "task_id"): - if subject.get(key) is None and (value := _get(request, key)) is not None: - subject[key] = value - if cube_ids := _cube_ids(request): - subject["mem_cube_ids"] = cube_ids - return subject - - @staticmethod - def _search_scope_id(subject: Any) -> str: - """Identify one search operation within a possibly shared trace.""" - operation_id = _get(subject, "operation_id") - return str(operation_id) if operation_id is not None else _current_trace_id(subject) - - def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: - return ( - _current_trace_id(subject), - self._search_scope_id(subject), - stage, - ) - - def _clear_search_stages(self, subject: Any) -> None: - trace_id = _current_trace_id(subject) - search_scope_id = self._search_scope_id(subject) - with self._search_stage_items_lock: - for key in [ - key for key in self._search_stage_nodes if key[:2] == (trace_id, search_scope_id) - ]: - del self._search_stage_nodes[key] - - # Candidate snapshots: keep stage-local rank separate from cross-stage identity. - def _search_query_value( - self, - subject: Any, - request: Any, - *, - identity_only: bool, - ) -> TraceValue: - search_scope_id = self._search_scope_id(subject) - return self._value( - value=_get(request, "query", ""), - identity=f"search_query:{search_scope_id}", - category="search_query", - class_name="str", - identity_only=identity_only, - comment="Query submitted to the MemOS Search API for memory retrieval.", - metadata={"memos_stage": "search.before"}, - ) - - # Flatten buckets while retaining bucket-local rank and traversal order. @staticmethod def _search_result_items(results: Any) -> list[_SearchItem]: if not isinstance(results, Mapping): @@ -459,9 +406,8 @@ def _search_memory_value( cube_scope = cube_id if cube_id is not None else f"bucket-{bucket_index}" score = item["score"] comment_parts = [ - f"Memory candidate observed at search stage={stage}", - f"rank={rank} within result-type/cube bucket {rank_scope} (1-based)", - f"global_position={global_position} is traversal order, not a cross-cube rank", + f"Memory candidate observed at search {stage} stage", + f"rank={rank} within result-type/cube bucket {rank_scope}", ] if score is not None: comment_parts.append(f"score={score}") @@ -493,24 +439,21 @@ def _search_memory_value( "score": score, }, ) - - # Match using the original item, never the truncated snapshot. + # Derive the matching key from the original candidate content. return replace(node, match_key=self._search_memory_match_key(item)) - # Removed candidates share one terminal node per stage. - def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: - search_scope_id = self._search_scope_id(subject) - return self._value( - value="filtered", - identity=f"search_filter_result:{search_scope_id}:{stage}", - category="search_filter_result", - class_name="str", - identity_only=False, - comment=(f"Previous-stage candidates no longer present after search stage={stage}."), - metadata={"stage": stage, "status": "filtered"}, + def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: + return ( + _current_trace_id(subject), + self._search_scope_id(subject), + stage, ) - # Cross-stage lineage: match by type/cube/ID or original text, consuming duplicates FIFO. + # Candidate continuity across stages + # Memory type, cube, and memory ID identify a candidate across stages; + # anonymous candidates use their original text. Repeated occurrences + # pair in arrival order. Previous occurrences left after matching share + # the current stage's filtering outcome. @staticmethod def _search_memory_match_key(value: _SearchItem) -> str: memory_id = value["memory_id"] @@ -520,7 +463,6 @@ def _search_memory_match_key(value: _SearchItem) -> str: raw = json.dumps([value["result_type"], cube_scope, identity], ensure_ascii=False) return hashlib.sha256(raw.encode("utf-8")).hexdigest() - # Consume duplicate candidates FIFO so each source matches at most once. def _match_search_nodes( self, previous_nodes: tuple[TraceValue, ...], @@ -542,3 +484,67 @@ def _match_search_nodes( removed = tuple(node for node in previous_nodes if node.identity not in matched_identities) return tuple(matched), removed + + def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: + search_scope_id = self._search_scope_id(subject) + return self._value( + value="filtered", + identity=f"search_filter_result:{search_scope_id}:{stage}", + category="search_filter_result", + class_name="str", + identity_only=False, + comment=(f"The retrieved memory candidate which is filtered out at the {stage} stage."), + metadata={"stage": stage, "status": "filtered"}, + ) + + # Search completion + # Reranking completes the search result flow. A post-processing exception + # links the query to a failure status. Both outcomes release the cached + # candidate stages for that search operation. + def on_search_post_process_failed( + self, + *, + handler: Any, + search_req: Any, + error: BaseException, + hook_context: HookContext | None = None, + **_kwargs: Any, + ) -> None: + stage = "search.post_process.failed" + subject = self._search_subject(hook_context, search_req) + query = self._search_query_value(subject, search_req, identity_only=True) + status = self._operation_status_value(subject, error, stage=stage) + self._emit( + self._event( + subject, + operation=f"memos.{stage}", + category="search_failure", + comment="Observe an exception during Search API post-processing for this query.", + inputs=(query,), + outputs=(status,), + links=( + TraceLink( + source=query, + target=status, + category="operation_failed", + comment="Post-processing this query raised an observed exception.", + ), + ), + metadata={ + "memos_stage": stage, + "status": "failed", + "error_type": error.__class__.__name__, + "handler": handler.__class__.__name__, + }, + ) + ) + self._clear_search_stages(subject) + + def _clear_search_stages(self, subject: Any) -> None: + trace_id = _current_trace_id(subject) + search_scope_id = self._search_scope_id(subject) + with self._search_stage_items_lock: + for key in [ + key for key in self._search_stage_nodes if key[:2] == (trace_id, search_scope_id) + ]: + del self._search_stage_nodes[key] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py index 62307b3ff..defde7da5 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py @@ -40,6 +40,7 @@ def on_load(self) -> None: self.adapter.submit, max_value_chars=settings.max_value_chars, ) + # start the adapter, which will begin processing trace events asynchronously. self.adapter.start() for hook, callback in ( From ceb40db7e77913dc89c5df796efcd9f4a044eb6e Mon Sep 17 00:00:00 2001 From: qzs Date: Thu, 15 Oct 2026 20:04:36 +0800 Subject: [PATCH 07/22] feat: Add a connection between the add trace graph and the search trace graph; Add memory type and memory cube node in search trace graph; Change search memory nodes into a search memory list; Convert each stage of search to a dynamic import. --- .../memos_smartcomment/CONVENTIONS.md | 114 ++-- .../memos_smartcomment/adapter.py | 67 ++- .../memos_smartcomment/config.py | 6 + .../memos_smartcomment/events.py | 1 - .../handlers/add_handler.py | 9 +- .../handlers/search_handler.py | 527 ++++++++++-------- .../memos_smartcomment/lineage.py | 249 +++++++++ .../memos_smartcomment/plugin.py | 3 +- .../memos_smartcomment/storage.py | 293 ++++++++++ 9 files changed, 970 insertions(+), 299 deletions(-) create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/lineage.py create mode 100644 apps/memtrace-memos-integration/memos_smartcomment/storage.py diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md index 6f4374e6b..e3746b8cb 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -10,7 +10,7 @@ | 对象 | 表示什么 | 关键字段 | | --- | --- | --- | -| `TraceValue` | 一份业务值快照,最终注册为图节点 | `value`、`identity`、`class_name`、`category`、`identity_only`、`comment`、`metadata`、`match_key` | +| `TraceValue` | 一份业务值快照,最终注册为图节点 | `value`、`identity`、`class_name`、`category`、`identity_only`、`comment`、`metadata` | | `TraceEvent` | 一次 Hook 观测,最终建立一个 session 和一个 operation | `operation`、`category`、`comment`、`trace_id`、用户/会话/cube 关联信息、`inputs`、`outputs`、`links`、`metadata`、`event_id`、`created_at` | | `TraceLink` | 来源值到目标值的显式数据依赖 | `source`、`target`、`category`、`comment`、`metadata` | @@ -46,7 +46,7 @@ Recorder 用 `id_strategy` 返回 `TraceValue.identity`,因此映射如下: | MemOS `session_id` | 业务会话 ID,导出时放在 session/operation metadata 的 `memos_session_id` 中 | | SmartComment `op_id` | 由 SmartComment 创建 operation 时分配;业务 `operation_id` 仍在 metadata 中 | -即使两个 trace 出现相同 `identity`,也不会跨图合并。持久化记忆 identity 不含 trace,并不意味着全局只有一个节点。 +默认情况下,即使两个 trace 出现相同 `identity`,也不会跨图合并。开启 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` 后,writer 会按最终 Search 命中的持久化记忆导入其 Add 上游子图,见第 4.1 节;不是全局合并 trace。 `trace_id` 解析顺序: @@ -80,7 +80,7 @@ cube 列表取第一个非空字段:`writable_cube_ids → readable_cube_ids | 节点 `class_name` | SmartComment 类型/身份命名空间,不是阶段名;`memory` 共用于提取、增强和持久化记忆,`str` 用于查询和状态。改名会改变 `full_node_id`。 | | `TraceEvent.operation` | 稳定的操作标识,以 `memos.` 开头,点号分隔流程和动作;Scheduler 保留调用点及 `after/failed` 后缀。与业务关联 ID `operation_id` 不同。 | | 事件/边 `category` | 操作或依赖关系的类别,如 `memory_persistence`、`memory_filtered`,不要求与节点类别相同。 | -| 节点 `comment` | 说明捕获了什么值及其观测边界;候选排名注明桶内范围,状态节点不冒充业务输出。 | +| 节点 `comment` | 说明捕获了什么值及其观测边界;搜索列表注明阶段并保留返回顺序,状态节点不冒充业务输出。 | | 事件 `comment` | 说明本次操作的业务意图和实际观测结果,写入导出 operation/session 的 `comment`。自定义事件未提供时保留按 operation 生成的默认说明。 | | 连线 `comment` | 解释为什么从来源值连向目标值;所有内置流程显式提供,不用操作名代替依赖含义。 | @@ -121,33 +121,34 @@ Recorder 对导出的 `persisted_memory.metadata` 额外添加 `snapshot_status= 消息身份的关联规则:每次 `add.before` 分配新 UUID,即便重复使用同一个 messages 列表。短期缓存键为 `(trace_id, user_id, id(messages))`,验证请求仍持有同一个 messages 对象,再让各 cube 的 MemReader 引用同一批次。`id(messages)` 仅用于内存关联,不是图身份;未命中时分配独立 UUID。缓存默认最多 1024 个 pending add,各 cube 消费完、`add.after` 或 shutdown 时清理。 -### 2.2 SearchHandler:查询与各阶段候选 +### 2.2 SearchHandler:查询与各阶段记忆列表 | 业务含义 | `category` | `class_name` | `identity` | `value` | | --- | --- | --- | --- | --- | | 搜索查询 | `search_query` | `str` | `search_query:{search_scope}` | 请求 query;无字段时默认空字符串 | -| 阶段候选 | `search_memory` | `search_memory` | `search_memory:{search_scope}:{stage}:{result_type}:{cube_scope}:{occurrence_id}:{global_position}` | `memory_id`、`memory`、`result_type`、`cube_id`、`score` 五字段字典 | -| 候选移除状态 | `search_filter_result` | `str` | `search_filter_result:{search_scope}:{stage}` | 固定字符串 `"filtered"` | +| 召回类型分支 | `search_memory_type` | `search_memory_type` | `search_memory_type:{search_scope}:{result_type}` | results 顶层类型字符串,例如 `text_mem`、`tool_mem` | +| 召回 cube 分支 | `search_cube` | `search_cube` | `search_cube:{search_scope}:{result_type}:{bucket_index}` | cube ID;缺失时为 `bucket-{bucket_index}` | +| 阶段结果列表 | `search_memory_list` | `search_memory_list` | `search_memory_list:{search_scope}:{stage}:{bucket_key}` | 当前桶的记忆列表,每项包含 `memory_id`、`memory`、`score` | +| 被过滤记忆列表 | `search_filter_result` | `search_memory_list` | `search_filter_result:{search_scope}:{stage}:{bucket_key}` | 当前阶段从前一阶段移除的记忆列表,条目形状与结果列表一致 | | 搜索身份/位置字段 | 规则 | | --- | --- | | `stage` | `raw`、`threshold`、`dedup`、`rerank` | -| `result_type` | results 顶层 key 转字符串,例如 `text_mem`、`pref_mem`;handler 没有限定枚举,也不是源记忆的 `memory_type` | -| `cube_scope` | cube ID 非 `None` 时使用该 ID,否则使用 `bucket-{bucket_index}` | -| `occurrence_id` | memory ID 非 `None` 时使用该 ID,否则 `anonymous`;Search 取 ID 的顺序为 `id or memory_id` | +| `result_type` | results 顶层 key 转字符串,例如 `text_mem`、`tool_mem`;handler 没有限定枚举,也不是源记忆的 `memory_type` | | `bucket_index` | 同一 result type 内的桶序号,从 0 开始 | -| `rank` | 当前桶内排名,从 1 开始;不代表所有 cube 合并后的排名 | -| `rank_scope` | `f"{result_type}:{cube_id or bucket_index}"`,用于说明排名所在桶;它与 `cube_scope` 的兜底写法不同 | -| `global_position` | 当前阶段遍历所有 result type 和桶时的位置,从 1 开始;不是全局按分数排序的名次 | +| `bucket_key` | 紧凑 JSON 数组 `[result_type, cube_id, occurrence]` 经 URL 百分号编码后的字符串(`quote(..., safe="")`),避免引号和反斜杠破坏 DOT 节点 ID;cube ID 非 `None` 时 occurrence 为同 type 下该 cube 的出现序号,从 0 开始;缺失 ID 时使用 bucket_index | +| `memory_id` | 条目中的 `id or memory_id`;无 ID 时保留 `None` | | `score` | 依次取源记忆 `metadata.relativity → metadata.score → memory.score`,仅遇到 `None` 才继续兜底,因此 `0` 有效 | +有 cube ID 的桶改变排列位置后仍连接原有 cube 的列表;重复 cube ID 按出现次序区分。缺失 cube ID 或重复 cube ID 的桶没有更强的唯一标识,跨阶段关联依赖其位置或同名桶次序保持稳定。 + | 节点 | `TraceValue.metadata` | | --- | --- | | `search_query` | `memos_stage="search.before"` | -| `search_memory` | `stage`、`result_type`、`cube_id`、`memory_id`、`rank`、`rank_scope`、`global_position`、`score` | -| `search_filter_result` | `stage`、`status="filtered"` | - -候选的 `comment` 同时写明 stage、rank、rank_scope、global_position,以及非 `None` 的 score。搜索节点的阶段字段叫 `stage`,不是 `memos_stage`。 +| `search_memory_type` | `stage="raw"`、`result_type` | +| `search_cube` | `stage="raw"`、`result_type`、`cube_id`、`bucket_index` | +| `search_memory_list` | `stage`、`result_type`、`cube_id`、`bucket_index`、`memory_count` | +| `search_filter_result` | 与结果列表相同,另加 `status="filtered"` | Search 消费的形状是 `result_type → 桶列表 → memories 列表`,例如: @@ -166,16 +167,13 @@ results = { 当前实现不解包 `{"data": results}`。传入回调时应提供业务 results 字典。 -每个阶段都会建立新的 `search_memory` 节点,即使正文和顺序未变;同 ID 重复出现也由 `global_position` 区分。它们不复用 Add 的 `persisted_memory` 节点。 +raw 阶段显式建立 `search_query → search_memory_type → search_cube → search_memory_list`。每个非空桶只创建一个记忆列表节点;空 type 和 cube 桶不产生分支。后续仅为已启用且收到 Hook 的阶段创建列表节点,连接最近实际记录的列表。关闭的阶段不产生事件、节点或边;已启用阶段即使结果没有改变也保留。若桶被全部过滤或从返回结构中消失,后续结果列表为 `[]`,链路继续保留。首次在后续阶段出现的非空桶保留独立列表,不猜测来源边。 -跨阶段配对使用另外的 `match_key`: +列表按 Hook 实际返回顺序保存;handler 不生成逐条 memory 节点,不计算 rank、rank_scope 或 global_position,也不做节点级配对。开启 Add 关联时,writer 额外导入命中记忆的 `persisted_memory` 及祖先,并连接到最终列表,见第 4.1 节。 -```text -SHA256(UTF8(JSON([result_type, cube_scope, ["id", str(memory_id)]]))) -无 ID 时,最后一项改为 ["text", 原始未截断正文] -``` +被过滤的内容只通过相邻列表的多重集差集计算,不用于决定阶段连线。先扣除完全相同的条目,再按 memory ID 扣除剩余项;没有 ID 时使用原始正文。这样既保留重复项的数量,又不会将同 ID 的正文扩展或分数变化误报为过滤。无法区分的同 ID 剩余项按原列表次序扣除。过滤列表保留前一阶段的内容和顺序,仅在实际有移除项时生成。 -具体 JSON 编码使用 `json.dumps(..., ensure_ascii=False)`。匹配不包含 stage、rank、score;有 ID 时正文改变仍可关联。重复项按 FIFO 一对一消费。`match_key` 保存在 `TraceValue` 中供 handler 缓存使用,当前 recorder 不将它导出到节点或 metadata。 +内部保留脱离业务对象的完整条目用于差集计算,显示时逐条调用快照处理;外层结果/过滤列表不受通用 200 项容器截断限制,条目内部仍遵守文本长度、脱敏及向量省略规则。`memory_count` 为该列表实际条目数。 ### 2.3 BaseHandler:通用失败状态 @@ -189,7 +187,7 @@ SHA256(UTF8(JSON([result_type, cube_scope, ["id", str(memory_id)]]))) MemRead 调度失败使用上一节的 `scheduler_result`,不使用这个节点。异常类型 `error_type` 放在事件/操作 metadata 中;异常对象和异常消息不作为失败节点内容。 -当前 handler 共使用 **4 种 class_name**:`message_batch`、`memory`、`str`、`search_memory`;共使用 **9 种节点 category**:以上表格中的五种 Add、三种 Search 和一种通用失败状态。`TraceValue` 自身的默认 `category="variable"`、`class_name=None` 供自定义调用使用,当前 handler 显式指定这两个字段。 +当前 handler 共使用 **6 种 class_name**:`message_batch`、`memory`、`str`、`search_memory_type`、`search_cube`、`search_memory_list`;共使用 **11 种节点 category**:以上表格中的五种 Add、五种 Search 和一种通用失败状态。`TraceValue` 自身的默认 `category="variable"`、`class_name=None` 供自定义调用使用,当前 handler 显式指定这两个字段。 ## 3. AddHandler 的 operation 与连线 @@ -291,48 +289,71 @@ fine-transfer 成功且存在 enhanced_memory 输出时,`lineage_status` 取 ` ```mermaid flowchart LR - Q[search_query] -->|memory_retrieval| R[raw] + Q[search_query] -->|memory_type_branch| MT[search_memory_type] + MT -->|memory_cube_branch| C[search_cube] + C -->|memory_retrieval| R[raw] R -->|memory_threshold_passed| T[threshold] T -->|memory_deduplicated| D[dedup] D -->|memory_reranked| RR[rerank] - R -->|memory_filtered| F1[threshold: filtered] - T -->|memory_filtered| F2[dedup: filtered] - D -->|memory_filtered| F3[rerank: filtered] + R -->|memory_filtered| F1[threshold: filtered list] + T -->|memory_filtered| F2[dedup: filtered list] + D -->|memory_filtered| F3[rerank: filtered list] ``` -raw、threshold、dedup、rerank 框都表示 `search_memory` 节点。 +上图展示全部阶段启用时的链路。raw、threshold、dedup、rerank 框都表示一个桶的 `search_memory_list` 节点;filtered list 表示该桶在当前阶段移除的记忆列表。 + +插件按当前业务条件判断阶段是否启用:`relativity > 0` 才记录 threshold,`dedup` 为 `sim` 或 `mmr` 才记录 dedup。业务层即使跳过处理仍会发出这两个 Hook,插件会直接忽略它们,并保留最近记录的列表。只关闭 threshold 时链路为 `raw → dedup → rerank`;只关闭 dedup 时为 `raw → threshold → rerank`;两者都关闭时为 `raw → rerank`。中间 Hook 未触发时,后续阶段同样连接最近实际记录的列表。 + +API 后处理当前始终调用 `rerank_knowledge_mem`,因此收到 rerank Hook 就记录该阶段。请求中的 `rerank=False` 只控制 raw 之前的检索内部重排,不控制这个后处理阶段;插件不会用它隐藏后处理 rerank,也不会修改业务执行流程。 | Hook | `TraceEvent.operation` | 事件/操作 `category` | 主要边 category | | --- | --- | --- | --- | | `search.before` | `memos.search.request` | `search_request` | query 根节点,无边 | -| `search.memory_results` | `memos.search.retrieve` | `memory_retrieval` | query → raw;`memory_retrieval` | -| `search.results.after_threshold` | `memos.search.threshold_filter` | `memory_filtering` | raw → threshold;`memory_threshold_passed` | -| `search.results.after_dedup` | `memos.search.deduplicate` | `memory_deduplication` | threshold → dedup;`memory_deduplicated` | -| `search.results.after_rerank` | `memos.search.rerank` | `memory_reranking` | dedup → rerank;`memory_reranked` | +| `search.memory_results` | `memos.search.retrieve` | `memory_retrieval` | query → type:`memory_type_branch`;type → cube:`memory_cube_branch`;cube → raw:`memory_retrieval` | +| `search.results.after_threshold`(启用时) | `memos.search.threshold_filter` | `memory_filtering` | 最近记录列表 → threshold;`memory_threshold_passed` | +| `search.results.after_dedup`(启用时) | `memos.search.deduplicate` | `memory_deduplication` | 最近记录列表 → dedup;`memory_deduplicated` | +| `search.results.after_rerank` | `memos.search.rerank` | `memory_reranking` | 最近记录列表 → rerank;`memory_reranked` | | `search.post_process.failed` | `memos.search.post_process.failed` | `search_failure` | query → operation_status;`operation_failed` | -threshold、dedup、rerank 中,前一阶段未匹配到的候选都以 `memory_filtered` 连到当前阶段共用的 `search_filter_result`。因此 `filtered` 不只表示阈值未通过,也包括去重删除及 rerank 阶段消失的候选。新出现且找不到来源的候选仍保留,不连推测边。 +已记录的 threshold、dedup、rerank 中,每个桶最近实际记录的结果列表连接当前结果列表;若有条目移除,再以一条 `memory_filtered` 边连接该桶的 `search_filter_result` 列表。差集和来源边均以最近记录的列表为准,因此跳过阶段不会丢失过滤内容。被过滤列表也包括去重及 rerank 阶段移除的记忆,且不再使用固定字符串状态节点。 | 操作 | 由 handler 添加的 `TraceEvent.metadata`,不含公共补充字段 | | --- | --- | | request | `memos_stage="search.before"` | | retrieve | `memos_stage="search.memory_results"`、`top_k` | -| threshold_filter | `memos_stage="search.results.after_threshold"`、`threshold`(request.relativity,空值按 0)、`score_field="metadata.relativity"`、`applied=(threshold > 0)` | -| deduplicate | `memos_stage="search.results.after_dedup"`、`method`(request.dedup,缺省/空值按 `no`)、`applied=(method in {sim, mmr})`、`top_k` | +| threshold_filter | `memos_stage="search.results.after_threshold"`、`threshold`(request.relativity,空值按 0;仅大于 0 时发事件)、`score_field="metadata.relativity"`、`applied=True` | +| deduplicate | `memos_stage="search.results.after_dedup"`、`method`(request.dedup,仅 `sim`/`mmr` 时发事件;缺省/空值按 `no` 跳过)、`applied=True`、`top_k` | | rerank | `memos_stage="search.results.after_rerank"`、`algorithm="relativity_sort"`、`order_by="metadata.relativity"`、`direction="descending"`、`top_k` | | post_process.failed | `memos_stage="search.post_process.failed"`、`status="failed"`、`error_type`、`handler`(API handler 的类名) | -四个结果阶段另加 `link_strategy="explicit_pairwise"` 和 `filtered_count`。raw 阶段 filtered_count 为 0;其余阶段是前一阶段未匹配候选数。rerank 的 algorithm/order_by/direction 是插件当前固定写入的描述,不是对实际后端算法的动态识别;插件也不自己重新过滤、去重或排序。`top_k` 是当前 Hook 请求中的值,可能已经被业务层扩充。 +四个结果阶段另加 `link_strategy="bucket_lists"` 和 `filtered_count`。raw 阶段 filtered_count 为 0;其余阶段为所有桶的过滤列表条目数之和,不是过滤节点数。rerank 的 algorithm/order_by/direction 是插件当前固定写入的描述,不是对实际后端算法的动态识别;插件也不自己重新过滤、去重或排序。`top_k` 是当前 Hook 请求中的值,可能已经被业务层扩充。 -缓存键为 `(trace_id, search_scope_id, stage)`。rerank 和 post_process.failed 清理本次搜索的所有阶段缓存,shutdown 清理全部。若缺少 `operation_id`,search_scope 退回 trace,此时同 trace 多次搜索不再具有独立身份,调用方应提供一次搜索各 Hook 共享的 operation_id。 +缓存键为 `(trace_id, search_scope_id)`,每次搜索只保存最近实际记录阶段的列表快照,按 bucket_key 索引。已记录阶段替换前一份快照;关闭的阶段不触碰缓存,raw 开始时丢弃同 scope 的旧快照;rerank 和 post_process.failed 清理本次搜索的缓存,shutdown 清理全部。若缺少 `operation_id`,search_scope 退回 trace,此时同 trace 多次搜索不再具有独立身份,调用方应提供一次搜索各 Hook 共享的 operation_id。 观测终点为 `search.results.after_rerank`;插件没有订阅 `search.context.render` 或 `search.after`。 +### 4.1 可选的 Add → Search 记忆流转 + +`MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` 默认 `false`,在插件加载时读取。 +开启后,异步 writer 只在 `memos.search.rerank` 事件落盘前进行关联: + +1. 从本次事件的最终 `search_memory_list` 读取 `(cube_id, memory_id)`,不读取 raw 或 filtered list 作为命中集合。 +2. 在相同 user/project 的已落盘图中匹配持久化锚点。没有 ID、没有 Add 观测、身份有歧义时不猜测、不按正文匹配。 +3. 仅逆向遍历 `memory_extraction`、`memory_persistence`、`memory_refinement` 入边。共同祖先去重,可跨 trace 追溯 MemRead 增强的原始来源;不扩展兄弟输出、归档/删除分支或其他 Search。 +4. 导入选中的节点、边及其所需 operation/session,保留原 ID,graph_id 改为当前 Search trace,metadata 中以 `lineage_source_trace_id` 保存原 trace。源文件不变;已导入的副本不会成为后续 Search 的新来源。 +5. 用显式 `TraceLink` 创建 `persisted_memory → 最终 search_memory_list`,operation 为 `memos.search.link_add_lineage`,边 category 为 `memory_search_lineage`。一份列表中同一 memory 只连一次,不同列表各自连线。 + +关联事件 metadata 中的 `requested_count`、`linked_count`、`unresolved_count` 按每个最终列表内去重后的、具有 cube/memory ID 的条目计数,`lineage_status` 为 `resolved/partial/unresolved`。边额外记录 `cube_id`、`memory_id`、`source_trace_id`。 + +图切片以已观测节点为单位:共享 message batch 保留完整捕获内容,不推测批次内单条消息的贡献;合并记忆真实依赖的多个来源均属于相关祖先。当前 trace 原有节点不被裁剪,开关关闭后也不删除历史连线。 + +writer 每次最终 Search 扫描当前用户的 JSON 历史,临时索引大小随历史增长;缓存淘汰及进程重启不影响查找。仅能使用当时已写入相同输出目录的历史,迟到的 Add 事件不会自动回填已完成 Search。关联异常会保留本次 Search 自身的图。 + ## 5. metadata 分层与合并规则 | 层级 | 内容 | 自动继承情况 | | --- | --- | --- | -| 节点 `TraceValue.metadata` | 第 2 节列出的值身份、阶段、排名或输入参数 | 不自动继承事件 metadata、HookContext.attributes 或源记忆的完整 metadata | +| 节点 `TraceValue.metadata` | 第 2 节列出的值身份、阶段、列表条目数或输入参数 | 不自动继承事件 metadata、HookContext.attributes 或源记忆的完整 metadata | | 事件 `TraceEvent.metadata` | 第 3/4 节列出的本次操作参数和结果说明 | `_event()` 在 subject.operation_id 非 `None` 时用 setdefault 补 `operation_id` | | 导出 operation metadata | 事件 metadata,加关联公共字段 | 公共字段覆盖事件 metadata 同名项 | | 导出 edge metadata | operation metadata,加 `TraceLink.metadata` | link 字段优先;持久化边额外有 pair_index/input_index | @@ -364,7 +385,7 @@ session_metadata = { 内置 handler 为事件及每条边分别提供说明;自定义 `TraceLink.comment=None` 仍继承操作说明。说明文字不放进 metadata,也不参与节点身份或跨阶段匹配。 -目前只有持久化配对边显式添加 link metadata,其他 handler 边使用空字典并继承 operation metadata。 +持久化配对边和可选的 Add → Search 关联边显式添加 link metadata,其他 handler 边使用空字典并继承 operation metadata。 ## 6. 快照、导出与边界 @@ -373,7 +394,7 @@ session_metadata = { | 规则 | 当前行为 | | --- | --- | | 字符串长度 | 默认保留前 20,000 字符,超出追加 `…`;限制按每个字符串计算 | -| 容器大小 | 默认每层最多 200 项,附截断提示;不等于一个事件最多 200 个节点 | +| 容器大小 | 默认每层最多 200 项,附截断提示;Search 结果和过滤列表逐项快照,保留全部外层条目 | | 敏感字段 | 字段名规范化后包含 api_key/apikey/access_key/password/secret/authorization/cookie/credential/token 时,值替换为 `` | | 向量字段 | embedding/embeddings/vector/vectors 的值替换为 `` | | argument/arguments JSON 字符串 | 先解析并递归脱敏,再编码和截断;不能解析为 dict/list 则替换为 `` | @@ -384,16 +405,21 @@ session_metadata = { 身份和 comment 并不经过 `_value()` 的 value/metadata 快照流程。队列入满会丢弃追踪事件并告警,所以图用于记录已观测到的数据依赖,不能把缺失节点/边直接解释为业务步骤从未发生。 -输出文件按 user/trace 隔离,目录名和文件名为经过清洗且最多 80 字符的可读前缀,加原始 ID 的 JSON 编码的 SHA-256;文件原子替换,后续事件可恢复已有图。缓存淘汰不删除磁盘快照。配置和运行方式见 [应用 README](../README.md)。 +输出文件按 `(user_id, trace_id)` 隔离,路径为 `用户--短哈希/日期/入口类型/时间__trace前缀--短哈希/graph.json`。可读前缀保留 Unicode 字母、最多 32 字符,哈希初始 12 位,发生碰撞时延长;完整身份保存在 `_user.json` 和每张图的 `meta.json` 中。入口类型分为 `search/add/memread/other`,由首次记录事件决定。日期和时间默认使用运行 MemOS 服务的机器或容器当地时区(配置值 `local`),按事件发生日期应用夏令时规则;通过 `MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE` 可指定 `Asia/Shanghai`、`UTC` 等 IANA 时区覆盖默认值。目录和索引使用当地时间,事件原始 UTC 时间戳保持不变。 + +同一 trace 首次写入后固定目录,跨天异步事件、缓存淘汰、重启和时区变更不改变路径;重用 trace 的多次请求仍进入同一张图。每个用户的 `index.md` 展示按请求时间倒序的类型、摘要、完整 trace ID 和图链接;`_index.json` 提供 trace 到元数据的映射。摘要来自已快照的 query/message,最多 160 字符加省略号,不调用模型。首次访问用户时读取 `meta.json` 重建内存索引,后续成功写入重新生成磁盘索引;元数据先预留路径,图原子写入后再发布索引,避免失败写入产生新的可点击链接。图、元数据、索引分别原子替换,不是跨文件事务。 + +`output_path()` 只查询已经分配的路径,未知 trace 抛出 `FileNotFoundError`。Search lineage 通过用户索引遍历所有日期、入口类型下的 `graph.json`,不把 `meta.json` 或 `_index.json` 当作图。旧的平铺 JSON 格式不读取、不迁移,也不删除。缓存淘汰不删除磁盘快照。配置和运行方式见 [应用 README](../README.md)。 ## 7. 查代码与扩展时的检查点 阅读顺序: - `add_handler.py`:同步 Add 回调 → MemRead 回调及调度流程 → 消息关联状态 → 节点构造 → 来源配对。 -- `search_handler.py`:按阶段排列的回调 → 共用阶段处理流程 → operation 关联及清理 → 候选快照 → 跨阶段匹配。 +- `search_handler.py`:按阶段排列的回调 → 列表阶段链路 → raw 分桶分支 → 列表快照 → 过滤差集 → operation 清理。 - `base.py`:上下文字段适配 → 快照/事件构造与提交 → Hook 关联 → 公用失败节点与去重。 - `adapter.py`:请求侧 start/submit/flush/close → 工作线程的快照读写 → 缓存管理与处理循环。 +- `storage.py`:用户身份与短哈希 → 固定请求目录 → 图和元数据原子写入 → 用户索引与历史图查找。 - `recorder.py`:值注册与显式连边 → 占位补全 → 当前格式恢复与导出。 | 问题 | 主要实现 | @@ -402,7 +428,7 @@ session_metadata = { | Add 身份与 metadata | [add_handler.py](handlers/add_handler.py):`_message_value`、`_extracted_memory_value`、`_persisted_memory_value`、`_enhanced_memory_value` | | 返回 ID 怎么连回输入 | [add_handler.py](handlers/add_handler.py):`_persistence_values`、`_persistence_lineage` | | 后台增强的来源关系 | [add_handler.py](handlers/add_handler.py):`_fine_transfer_values`、`_on_scheduler_memory_operation` | -| 搜索节点、排名、跨阶段配对 | [search_handler.py](handlers/search_handler.py):`_search_result_items`、`_search_memory_value`、`_search_memory_match_key`、`_match_search_nodes` | +| 搜索列表、阶段连线和过滤内容 | [search_handler.py](handlers/search_handler.py):`_on_search_results`、`_search_result_buckets`、`_search_memory_list_value`、`_removed_memories` | | 图中身份、metadata、占位补全 | [recorder.py](recorder.py):`_as_comment_item`、`record`、`_complete_memory_snapshots` | | 快照与落盘 | [serialization.py](serialization.py)、[adapter.py](adapter.py) | diff --git a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py index 594cbe3ac..e5b8a0f6e 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py @@ -1,20 +1,18 @@ # One worker serializes events into per-user, per-trace graph snapshots. from __future__ import annotations -import hashlib import json -import os import queue -import re import threading import time from collections import OrderedDict from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, Protocol -from uuid import uuid4 +from memos_smartcomment.lineage import connect_search_lineage from memos_smartcomment.recorder import SmartCommentRecorder +from memos_smartcomment.storage import TraceStore # isort: split @@ -23,7 +21,7 @@ if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterator from pathlib import Path from memos_smartcomment.config import SmartCommentSettings @@ -58,14 +56,6 @@ class _CachedRecorder: graph_items: int -# Hash the original ID to prevent collisions after sanitizing or truncating the prefix. -def _safe_path_component(value: str | None, fallback: str) -> str: - normalized = re.sub(r"[^A-Za-z0-9._-]+", "_", value or fallback).strip("._") - prefix = (normalized or fallback)[:80] - digest = hashlib.sha256(json.dumps(value, ensure_ascii=False).encode("utf-8")).hexdigest() - return f"{prefix}-{digest}" - - class AsyncTraceAdapter: """Bounded event queue and graph cache, with one snapshot-writing worker.""" @@ -76,6 +66,7 @@ def __init__( recorder_factory: Callable[[TraceEvent], Recorder] | None = None, ) -> None: self.settings = settings + self._store = TraceStore(settings) self._recorder_factory = recorder_factory or ( lambda event: SmartCommentRecorder(settings, event) ) @@ -153,29 +144,49 @@ def close(self, timeout: float = 5.0) -> None: # Worker-owned persistence: restore durable snapshots and atomically replace them. def output_path(self, trace_id: str, user_id: str | None = None) -> Path: - user_part = _safe_path_component(user_id, "anonymous") - trace_part = _safe_path_component(trace_id, "trace") - return self.settings.output_dir / user_part / f"{trace_part}.json" + """Resolve an allocated trace; raises FileNotFoundError before its first write.""" + return self._store.output_path(trace_id, user_id) def _read_snapshot(self, event: TraceEvent) -> dict[str, Any] | None: - path = self.output_path(event.trace_id, event.user_id) + try: + path = self.output_path(event.trace_id, event.user_id) + except FileNotFoundError: + return None if path.exists(): return json.loads(path.read_text(encoding="utf-8")) return None + def _link_search_lineage(self, event: TraceEvent, recorder: Recorder) -> None: + current = recorder.export_graph() + + def sources() -> Iterator[dict[str, Any]]: + # The current trace may not have a path yet (e.g. rerank is its first event). + for candidate in self._store.graph_paths(event.user_id): + try: + snapshot = json.loads(candidate.read_text(encoding="utf-8")) + if ( + isinstance(snapshot, dict) + and snapshot.get("graph_id") != event.trace_id + and isinstance(snapshot.get("data"), dict) + ): + yield snapshot + except (OSError, ValueError): + logger.warning("Unable to read Add lineage snapshot: %s", candidate.name) + + try: + merged, link_event = connect_search_lineage(current, event, sources()) + if link_event is not None: + recorder.restore_graph(merged) + recorder.record(link_event) + except Exception: + # Optional historical enrichment must not lose this Search's own graph. + recorder.restore_graph(current) + logger.exception("Failed to connect Add lineage for Search trace %s", event.trace_id) + # Replace atomically so readers cannot observe a partial snapshot. def _export(self, event: TraceEvent, recorder: Recorder) -> int: exported = recorder.export_graph() - output_path = self.output_path(event.trace_id, event.user_id) - output_path.parent.mkdir(parents=True, exist_ok=True) - temporary_path = output_path.with_name(f".{output_path.name}.{uuid4().hex}.tmp") - try: - temporary_path.write_text( - json.dumps(exported, ensure_ascii=False, indent=2), encoding="utf-8" - ) - os.replace(temporary_path, output_path) - finally: - temporary_path.unlink(missing_ok=True) + self._store.write(event, exported) data = exported.get("data", {}) return sum(len(data.get(key, ())) for key in ("nodes", "edges", "operations", "sessions")) @@ -207,6 +218,8 @@ def _process(self, event: TraceEvent) -> None: recorder = cached.recorder # Cache only committed snapshots; failures reload the last durable graph. recorder.record(event) + if self.settings.link_search_to_add and event.operation == "memos.search.rerank": + self._link_search_lineage(event, recorder) graph_items = self._export(event, recorder) self._recorders[key] = _CachedRecorder(recorder, time.monotonic(), graph_items) self._cached_graph_items += graph_items diff --git a/apps/memtrace-memos-integration/memos_smartcomment/config.py b/apps/memtrace-memos-integration/memos_smartcomment/config.py index ac4fb3901..71e0593d6 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/config.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/config.py @@ -38,6 +38,10 @@ class SmartCommentSettings: max_cached_traces: int = 64 max_cached_graph_items: int = 50_000 cache_ttl_seconds: float = 300 + # Import only the persisted ancestors of final Search results. + link_search_to_add: bool = True + # "local" follows the service machine; an IANA timezone overrides it. + output_timezone: str = "local" @classmethod def from_env(cls) -> SmartCommentSettings: @@ -57,4 +61,6 @@ def from_env(cls) -> SmartCommentSettings: "MEMOS_SMARTCOMMENT_MAX_CACHED_GRAPH_ITEMS", 50_000 ), cache_ttl_seconds=_positive_int("MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS", 300), + link_search_to_add=_as_bool("MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD"), + output_timezone=os.getenv("MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE", "local"), ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/events.py b/apps/memtrace-memos-integration/memos_smartcomment/events.py index 78cf981a1..cc81ddd23 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/events.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/events.py @@ -21,7 +21,6 @@ class TraceValue: identity_only: bool = False comment: str | None = None metadata: dict[str, Any] = field(default_factory=dict) - match_key: str | None = None # Correlation key computed before snapshot truncation. # Category/comment inherit operation defaults; link metadata overrides operation metadata. diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py index f1ad4ac1b..9dbff3802 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -312,13 +312,14 @@ def _extracted_memory_value( metadata: dict[str, Any] | None = None, ) -> TraceValue: trace_id = _current_trace_id(subject) + mode = (metadata or {}).get("mode") or "unknown" return self._value( value=_memory_text(memory), identity=f"extracted_memory:{trace_id}:{cube_id or 'unknown'}:{memory_id}", category="extracted_memory", class_name="memory", identity_only=identity_only, - comment="Memory candidate after MemReader extraction, not yet persisted.", + comment=f"Memory candidates are extracted from the user message at the MemReader stage (mode: {mode}).", metadata=metadata, ) @@ -491,7 +492,7 @@ def _persisted_memory_value( class_name="memory", identity_only=memory is None, comment=( - f"Persisted memory unit matched to an ID returned at {stage}." + f"The persisted memory unit that is written to the database and whose memory ID is {memory_id}." if memory is not None else f"Write returned this memory ID at {stage}. Its source and content are unresolved." ), @@ -781,7 +782,7 @@ def _fine_transfer_values( target=target, category="memory_refinement", comment=( - "Refine the memory unit through the fine-transfer stage of the MemRead Scheduler." + "Refine the memory unit asynchronously in the background through the fine-transfer stage of the MemRead Scheduler." ), ) for source in source_values @@ -808,6 +809,6 @@ def _enhanced_memory_value( category="enhanced_memory", class_name="memory", identity_only=identity_only, - comment="Enhanced memory candidate produced by the MemRead Scheduler's fine-transfer stage.", + comment="Refined memory candidates.", metadata={"cube_id": cube_id, "memory_id": memory_id}, ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index aa98901dd..d1077d300 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -1,15 +1,16 @@ -"""Capture Search candidates and their filtering and ranking lineage.""" +"""Capture Search result lists and the memories removed between stages.""" from __future__ import annotations -import hashlib import json -from collections import defaultdict, deque +from collections import Counter from collections.abc import Callable, Mapping -from dataclasses import replace +from copy import deepcopy +from dataclasses import dataclass, replace from threading import Lock -from typing import TYPE_CHECKING, Any, TypedDict +from typing import TYPE_CHECKING, Any +from urllib.parse import quote from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue from memos_smartcomment.handlers.base import ( @@ -31,23 +32,44 @@ "rerank": "memory_reranked", } +_SEARCH_STAGE_LABELS = { + "raw": "initial retrieval", + "threshold": "relevance filtering", + "dedup": "deduplication", + "rerank": "reranking", +} + +_BucketKey = tuple[str, str | None, int] + -class _SearchItem(TypedDict): - """One candidate flattened from a result-type/cube bucket, before snapshotting.""" +@dataclass(frozen=True) +class _SearchBucket: + """One detached list, identified independently of its memory contents.""" result_type: str - bucket_index: int cube_id: str | None - memory_id: Any - memory: Any - rank: int - rank_scope: str - global_position: int - score: Any + bucket_index: int + occurrence: int + memories: list[dict[str, Any]] + + @property + def key(self) -> _BucketKey: + # Cube IDs survive bucket reordering; missing IDs fall back to position. + return ( + self.result_type, + self.cube_id, + self.occurrence if self.cube_id is not None else self.bucket_index, + ) + + +@dataclass(frozen=True) +class _SearchList: + bucket: _SearchBucket + node: TraceValue class SearchHandler(BaseHandler): - """Capture Search API stages with operation-scoped candidate correlation.""" + """Capture one result-list node per type/cube bucket at each enabled Search stage.""" def __init__( self, @@ -56,13 +78,13 @@ def __init__( max_value_chars: int = 20_000, ) -> None: super().__init__(submit, max_value_chars=max_value_chars) - self._search_stage_nodes: dict[tuple[str, str, str], tuple[TraceValue, ...]] = {} - self._search_stage_items_lock = Lock() + self._search_stages: dict[tuple[str, str], dict[_BucketKey, _SearchList]] = {} + self._search_stages_lock = Lock() def clear(self) -> None: """Release cached search-stage nodes at shutdown.""" - with self._search_stage_items_lock: - self._search_stage_nodes.clear() + with self._search_stages_lock: + self._search_stages.clear() # Query intake and search identity # A search starts with a query and the readable cubes from its request. @@ -123,15 +145,14 @@ def _search_query_value( category="search_query", class_name="str", identity_only=identity_only, - comment="Query submitted to the MemOS Search API for memory retrieval.", + comment="The raw query the user uploaded for memory retrieval.", metadata={"memos_stage": "search.before"}, ) # Retrieved candidates # Retrieval returns candidates grouped by memory type and storage cube. - # Each occurrence receives a snapshot with its content, score, and rank - # within that bucket. Stage and traversal position distinguish repeated - # occurrences, while the original candidate supplies its matching key. + # Each bucket becomes one list node, preserving the observed memory order. + # Stage links connect entire lists; removed memories form a separate list. def on_search_memory_results( self, *, @@ -148,7 +169,6 @@ def on_search_memory_results( category="memory_retrieval", comment="Observe the memory candidates returned for this query before post-processing.", stage="raw", - previous_stage=None, metadata={ "memos_stage": "search.memory_results", "top_k": _get(search_req, "top_k"), @@ -164,23 +184,22 @@ def on_search_results_after_threshold( **_kwargs: Any, ) -> None: threshold = _get(search_req, "relativity", 0) or 0 + # MemOS emits this Hook even when the threshold function returns early. + if threshold <= 0: + return self._on_search_results( request=search_req, results=results, hook_context=hook_context, operation="memos.search.threshold_filter", category="memory_filtering", - comment=( - "Observe candidates after the threshold stage; " - "metadata.applied indicates whether filtering was enabled." - ), + comment="Observe candidates after the threshold stage.", stage="threshold", - previous_stage="raw", metadata={ "memos_stage": "search.results.after_threshold", "threshold": threshold, "score_field": "metadata.relativity", - "applied": threshold > 0, + "applied": True, }, ) @@ -193,22 +212,21 @@ def on_search_results_after_dedup( **_kwargs: Any, ) -> None: method = _get(search_req, "dedup", "no") or "no" + # Only these methods execute the API's explicit deduplication stage. + if method not in {"sim", "mmr"}: + return self._on_search_results( request=search_req, results=results, hook_context=hook_context, operation="memos.search.deduplicate", category="memory_deduplication", - comment=( - "Observe candidates after the deduplication stage; " - "metadata.applied indicates whether deduplication was enabled." - ), + comment="Observe candidates after the deduplication stage.", stage="dedup", - previous_stage="threshold", metadata={ "memos_stage": "search.results.after_dedup", "method": str(method), - "applied": method in {"sim", "mmr"}, + "applied": True, "top_k": _get(search_req, "top_k"), }, ) @@ -221,6 +239,7 @@ def on_search_results_after_rerank( hook_context: HookContext | None = None, **_kwargs: Any, ) -> None: + # request.rerank controls retrieval before raw, not this API post-processing. self._on_search_results( request=search_req, results=results, @@ -229,7 +248,6 @@ def on_search_results_after_rerank( category="memory_reranking", comment="Observe candidate order and content after rerank stage.", stage="rerank", - previous_stage="dedup", metadata={ "memos_stage": "search.results.after_rerank", "algorithm": "relativity_sort", @@ -249,86 +267,110 @@ def _on_search_results( category: str, comment: str, stage: str, - previous_stage: str | None, metadata: dict[str, Any], ) -> None: if request is None: return subject = self._search_subject(hook_context, request) - current_items = self._search_result_items(results) - with self._search_stage_items_lock: - previous_nodes = ( - self._search_stage_nodes.get( - self._search_stage_key(subject, previous_stage), - (), - ) - if previous_stage is not None - else () - ) - - inputs: tuple[TraceValue, ...] - outputs: tuple[TraceValue, ...] - links: tuple[TraceLink, ...] - removed_nodes: tuple[TraceValue, ...] = () - current_nodes = tuple( - self._search_memory_value(subject, stage=stage, item=item) for item in current_items + buckets = self._search_result_buckets(results) + scope_key = self._search_scope_key(subject) + + stage_label = _SEARCH_STAGE_LABELS.get(stage, stage) + retained_comment = { + "raw": ( + "Raw memory candidates returned by retrieval, before API-level " + "relevance filtering, deduplication, and reranking." + ), + "threshold": ( + "Memory candidates retained after relevance-threshold filtering. " + "This stage filters text_mem and pref_mem type memories. " + "Other memory types pass through unchanged." + ), + "dedup": ( + f"Memory candidates retained after the {metadata.get('method')} deduplication stage." + if metadata.get("method") + else "Memory candidates retained after the deduplication stage." + ), + "rerank": ( + "Memory candidates retained after the reranking stage. This stage only processes memories of the text_mem type." + ), + }.get( + stage, + f"Memory candidates retained after the {stage} stage.", ) - - # Retrieval establishes the query-to-candidate links. - if previous_stage is None: + with self._search_stages_lock: + # Skipped or missing stages leave the last observed lists as predecessors. + previous = self._search_stages.pop(scope_key, {}) + if stage == "raw": + previous = {} + + # A disappearing bucket still has an empty result at the next stage. + for key, prior in previous.items(): + buckets.setdefault(key, replace(prior.bucket, memories=[])) + + current = { + key: _SearchList( + bucket, + self._search_memory_list_value( + subject, stage=stage, bucket=bucket, retained_comment=retained_comment + ), + ) + for key, bucket in buckets.items() + if bucket.memories or key in previous + } + outputs = tuple(result.node for result in current.values()) + filtered_count = 0 + if stage == "raw": query = self._search_query_value(subject, request, identity_only=True) inputs = (query,) - outputs = current_nodes - links = tuple( - TraceLink( - source=query, - target=node, - category="memory_retrieval", - comment="Return memory candidates in raw search results.", - ) - for node in current_nodes + branches, links = self._search_retrieval_branches( + subject, query=query, results=tuple(current.values()) ) + outputs = branches + outputs else: - matched_nodes, removed_nodes = self._match_search_nodes( - previous_nodes, - current_nodes, - ) - - paired_links: list[TraceLink] = [] - for source, candidate in zip(matched_nodes, current_nodes, strict=True): - if source is not None: - paired_links.append( - TraceLink( - source=replace(source, identity_only=True), - target=candidate, - category=_SEARCH_LINK_CATEGORIES[stage], - comment=(f"Perform post-processing via the stage {stage}."), - ) - ) - # Publish every current candidate, including newly appearing candidates. - outputs = current_nodes - if removed_nodes: - filtered = self._search_filter_result_value(subject, stage=stage) - outputs += (filtered,) - paired_links.extend( + stage_links: list[TraceLink] = [] + for key, result in current.items(): + if key not in previous: + continue + prior = previous[key] + source = replace(prior.node, identity_only=True) + stage_links.append( TraceLink( - source=replace(node, identity_only=True), - target=filtered, - category="memory_filtered", - comment=(f"Filter out this memory candidate via the stage {stage}."), + source=source, + target=result.node, + category=_SEARCH_LINK_CATEGORIES[stage], + comment=f"Process memory candidates through the {stage_label} stage.", ) - for node in removed_nodes ) - links = tuple(paired_links) + removed = self._removed_memories(prior.bucket.memories, result.bucket.memories) + if removed: + filtered = self._search_memory_list_value( + subject, + stage=stage, + bucket=replace(prior.bucket, memories=removed), + filtered=True, + retained_comment=retained_comment, + ) + outputs += (filtered,) + filtered_count += len(removed) + stage_links.append( + TraceLink( + source=source, + target=filtered, + category="memory_filtered", + comment=f"Filter memory candidates through the {stage_label} stage.", + ) + ) + links = tuple(stage_links) inputs = self._unique_values([link.source for link in links]) - with self._search_stage_items_lock: - self._search_stage_nodes[self._search_stage_key(subject, stage)] = current_nodes + with self._search_stages_lock: + self._search_stages[scope_key] = current event_metadata = { **metadata, - "link_strategy": "explicit_pairwise", - "filtered_count": len(removed_nodes), + "link_strategy": "bucket_lists", + "filtered_count": filtered_count, } self._emit( self._event( @@ -346,24 +388,99 @@ def _on_search_results( if stage == "rerank": self._clear_search_stages(subject) + def _search_retrieval_branches( + self, + subject: Any, + *, + query: TraceValue, + results: tuple[_SearchList, ...], + ) -> tuple[tuple[TraceValue, ...], tuple[TraceLink, ...]]: + """Expand raw retrieval into query -> result type -> cube -> memory list.""" + search_scope_id = self._search_scope_id(subject) + types: dict[str, TraceValue] = {} + branches: list[TraceValue] = [] + links: list[TraceLink] = [] + for result in results: + bucket = result.bucket + result_type = bucket.result_type + if result_type not in types: + type_node = self._value( + value=result_type, + identity=f"search_memory_type:{search_scope_id}:{result_type}", + category="search_memory_type", + class_name="search_memory_type", + identity_only=False, + comment=f"Branch of the {result_type} memory type. In this branch, all retrieved memories belong to the {result_type} memory type.", + metadata={"stage": "raw", "result_type": result_type}, + ) + types[result_type] = type_node + branches.append(type_node) + links.append( + TraceLink( + source=query, + target=type_node, + category="memory_type_branch", + comment="Group raw search candidates by memory type.", + ) + ) + + cube_id = bucket.cube_id + cube_label = cube_id if cube_id is not None else f"bucket-{bucket.bucket_index}" + cube_node = self._value( + value=cube_label, + identity=f"search_cube:{search_scope_id}:{result_type}:{bucket.bucket_index}", + category="search_cube", + class_name="search_cube", + identity_only=False, + comment=( + f"Branch for the {result_type} memory type and the {cube_label} memory cube. In this branch, all retrieved memories are of the {result_type} memory type and belong to the {cube_label} memory cube." + ), + metadata={ + "stage": "raw", + "result_type": result_type, + "cube_id": cube_id, + "bucket_index": bucket.bucket_index, + }, + ) + branches.append(cube_node) + links.extend( + ( + TraceLink( + source=types[result_type], + target=cube_node, + category="memory_cube_branch", + comment=f"Group {result_type} memory-type raw search candidates by memory cube.", + ), + TraceLink( + source=cube_node, + target=result.node, + category="memory_retrieval", + comment="Return the list of memories retrieved under this branch.", + ), + ) + ) + return tuple(branches), tuple(links) + @staticmethod - def _search_result_items(results: Any) -> list[_SearchItem]: + def _search_result_buckets(results: Any) -> dict[_BucketKey, _SearchBucket]: if not isinstance(results, Mapping): - return [] + return {} - items: list[_SearchItem] = [] - global_position = 0 - for result_type, buckets in results.items(): - if not isinstance(buckets, list | tuple): + buckets: dict[_BucketKey, _SearchBucket] = {} + for result_type, raw_buckets in results.items(): + if not isinstance(raw_buckets, list | tuple): continue - for bucket_index, bucket in enumerate(buckets): - memories = _get(bucket, "memories") + occurrences: Counter[str | None] = Counter() + for bucket_index, raw_bucket in enumerate(raw_buckets): + cube_id = _get(raw_bucket, "cube_id") + cube_id = str(cube_id) if cube_id is not None else None + occurrence = occurrences[cube_id] + occurrences[cube_id] += 1 + memories = _get(raw_bucket, "memories") if not isinstance(memories, list | tuple): continue - cube_id = _get(bucket, "cube_id") - rank_scope = f"{result_type}:{cube_id or bucket_index}" - for rank, memory in enumerate(memories, start=1): - global_position += 1 + items = [] + for memory in memories: metadata = _get(memory, "metadata", {}) # Zero is a valid score; only None falls through to the next field. score = _get(metadata, "relativity") @@ -373,134 +490,105 @@ def _search_result_items(results: Any) -> list[_SearchItem]: score = _get(memory, "score") items.append( { - "result_type": str(result_type), - "bucket_index": bucket_index, - "cube_id": str(cube_id) if cube_id is not None else None, "memory_id": _get(memory, "id") or _get(memory, "memory_id"), "memory": _memory_text(memory), - "rank": rank, - "rank_scope": rank_scope, - "global_position": global_position, "score": score, } ) - return items + bucket = _SearchBucket( + result_type=str(result_type), + cube_id=cube_id, + bucket_index=bucket_index, + occurrence=occurrence, + # Keep full content for list differences before display truncation. + memories=deepcopy(items), + ) + buckets[bucket.key] = bucket + return buckets - def _search_memory_value( + def _search_memory_list_value( self, subject: Any, *, stage: str, - item: _SearchItem, + bucket: _SearchBucket, + retained_comment: str = "", + filtered: bool = False, ) -> TraceValue: - search_scope_id = self._search_scope_id(subject) - result_type = item["result_type"] - cube_id = item["cube_id"] - bucket_index = item["bucket_index"] - rank = item["rank"] - rank_scope = item["rank_scope"] - global_position = item["global_position"] - memory_id = item["memory_id"] - # Stage and position distinguish repeated occurrences of the same memory. - occurrence_id = memory_id if memory_id is not None else "anonymous" - cube_scope = cube_id if cube_id is not None else f"bucket-{bucket_index}" - score = item["score"] - comment_parts = [ - f"Memory candidate observed at search {stage} stage", - f"rank={rank} within result-type/cube bucket {rank_scope}", - ] - if score is not None: - comment_parts.append(f"score={score}") - - node = self._value( - value={ - "memory_id": memory_id, - "memory": item["memory"], - "result_type": result_type, - "cube_id": cube_id, - "score": score, - }, - identity=( - f"search_memory:{search_scope_id}:{stage}:{result_type}:{cube_scope}:" - f"{occurrence_id}:{global_position}" + category = "search_filter_result" if filtered else "search_memory_list" + # DOT renderers interpolate IDs inside quotes; encode the structured key + # so its quotes and backslashes cannot split one node into multiple IDs. + bucket_key = quote( + json.dumps(bucket.key, ensure_ascii=False, separators=(",", ":")), safe="" + ) + metadata = { + "stage": stage, + "result_type": bucket.result_type, + "cube_id": bucket.cube_id, + "bucket_index": bucket.bucket_index, + "memory_count": len(bucket.memories), + } + + if filtered: + metadata["status"] = "filtered" + return TraceValue( + # Bound each entry's fields without truncating the outer result list. + value=[self._snapshot(memory) for memory in bucket.memories], + identity=f"{category}:{self._search_scope_id(subject)}:{stage}:{bucket_key}", + category=category, + class_name="search_memory_list", + comment=( + f"Memories filtered out during the {stage} stage." if filtered else retained_comment ), - category="search_memory", - class_name="search_memory", - identity_only=False, - comment="; ".join(comment_parts) + ".", - metadata={ - "stage": stage, - "result_type": result_type, - "cube_id": cube_id, - "memory_id": memory_id, - "rank": rank, - "rank_scope": rank_scope, - "global_position": global_position, - "score": score, - }, + metadata=self._snapshot(metadata), ) - # Derive the matching key from the original candidate content. - return replace(node, match_key=self._search_memory_match_key(item)) - def _search_stage_key(self, subject: Any, stage: str) -> tuple[str, str, str]: + def _search_scope_key(self, subject: Any) -> tuple[str, str]: return ( _current_trace_id(subject), self._search_scope_id(subject), - stage, ) - # Candidate continuity across stages - # Memory type, cube, and memory ID identify a candidate across stages; - # anonymous candidates use their original text. Repeated occurrences - # pair in arrival order. Previous occurrences left after matching share - # the current stage's filtering outcome. @staticmethod - def _search_memory_match_key(value: _SearchItem) -> str: - memory_id = value["memory_id"] - identity = ("id", str(memory_id)) if memory_id is not None else ("text", value["memory"]) - cube_id = value["cube_id"] - cube_scope = cube_id if cube_id is not None else f"bucket-{value['bucket_index']}" - raw = json.dumps([value["result_type"], cube_scope, identity], ensure_ascii=False) - return hashlib.sha256(raw.encode("utf-8")).hexdigest() - - def _match_search_nodes( - self, - previous_nodes: tuple[TraceValue, ...], - current_nodes: tuple[TraceValue, ...], - ) -> tuple[tuple[TraceValue | None, ...], tuple[TraceValue, ...]]: - # Each node already carries its original candidate's match key. - pools: dict[str | None, deque[TraceValue]] = defaultdict(deque) - for node in previous_nodes: - pools[node.match_key].append(node) - - matched: list[TraceValue | None] = [] - matched_identities: set[str] = set() - for candidate in current_nodes: - candidates = pools[candidate.match_key] - node = candidates.popleft() if candidates else None - matched.append(node) - if node is not None: - matched_identities.add(node.identity) - - removed = tuple(node for node in previous_nodes if node.identity not in matched_identities) - return tuple(matched), removed - - def _search_filter_result_value(self, subject: Any, *, stage: str) -> TraceValue: - search_scope_id = self._search_scope_id(subject) - return self._value( - value="filtered", - identity=f"search_filter_result:{search_scope_id}:{stage}", - category="search_filter_result", - class_name="str", - identity_only=False, - comment=(f"The retrieved memory candidate which is filtered out at the {stage} stage."), - metadata={"stage": stage, "status": "filtered"}, - ) + def _removed_memories( + previous: list[dict[str, Any]], current: list[dict[str, Any]] + ) -> list[dict[str, Any]]: + """List multiset difference for reporting removals, never for graph links.""" + + def encode(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + + def identity(memory: dict[str, Any]) -> str: + memory_id = memory["memory_id"] + return encode( + ["id", str(memory_id)] if memory_id is not None else ["text", memory["memory"]] + ) + + remaining = Counter(identity(memory) for memory in current) + exact = Counter(encode(memory) for memory in current) + unmatched = [] + # Preserve the right occurrence when equal IDs have different scores/text. + for memory in previous: + snapshot = encode(memory) + if exact[snapshot]: + exact[snapshot] -= 1 + remaining[identity(memory)] -= 1 + else: + unmatched.append(memory) + removed = [] + for memory in unmatched: + key = identity(memory) + if remaining[key]: + # The same memory can gain expanded text or a new score during rerank. + remaining[key] -= 1 + else: + removed.append(memory) + return removed # Search completion # Reranking completes the search result flow. A post-processing exception # links the query to a failure status. Both outcomes release the cached - # candidate stages for that search operation. + # result lists for that search operation. def on_search_post_process_failed( self, *, @@ -541,10 +629,5 @@ def on_search_post_process_failed( self._clear_search_stages(subject) def _clear_search_stages(self, subject: Any) -> None: - trace_id = _current_trace_id(subject) - search_scope_id = self._search_scope_id(subject) - with self._search_stage_items_lock: - for key in [ - key for key in self._search_stage_nodes if key[:2] == (trace_id, search_scope_id) - ]: - del self._search_stage_nodes[key] + with self._search_stages_lock: + self._search_stages.pop(self._search_scope_key(subject), None) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/lineage.py b/apps/memtrace-memos-integration/memos_smartcomment/lineage.py new file mode 100644 index 000000000..9a0504d8c --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/lineage.py @@ -0,0 +1,249 @@ +"""Slice durable Add ancestry and link it to final Search result lists.""" + +from __future__ import annotations + +import json + +from collections import defaultdict +from copy import deepcopy +from dataclasses import replace +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue + + +if TYPE_CHECKING: + from collections.abc import Iterable + + +_NODE_CATEGORIES = { + "input_message_batch", + "message_batch", + "extracted_memory", + "enhanced_memory", + "persisted_memory", +} +_EDGE_CATEGORIES = {"memory_extraction", "memory_persistence", "memory_refinement"} +_SOURCE_TRACE = "lineage_source_trace_id" +_IDS = { + "nodes": "full_node_id", + "edges": "edge_id", + "operations": "op_id", + "sessions": "session_id", +} + + +def _memory_key(cube_id: Any, memory_id: Any) -> tuple[str, str] | None: + # Missing IDs cannot be correlated safely, even if text happens to match. + if cube_id is None or memory_id is None or str(cube_id) == "" or str(memory_id) == "": + return None + return str(cube_id), str(memory_id) + + +def _complete(node: dict[str, Any]) -> bool: + return node.get("metadata", {}).get("snapshot_status") == "complete" + + +def _write_observed(node: dict[str, Any]) -> bool: + return node.get("metadata", {}).get("memos_stage") in { + "text_memory.add.after", + "scheduler.mem_read.add_enhanced_memories.after", + } + + +class _Ancestry: + """An ephemeral index of native Add data, shared across all hits of one Search.""" + + def __init__(self) -> None: + self.items: dict[str, dict[str, dict[str, Any]]] = {field: {} for field in _IDS} + self.ambiguous: set[str] = set() + + def include(self, graph: dict[str, Any]) -> None: + data = graph["data"] + for field, id_field in _IDS.items(): + for item in data.get(field, ()): + # An imported slice is never a new source of truth for another Search. + if _SOURCE_TRACE in item.get("metadata", {}): + continue + if field == "nodes" and item.get("category") not in _NODE_CATEGORIES: + continue + if field == "edges" and item.get("category") not in _EDGE_CATEGORIES: + continue + key = item[id_field] + prior = self.items[field].get(key) + if field == "nodes" and prior is not None: + # Legacy colon-concatenated identities can collide. Do not guess. + if item["category"] == prior["category"] == "persisted_memory": + a, b = item.get("metadata", {}), prior.get("metadata", {}) + if _memory_key(a.get("cube_id"), a.get("memory_id")) != _memory_key( + b.get("cube_id"), b.get("memory_id") + ): + self.ambiguous.add(key) + # A scheduler reference must not hide the original write snapshot. + prior_priority = (_complete(prior), _write_observed(prior)) + item_priority = (_complete(item), _write_observed(item)) + if prior_priority > item_priority: + continue + if ( + prior_priority == item_priority + and prior["created_at"] <= item["created_at"] + ): + continue + elif prior.get("value") != item.get("value"): + self.ambiguous.add(key) + continue + self.items[field][key] = item + + def roots(self) -> dict[tuple[str, str], str]: + candidates: dict[tuple[str, str], set[str]] = defaultdict(set) + writes = { + e["target_full_node_id"] + for e in self.items["edges"].values() + if e.get("category") == "memory_persistence" + } + for node_id, node in self.items["nodes"].items(): + metadata = node.get("metadata", {}) + # A bare scheduler reference is not evidence that an Add was observed. + if ( + node_id in self.ambiguous + or node.get("category") != "persisted_memory" + or not (_complete(node) or node_id in writes or _write_observed(node)) + ): + continue + key = _memory_key(metadata.get("cube_id"), metadata.get("memory_id")) + if key is not None: + candidates[key].add(node_id) + return {key: next(iter(ids)) for key, ids in candidates.items() if len(ids) == 1} + + def slice(self, roots: Iterable[str]) -> dict[str, dict[str, dict[str, Any]]]: + selected: dict[str, dict[str, dict[str, Any]]] = {field: {} for field in _IDS} + incoming: dict[str, list[dict[str, Any]]] = defaultdict(list) + for edge in self.items["edges"].values(): + source, target = edge["source_full_node_id"], edge["target_full_node_id"] + if ( + source in self.items["nodes"] + and target in self.items["nodes"] + and source not in self.ambiguous + and target not in self.ambiguous + ): + incoming[target].append(edge) + pending = list(roots) + while pending: + node_id = pending.pop() + if node_id in selected["nodes"]: + continue + selected["nodes"][node_id] = self.items["nodes"][node_id] + # Reverse traversal never expands siblings or downstream status/deletion nodes. + for edge in incoming[node_id]: + selected["edges"][edge["edge_id"]] = edge + pending.append(edge["source_full_node_id"]) + op_ids = {edge["op_id"] for edge in selected["edges"].values()} + session_ids = { + item["session_id"] for field in ("nodes", "edges") for item in selected[field].values() + } + for op_id, operation in self.items["operations"].items(): + if op_id in op_ids or operation["session_id"] in session_ids: + selected["operations"][op_id] = operation + session_ids.add(operation["session_id"]) + selected["sessions"] = { + key: session for key, session in self.items["sessions"].items() if key in session_ids + } + return selected + + +def connect_search_lineage( + current: dict[str, Any], + event: TraceEvent, + sources: Iterable[dict[str, Any]], +) -> tuple[dict[str, Any], TraceEvent | None]: + """Return a detached graph slice and an explicit persisted-memory -> list event.""" + targets: list[tuple[tuple[str, str], TraceValue]] = [] + for output in event.outputs: + if output.category != "search_memory_list" or output.metadata.get("stage") != "rerank": + continue + seen = set() + for entry in output.value: + key = _memory_key(output.metadata.get("cube_id"), entry.get("memory_id")) + if key is not None and key not in seen: + targets.append((key, output)) + seen.add(key) + if not targets: + return current, None + + ancestry = _Ancestry() + for source in sources: + if all(source.get(field) == current.get(field) for field in ("user_id", "project_id")): + ancestry.include(source) + ancestry.include(current) + roots = ancestry.roots() + matched = [(key, target) for key, target in targets if key in roots] + selected = ancestry.slice(roots[key] for key, _ in matched) + merged = deepcopy(current) + for field, id_field in _IDS.items(): + items = {item[id_field]: item for item in merged["data"][field]} + for key, original in selected[field].items(): + prior = items.get(key) + if prior is not None and not ( + field == "nodes" + and prior.get("category") == "persisted_memory" + and not _complete(prior) + and _complete(original) + ): + continue + item = deepcopy(original) + if item["graph_id"] != current["graph_id"]: + item["metadata"] = {**item.get("metadata", {}), _SOURCE_TRACE: item["graph_id"]} + item["graph_id"] = current["graph_id"] + items[key] = item + merged["data"][field] = list(items.values()) + + links = [] + for key, target in matched: + node = ancestry.items["nodes"][roots[key]] + source = TraceValue( + value=json.loads(node["value"]), + identity=node["name"], + category=node["category"], + class_name=node.get("class_name"), + identity_only=True, + comment=node.get("comment"), + metadata=deepcopy(node["metadata"]), + ) + links.append( + TraceLink( + source=source, + target=replace(target, identity_only=True), + category="memory_search_lineage", + comment="This persisted memory is retrieved in the final search result list.", + metadata={ + "cube_id": key[0], + "memory_id": key[1], + "source_trace_id": node["graph_id"], + }, + ) + ) + return merged, replace( + event, + operation="memos.search.link_add_lineage", + category="memory_search_lineage", + comment="Connect final Search hits to their observed Add ancestry.", + event_id=uuid4().hex, + inputs=(), + outputs=(), + links=tuple(links), + metadata={ + "memos_stage": "search.results.after_rerank", + "operation_id": event.metadata.get("operation_id"), + "requested_count": len(targets), + "linked_count": len(matched), + "unresolved_count": len(targets) - len(matched), + "lineage_status": ( + "resolved" + if len(matched) == len(targets) + else "partial" + if matched + else "unresolved" + ), + }, + ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py index defde7da5..472511dd8 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py @@ -64,9 +64,10 @@ def on_load(self) -> None: ): self.register_hook(hook, callback) logger.info( - "smartcomment plugin loaded; output_dir=%s queue_size=%d", + "smartcomment plugin loaded; output_dir=%s queue_size=%d link_search_to_add=%s", settings.output_dir, settings.queue_size, + settings.link_search_to_add, ) def init_components(self, context: dict[str, Any]) -> None: diff --git a/apps/memtrace-memos-integration/memos_smartcomment/storage.py b/apps/memtrace-memos-integration/memos_smartcomment/storage.py new file mode 100644 index 000000000..c35062987 --- /dev/null +++ b/apps/memtrace-memos-integration/memos_smartcomment/storage.py @@ -0,0 +1,293 @@ +"""Stable request folders and rebuildable per-user trace indexes.""" + +from __future__ import annotations + +import hashlib +import html +import json +import os +import re +import threading + +from collections import OrderedDict +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import quote +from uuid import uuid4 +from zoneinfo import ZoneInfo + +from memos.exceptions import MemOSError +from memos.log import get_logger + + +if TYPE_CHECKING: + from memos_smartcomment.config import SmartCommentSettings + from memos_smartcomment.events import TraceEvent + + +logger = get_logger(__name__) + + +def _digest(value: str | None) -> str: + return hashlib.sha256(json.dumps(value, ensure_ascii=False).encode("utf-8")).hexdigest() + + +def _prefix(value: str | None, fallback: str) -> str: + # Keep Unicode letters, but never separators, control characters or dot segments. + return (re.sub(r"[^\w.-]+", "_", value or fallback).strip("._") or fallback)[:32] + + +def _read(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("Expected a JSON object") + return value + + +def _timestamp(value: str) -> datetime: + timestamp = datetime.fromisoformat(value) + return timestamp if timestamp.tzinfo is not None else timestamp.replace(tzinfo=UTC) + + +def _atomic_write(path: Path, value: dict[str, Any] | str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp") + try: + temporary.write_text( + value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _entry_type(event: TraceEvent) -> str: + if event.operation.startswith("memos.search."): + return "search" + if event.operation.startswith(("memos.add.", "memos.mem_reader.", "memos.text_memory.")): + return "add" + if event.operation.startswith("memos.scheduler."): + return "memread" + return "other" + + +def _summary(event: TraceEvent) -> str: + values = [*event.inputs, *event.outputs] + values.extend(value for link in event.links for value in (link.source, link.target)) + for value in values: + if value.identity_only: + continue + text = "" + if value.category == "search_query" and isinstance(value.value, str): + text = value.value + elif value.category == "input_message_batch" and isinstance(value.value, list): + messages = value.value + snippets = [] + for message in messages: + content = message.get("content", "") if isinstance(message, dict) else message + if isinstance(content, str) and content.strip(): + snippets.append(content) + break + text = f"{len(messages)} messages" + (f": {snippets[0]}" if snippets else "") + if text: + text = " ".join(text.split()) + return text[:160] + ("…" if len(text) > 160 else "") + return "" + + +def _markdown(value: Any) -> str: + text = html.escape(" ".join(str(value or "").split()), quote=False) + return re.sub(r"([\\`*_{}\[\]()|!])", r"\\\1", text) + + +class TraceStore: + """One writer per output directory; metadata fixes each trace's location forever.""" + + def __init__(self, settings: SmartCommentSettings) -> None: + self.root = settings.output_dir + # None lets astimezone use local rules at the event's date, including DST. + self.timezone = ( + None if settings.output_timezone == "local" else ZoneInfo(settings.output_timezone) + ) + self.project_id = settings.project_id + self._lock = threading.RLock() + self._catalogs: OrderedDict[Path, dict[str, dict[str, Any]]] = OrderedDict() + + def _user_directory(self, user_id: str | None, *, create: bool = False) -> Path | None: + digest = _digest(user_id) + available = None + for length in (12, 20, 32, 64): + path = self.root / f"{_prefix(user_id, 'anonymous')}--{digest[:length]}" + if path.exists(): + try: + owner = _read(path / "_user.json") + if "user_id" in owner and owner["user_id"] == user_id: + return path + except (OSError, ValueError): + pass + continue + if available is None: + available = path + # An earlier colliding user may have been deleted; check longer candidates too. + if create: + if available is not None: + _atomic_write(available / "_user.json", {"user_id": user_id}) + return available + raise MemOSError("Cannot allocate a distinct memtrace user directory") + return None + + @staticmethod + def _path(user: Path, relative: str) -> Path: + path = user / relative + parts = Path(relative).parts + if ( + len(parts) != 4 + or parts[-1] != "graph.json" + or not path.resolve().is_relative_to(user.resolve()) + or ".." in parts + ): + raise ValueError("Invalid trace graph path") + return path + + def _catalog(self, user: Path, user_id: str | None) -> dict[str, dict[str, Any]]: + if user in self._catalogs: + self._catalogs.move_to_end(user) + return self._catalogs[user] + entries: dict[str, dict[str, Any]] = {} + # Rebuild the in-memory index from metadata once after startup/cache eviction. + # This also recovers a crash between graph replacement and index publication. + for meta_path in sorted(user.glob("*/*/*/meta.json")): + try: + meta = _read(meta_path) + path = self._path(user, meta["path"]) + if ( + meta.get("schema_version") != 1 + or meta["user_id"] != user_id + or path.parent != meta_path.parent + or not isinstance(meta["trace_id"], str) + ): + raise ValueError("Trace metadata does not match its location") + if meta["trace_id"] in entries: + raise MemOSError("Multiple folders claim the same memtrace trace ID") + entries[meta["trace_id"]] = meta + except (OSError, ValueError, KeyError, TypeError): + logger.warning("Unable to read memtrace metadata: %s", meta_path) + self._catalogs[user] = entries + if len(self._catalogs) > 8: + self._catalogs.popitem(last=False) + return entries + + def output_path(self, trace_id: str, user_id: str | None) -> Path: + """Find an allocated graph; unknown traces raise FileNotFoundError without writes.""" + with self._lock: + user = self._user_directory(user_id) + entry = self._catalog(user, user_id).get(trace_id) if user is not None else None + if entry is None: + raise FileNotFoundError(f"No recorded memtrace graph for trace {trace_id!r}") + return self._path(user, entry["path"]) + + def graph_paths(self, user_id: str | None) -> tuple[Path, ...]: + """Enumerate only this user's graphs, across all dates and entry types.""" + with self._lock: + user = self._user_directory(user_id) + if user is None: + return () + return tuple( + self._path(user, entry["path"]) + for entry in self._catalog(user, user_id).values() + if self._path(user, entry["path"]).is_file() + ) + + def _allocate(self, user: Path, event: TraceEvent) -> dict[str, Any]: + local = _timestamp(event.created_at).astimezone(self.timezone) + entry_type = _entry_type(event) + parent = user / local.strftime("%Y-%m-%d") / entry_type + prefix = f"{local.strftime('%H-%M-%S')}.{local.microsecond // 1000:03d}" + digest = _digest(event.trace_id) + for length in (12, 20, 32, 64): + path = parent / f"{prefix}__{_prefix(event.trace_id, 'trace')}--{digest[:length]}" + if not path.exists(): + break + else: + raise MemOSError("Cannot allocate a distinct memtrace request directory") + entry = { + "schema_version": 1, + "trace_id": event.trace_id, + "user_id": event.user_id, + "project_id": self.project_id, + "entry_type": entry_type, + "first_seen_at": event.created_at, + "last_seen_at": event.created_at, + "display_time": local.isoformat(timespec="milliseconds"), + "timezone": str(self.timezone or local.tzinfo), + "summary": _summary(event), + "session_id": event.session_id, + "task_id": event.task_id, + "cube_ids": list(event.cube_ids), + "operations": [event.operation], + "path": (path / "graph.json").relative_to(user).as_posix(), + } + # Reserve the identity before writing the graph so a failed write retries in place. + _atomic_write(path / "meta.json", entry) + return entry + + def write(self, event: TraceEvent, graph: dict[str, Any]) -> None: + """Commit the graph atomically, then publish metadata and navigable indexes.""" + with self._lock: + user = self._user_directory(event.user_id, create=True) + entries = self._catalog(user, event.user_id) + if event.trace_id not in entries: + entries[event.trace_id] = self._allocate(user, event) + entry = dict(entries[event.trace_id]) + if entry["project_id"] != self.project_id: + raise MemOSError("Trace directory belongs to a different project") + path = self._path(user, entry["path"]) + entry["last_seen_at"] = max((entry["last_seen_at"], event.created_at), key=_timestamp) + entry["summary"] = entry["summary"] or _summary(event) + entry["session_id"] = entry["session_id"] or event.session_id + entry["task_id"] = entry["task_id"] or event.task_id + entry["cube_ids"] = sorted(set(entry["cube_ids"]) | set(event.cube_ids)) + entry["operations"] = sorted(set(entry["operations"]) | {event.operation}) + _atomic_write(path, graph) + _atomic_write(path.with_name("meta.json"), entry) + entries[event.trace_id] = entry + self._publish(user, event.user_id, entries) + + def _publish(self, user: Path, user_id: str | None, entries: dict[str, dict[str, Any]]) -> None: + committed = { + key: value + for key, value in entries.items() + if self._path(user, value["path"]).is_file() + } + _atomic_write( + user / "_index.json", + { + "schema_version": 1, + "user_id": user_id, + "traces": committed, + }, + ) + rows = [ + "# Memtrace requests", + "", + "Times include their UTC offset. Graphs may receive asynchronous updates.", + "", + "| Time | Type | Request summary | Trace ID | Graph |", + "| --- | --- | --- | --- | --- |", + ] + for entry in sorted( + committed.values(), key=lambda item: _timestamp(item["first_seen_at"]), reverse=True + ): + cells = [ + _markdown(entry[key]) + for key in ( + "display_time", + "entry_type", + "summary", + "trace_id", + ) + ] + rows.append("| " + " | ".join(cells) + f" | [graph]({quote(entry['path'])}) |") + _atomic_write(user / "index.md", "\n".join(rows) + "\n") From f769604c62ac4e1283fa10f409457508538d7f77 Mon Sep 17 00:00:00 2001 From: qzs Date: Thu, 15 Oct 2026 20:09:41 +0800 Subject: [PATCH 08/22] docs: change readme --- apps/memtrace-memos-integration/.gitignore | 3 +- apps/memtrace-memos-integration/README.md | 216 ++++++++++++++++++--- 2 files changed, 186 insertions(+), 33 deletions(-) diff --git a/apps/memtrace-memos-integration/.gitignore b/apps/memtrace-memos-integration/.gitignore index ad1838dc3..131eab0ee 100644 --- a/apps/memtrace-memos-integration/.gitignore +++ b/apps/memtrace-memos-integration/.gitignore @@ -16,10 +16,11 @@ __pycache__/ *.py[cod] # Tests and tool output -tests/ +tests/* test_*.py *_test.py conftest.py +!tests/test_search_lineage.py .pytest_cache/ .ruff_cache/ .tests/ diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md index 4321054a6..7f26ceffb 100644 --- a/apps/memtrace-memos-integration/README.md +++ b/apps/memtrace-memos-integration/README.md @@ -54,10 +54,67 @@ Call a working MemOS `/product/add` or `/product/search` endpoint (available in After the operation, list the saved graphs: ```bash -find .memos/memtrace -type f -name '*.json' -print +find .memos/memtrace -type f -name 'index.md' -print ``` -Files are stored under a user-specific directory, one JSON graph per MemOS `trace_id`. Open a file to inspect `data.nodes`, `data.edges`, `data.operations`, and `data.sessions`. User and trace path components contain a readable prefix plus a hash, so the filename is not necessarily the raw trace ID. The graph's `graph_id` contains the trace ID. +Open your user's `index.md` to browse requests, newest first. Each row includes the +request time with UTC offset, entry type, a bounded query/message preview, the full +trace ID, and a relative link to its `graph.json`. + +```text +.memos/memtrace/ +└── alice--/ + ├── _user.json + ├── index.md + ├── _index.json + └── 2026-10-15/ + ├── search/ + │ └── 14-32-10.125__--/ + │ ├── graph.json + │ └── meta.json + └── add/ + └── 14-35-22.410__--/ + ├── graph.json + └── meta.json +``` + +The date, time, and entry-type folders are fixed when a trace is first recorded. +`search` covers Search events; `add` covers Add, MemReader, and text-memory events. +Traces first observed through scheduler events use `memread`; other events use +`other`. Later asynchronous events, cache eviction, restarts, and timezone changes +continue the original graph without moving it or splitting it across folders. + +Dates and times default to the **local timezone of the machine/container running +the MemOS service**, including daylight-saving rules at the event's date. No extra +configuration is needed. Set `MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE=local` explicitly +to restore this behavior if a timezone override was previously configured. +To force a specific timezone instead, set it before starting the service: + +```bash +export MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE=Asia/Shanghai +``` + +`meta.json` records the full user/trace/project IDs, first/latest event timestamps, +timezone, entry type, summary, session/task IDs, cubes, operations, and relative +graph path. `_index.json` maps full trace IDs to that metadata. The writer rebuilds +its in-memory catalog from per-trace metadata on first access, and republishes the +two user indexes on the next successful write, recovering missing, corrupt, or +stale indexes. `_user.json` retains the full user identity. Readable ID prefixes +keep Unicode letters, are limited to 32 characters, and carry a 12-character hash; +hash collisions extend the suffix instead of overwriting another request. + +Graph contents remain `data.nodes`, `data.edges`, `data.operations`, and +`data.sessions`, with `graph_id` equal to the complete trace ID. There is still one +graph per `(user_id, trace_id)`: reusing a trace across requests keeps them together. +For one graph per request, supply a unique trace ID at the request entry and carry +it through asynchronous tasks. This plugin does not change request trace IDs. +`adapter.output_path(trace_id, user_id)` resolves an allocated graph and raises +`FileNotFoundError` before the writer has allocated that trace. + +**The previous flat `/.json` layout is not read or migrated.** Existing +files are left untouched; graph restoration and Search lineage use only the new +layout. To list actual graphs, use `find .memos/memtrace -name graph.json` rather +than treating metadata/index JSON files as graphs. ## Configuration @@ -67,6 +124,7 @@ Set these environment variables before loading the plugin. Restart MemOS after c | --- | --- | --- | | `MEMOS_ENABLED_PLUGINS` | unset | Include `smartcomment` to enable this opt-in plugin. | | `MEMOS_SMARTCOMMENT_OUTPUT_DIR` | `.memos/memtrace` | Directory for JSON graphs. | +| `MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE` | `local` | Service machine's local timezone by default; an IANA name such as `Asia/Shanghai` or `UTC` overrides it. | | `MEMOS_SMARTCOMMENT_QUEUE_SIZE` | `1024` | Maximum queued trace events; a full queue drops events with a warning. | | `MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS` | `20000` | Maximum length of each captured string. | | `MEMOS_SMARTCOMMENT_STRICT` | `false` | Enable smartcomment identity checks. | @@ -74,10 +132,57 @@ Set these environment variables before loading the plugin. Restart MemOS after c | `MEMOS_SMARTCOMMENT_MAX_CACHED_TRACES` | `64` | Maximum graphs retained in memory. | | `MEMOS_SMARTCOMMENT_MAX_CACHED_GRAPH_ITEMS` | `50000` | Maximum cached nodes, edges, operations, and sessions. | | `MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS` | `300` | Idle cache expiration in seconds. | +| `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` | `false` | Include the Add ancestry of final Search hits and link persisted memory nodes to the final result lists. | Snapshots are written atomically. An idle or full in-memory cache can be evicted; a later event restores the saved graph and appends to it. Cache limits do not delete JSON files. Use a different output directory for each writer process, including when running multiple server workers. -Credential-named fields are redacted, embedding/vector fields are omitted, and long values are bounded before events enter the queue. JSON strings in `argument` and `arguments` are decoded and recursively redacted; unparseable argument strings are omitted. **Free-form message and memory text can still appear in graphs**, up to the length limit. Store the output directory with access appropriate for that data. +### Connect Search results to their Add history + +```bash +export MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true +``` + +Restart MemOS after setting the switch. At `search.results.after_rerank`, the writer +matches each final result by `(cube_id, memory_id)` within the same user and project, +reads the existing JSON snapshots, and imports only the matched persisted node and +its upstream Add/MemRead dependencies into the Search graph: + +```text +message batch -> extracted memory -> persisted memory -> enhanced memory + | + v + persisted memory + | + memory_search_lineage + v + final search memory list +``` + +The new edge points **from the persisted memory to the final Search list**. Only +explicit incoming dependencies are followed; sibling outputs from the same Add, +unrelated memories, downstream archive/delete branches, and prior Search graphs +are not imported. A shared ancestor is included once. If multiple source memories +actually contributed to a merged memory, all those ancestors remain relevant. +Message batches remain the original captured units; the plugin does not infer +which individual messages within a batch contributed to a memory. + +Raw candidates and filtered lists do not trigger Add imports. Repeated hits in one +list share a single link; the same memory in different final lists links to each. +Missing IDs, unavailable Add history, and ambiguous identities produce no guessed +links. `memos.search.link_add_lineage` reports linked/unresolved counts for distinct +identifiable hits per list. Imported items retain `lineage_source_trace_id` metadata; +the original Add files are unchanged. + +History lookup runs on the asynchronous writer and reads this user's indexed `graph.json` files across all dates and entry types +once per final Search event. Its temporary index grows with that user's captured +history; keep the output directory scoped to the history you need. It works after +cache eviction or restart and can use snapshots captured before enabling the switch. +Only history already written to this output directory is available at Search time; +late Add events do not retroactively update completed Search graphs. Existing nodes +already native to the same trace are retained. Turning the switch off affects future +Search events and does not remove links from previously saved graphs. + +Credential-named fields are redacted, embedding/vector fields are omitted, and long values are bounded before events enter the queue. JSON strings in `argument` and `arguments` are decoded and recursively redacted; unparseable argument strings are omitted. **Free-form message and memory text can still appear in graphs**, up to the length limit. Request summaries copy at most 160 characters (plus an ellipsis) from the already-captured query or message batch into metadata and indexes; they do not call an LLM. Markdown display escapes user text. Store the output directory with access appropriate for that data. ## If no graph appears @@ -170,37 +275,65 @@ successful database write. Failure Hooks only observe exceptions reaching their If an enhanced-write exception reaches that boundary, subsequent source-memory deletion is skipped for that operation. Failure does not imply rollback of writes already committed. -The Search API emits this ordered result pipeline: +With all post-processing stages enabled, the Search graph has this ordered result +pipeline for each populated type/cube bucket: ```text -search_query - -- memos.search.retrieve --> raw memory nodes - -- memos.search.threshold_filter --> threshold memory nodes; removed nodes -> "filtered" - -- memos.search.deduplicate --> deduplicated memory nodes; removed nodes -> "filtered" - -- memos.search.rerank --> reranked memory nodes +query -> memory type -> cube -> raw list -> threshold list -> dedup list -> rerank list + | | | + v v v + threshold's dedup's rerank's + filtered list filtered list filtered list ``` -Every Search stage materializes its current candidates as separate, stage-scoped -`search_memory` nodes. The node value contains the memory ID, text, score, result type, and -cube ID. Its comment records the one-based `rank`, stage, rank scope, and global position. -Rank scope is one result-type/cube bucket because the Search API returns bucketed results. -The current API rerank stage is recorded as a descending `metadata.relativity` sort -(`relativity_sort`). Search nodes describe the Search pipeline and do not reference -persisted memory units. - -Search stages use explicit links rather than the input/output Cartesian product. Threshold -and deduplication link each surviving source to its new stage node, while removed memories -link to one stage-scoped `"filtered"` string node. Rerank likewise links each -source to the new node with the same memory ID, including knowledge memories whose text is -expanded with original document content. Matching uses result type, cube and memory ID; -candidates without an ID use a content fingerprint computed before truncation. New -candidates are retained even when no source edge is known. This keeps every Hook visible as -one step in the Search flow, including stages whose values did not change. - -Search observation ends at `search.results.after_rerank`, which releases the operation's -cached stage nodes. All Search hooks use the shared `HookContext.operation_id` as their graph and cache -scope, so multiple searches in one trace cannot reuse or clear each other's stage nodes. -Post-processing failures clear only the failed operation's cached stage nodes. +Raw retrieval creates `search_memory_type` and `search_cube` branch nodes followed +by one `search_memory_list` node per nonempty bucket. Empty types and cubes are +omitted. The three retrieval links use `memory_type_branch`, `memory_cube_branch`, +and `memory_retrieval`, respectively. Every enabled, observed stage creates one new +list node per observed bucket and connects the last recorded list directly to it. +A fully removed bucket continues as an empty list, preserving the stage chain. + +The plugin follows the API's current stage conditions even though the API emits +Hooks for disabled stages: threshold is recorded only when `relativity > 0`, and +dedup only when `dedup` is `"sim"` or `"mmr"`. Disabled stages produce no events, +nodes, or edges. For example, `relativity=0` yields `raw -> dedup -> rerank` when +dedup is enabled; setting `dedup="no"` as well yields `raw -> rerank`. An enabled +stage is retained even if its results are unchanged. Missing intermediate Hooks +also leave the last recorded stage as the predecessor. + +The API always runs its post-processing `rerank_knowledge_mem` on a successful +search, so its rerank stage is still recorded when `search_req.rerank=False`. +That request flag controls the internal retrieval reranker before raw, not this +post-processing stage. The plugin does not change which business stages execute. + +List entries contain `memory_id`, `memory`, and `score`, in the order returned by +the Hook. Result type, cube ID, stage, bucket position, and `memory_count` belong to +the list node's metadata. Individual memories no longer create graph nodes, ranks, +or pairwise node matches. Snapshots are taken per entry so the outer list retains +all results, including lists longer than 200 entries; field truncation and redaction +still apply within each entry. With `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true`, +final lists additionally receive incoming links from matching persisted memory units. + +When a stage removes entries, its last recorded list also connects to a +`search_filter_result` node containing the removed entries, rather than a status +string. One filtered list is created per affected bucket. Differences preserve +duplicate counts and prefer identical entries before falling back to memory ID +(or original text for anonymous entries), so reordered results and expanded text +with the same ID are not treated as removals. `filtered_count` counts removed +memories across buckets. No filtered node is created when nothing was removed. + +Branches and lists use `HookContext.operation_id` as their search scope. List links +use result type, cube ID, and the occurrence of that cube within the type; buckets +without IDs use their position. Reordered unique cube IDs remain correctly linked. +Repeated or missing cube IDs rely on stable occurrence order or position. A new +bucket without a previous-stage list is retained without a guessed source edge. + +Search observation ends at `search.results.after_rerank`. Each search caches only +its latest recorded lists; skipped stages leave that cache intact. Rerank, +post-processing failure, and shutdown release +remaining caches in their respective scopes. The current rerank event describes +the API's descending `metadata.relativity` sort; the tracing handler preserves +observed order and does not sort results itself. The exported graph omits smartcomment's internal `COMMENT:NONE@1` sentinel node, its incident edges, and its sentinel session. Root inputs and outputs remain normal semantic @@ -226,7 +359,7 @@ Keep machine identifiers separate from reader-facing descriptions: explains the observation in the exported operation and session. Each built-in link has its own description of the source-to-target relationship. - Write comments describe returned IDs, not independently confirmed database writes. - Search comments distinguish bucket-local rank from traversal order, and removal + Search comments describe whole lists at each stage and distinguish removal from search results from deletion in storage. Scheduler comments identify the backend operation, including `delete` versus `soft_delete`, without claiming every input was removed. @@ -285,7 +418,26 @@ operation's data contract; unknown lineage keeps its nodes without a guessed edg ## Verify the integration -The plugin regression tests are maintained locally and are not included in this directory. +Run the Search-to-Add lineage regression tests from this directory: + +```bash +PYTHONPATH=. python -m pytest tests/test_search_lineage.py -q +``` + +They exercise actual Add/Search hooks, final-hit selection, shared ancestors, +cross-trace refinement, user/cube isolation, opt-in behavior, and durable restore. + +Run the local search-branch regression tests (ignored by Git) from this directory +with the integration and its dependencies installed: + +```bash +PYTHONPATH=. python -m pytest tests/handlers/test_search_handler.py -q +``` + +These tests cover populated and empty branches, list size independent of graph size, +filtered contents including duplicates, cube identity, operation isolation, and +exported list continuity through post-processing in strict and non-strict modes. Additional plugin regression tests are maintained locally. + For an installed integration, follow the Quick start package and entry-point check, enable the plugin, and run a memory operation. Inspect the resulting JSON graph's `graph_id`, `data.nodes`, and `data.edges`, and check the service logs for writer failures or dropped From 16f00d771b4215add48070587ebb036c27c36b81 Mon Sep 17 00:00:00 2001 From: qzs Date: Thu, 15 Oct 2026 20:35:06 +0800 Subject: [PATCH 09/22] fix(mem-reader): preserve input order in concurrent fine transfer --- src/memos/mem_reader/simple_struct.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/memos/mem_reader/simple_struct.py b/src/memos/mem_reader/simple_struct.py index 14521cf47..4db01d90f 100644 --- a/src/memos/mem_reader/simple_struct.py +++ b/src/memos/mem_reader/simple_struct.py @@ -815,7 +815,7 @@ def fine_transfer_simple_mem( if not input_memories: return [] - memory_list = [] + memory_list = [[] for _ in input_memories] if type == "chat": processing_func = self._process_transfer_chat_data @@ -826,15 +826,15 @@ def fine_transfer_simple_mem( # Process Q&A pairs concurrently with context propagation with ContextThreadPoolExecutor() as executor: - futures = [ - executor.submit(processing_func, scene_data_info, custom_tags, **kwargs) - for scene_data_info in input_memories - ] + futures = { + executor.submit(processing_func, scene_data_info, custom_tags, **kwargs): index + for index, scene_data_info in enumerate(input_memories) + } for future in concurrent.futures.as_completed(futures): try: res_memory = future.result() if res_memory is not None: - memory_list.append(res_memory) + memory_list[futures[future]] = res_memory except Exception as e: logger.error(f"Task failed with exception: {e}") logger.error(traceback.format_exc()) From 1378dec559c6621aa2e84f6f299381a36cfcd208 Mon Sep 17 00:00:00 2001 From: qzs Date: Thu, 15 Oct 2026 20:36:05 +0800 Subject: [PATCH 10/22] feat(hooks): expose fine-transfer result grouping --- .../handlers/mem_read_handler.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index a61744785..2e343bb07 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -182,7 +182,11 @@ def _process_memories_with_reader( operation="fine", target="textual_memory", operation_context=operation_context, - operation_input={"memories": memory_items, "type": "chat"}, + operation_input={ + "memories": memory_items, + "type": "chat", + "result_grouping": self._fine_transfer_result_grouping(mem_reader), + }, capture_failure=True, ) as observation: observation.result( @@ -574,3 +578,18 @@ def _try_submit_organize_task( ) except Exception as e: logger.error("Failed to enqueue MEM_ORGANIZE task: %s", e, exc_info=True) + + def _fine_transfer_result_grouping(self, mem_reader: Any) -> str: + """Describe known transfer implementations at the observation boundary.""" + + from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader + from memos.mem_reader.simple_struct import SimpleStructMemReader + + # Match the bound implementation so inherited methods work, while overrides + # with a different grouping contract remain unknown. + transfer = getattr(mem_reader.fine_transfer_simple_mem, "__func__", None) + if transfer is MultiModalStructMemReader.fine_transfer_simple_mem: + return "batch" + if transfer is SimpleStructMemReader.fine_transfer_simple_mem: + return "per_input" + return "unknown" From a57c5e13f34cd4aaa6a924805a7e6702979b5ecd Mon Sep 17 00:00:00 2001 From: Chen Jiang Date: Thu, 15 Oct 2026 20:50:50 +0800 Subject: [PATCH 11/22] test(hooks): streamline contracts and cover reader integration Port the reviewed Hook-only tests from smartcomment-integration. Keep upstream Search tests intact apart from required hook_context assertions, and isolate new behavior tests with an opt-in fixture. Cover Reader grouping/order and all five MemRead operation sites. Validation on feat/add-more-plugin-hooks: 49 targeted tests passed; related regression: 266 passed, 1 skipped. Individual test files, Ruff, staged pre-commit, and diff checks passed. Full make test stopped after 180 seconds at the existing Ollama network request. --- tests/api/test_search_hook_context.py | 178 +++++++++ tests/api/test_search_pipeline_hooks.py | 183 +--------- tests/conftest.py | 11 + tests/mem_reader/test_fine_transfer_order.py | 44 +++ tests/mem_reader/test_hook_pipeline.py | 84 ++--- tests/mem_scheduler/test_mem_read_grouping.py | 132 +++++++ .../mem_scheduler/test_operation_observer.py | 296 ++++++++++----- .../multi_mem_cube/test_single_cube_hooks.py | 70 ++-- tests/plugins/test_hook_context.py | 338 ++++++------------ 9 files changed, 746 insertions(+), 590 deletions(-) create mode 100644 tests/api/test_search_hook_context.py create mode 100644 tests/conftest.py create mode 100644 tests/mem_reader/test_fine_transfer_order.py create mode 100644 tests/mem_scheduler/test_mem_read_grouping.py diff --git a/tests/api/test_search_hook_context.py b/tests/api/test_search_hook_context.py new file mode 100644 index 000000000..a5d39b3e8 --- /dev/null +++ b/tests/api/test_search_hook_context.py @@ -0,0 +1,178 @@ +import pytest + +from memos.api.handlers.base_handler import HandlerDependencies +from memos.api.handlers.search_handler import SearchHandler +from memos.api.product_models import APISearchRequest +from memos.plugins.hook_context import HookContext +from memos.plugins.hook_defs import H +from memos.plugins.hooks import register_hook + + +pytestmark = pytest.mark.usefixtures("clean_hooks") + + +def _empty_search_results() -> dict: + return { + "text_mem": [], + "act_mem": [], + "para_mem": [], + "pref_mem": [], + "pref_note": "", + "tool_mem": [], + "skill_mem": [], + } + + +class _CubeView: + def __init__(self, results: dict): + self.results = results + self.request = None + + def search_memories(self, request): + self.request = request + return self.results + + +def _search_handler() -> tuple[SearchHandler, _CubeView]: + handler = SearchHandler( + HandlerDependencies( + naive_mem_cube=object(), + mem_scheduler=object(), + searcher=object(), + deepsearch_agent=object(), + ) + ) + cube_view = _CubeView(_empty_search_results()) + handler._build_cube_view = lambda _search_req: cube_view + return handler, cube_view + + +def test_search_success_hooks_run_in_order_and_share_context(monkeypatch): + events = [] + result_hook_calls = [] + handler, cube_view = _search_handler() + search_req = APISearchRequest( + user_id="user", + query="original query", + session_id="session-1", + readable_cube_ids=["cube-1", "cube-2", "cube-1"], + ) + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", + lambda: "trace-1", + ) + + def replace_request(*, hook_context, request): + events.append((H.SEARCH_BEFORE, hook_context)) + return request.model_copy(update={"query": "hooked query"}) + + def record_results(hook_name): + def callback(*, hook_context, **kwargs): + events.append((hook_name, hook_context)) + result_hook_calls.append(kwargs) + + return callback + + def replace_response(*, hook_context, result, **_kwargs): + events.append((H.SEARCH_AFTER, hook_context)) + return result.model_copy(update={"message": "hooked response"}) + + result_hooks = [ + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + H.SEARCH_CONTEXT_RENDER, + ] + register_hook(H.SEARCH_BEFORE, replace_request) + for hook_name in result_hooks: + register_hook(hook_name, record_results(hook_name)) + register_hook(H.SEARCH_AFTER, replace_response) + + response = handler.handle_search_memories(search_req) + + assert [name for name, _context in events] == [ + H.SEARCH_BEFORE, + *result_hooks, + H.SEARCH_AFTER, + ] + contexts = [context for _name, context in events] + assert all(context is contexts[0] for context in contexts) + context = contexts[0] + assert isinstance(context, HookContext) + assert context.trace_id == "trace-1" + assert context.user_id == "user" + assert context.session_id == "session-1" + assert context.cube_ids == ("cube-1", "cube-2") + assert context.source == "api.search" + assert context.attributes == {"mode": "fast"} + assert context.operation_id + assert cube_view.request.query == "hooked query" + assert all(call["handler"] is handler for call in result_hook_calls) + assert all(call["search_req"] is cube_view.request for call in result_hook_calls) + assert response.message == "hooked response" + + +@pytest.mark.parametrize( + "hook_name", + [ + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + H.SEARCH_CONTEXT_RENDER, + ], +) +def test_search_result_hooks_replace_memories_in_response(hook_name): + handler, _cube_view = _search_handler() + memory = { + "id": "plugin-memory", + "memory": "Remember tea", + "metadata": {"relativity": 1.0, "memory_type": "LongTermMemory"}, + } + + def replace_memories(*, results, **_kwargs): + return {**results, "text_mem": [{"cube_id": "cube-1", "memories": [memory]}]} + + register_hook(hook_name, replace_memories) + + response = handler.handle_search_memories( + APISearchRequest(user_id="user", query="drink", dedup="no") + ) + + assert response.data["text_mem"][0]["memories"] == [memory] + + +def test_search_post_processing_failure_triggers_only_failure_hook(monkeypatch): + error = RuntimeError("post processing failed") + events = [] + handler, _cube_view = _search_handler() + search_req = APISearchRequest(user_id="user", query="query") + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", + lambda: "trace-1", + ) + + def fail_post_processing(_results, _relativity): + raise error + + def record_failure(**kwargs): + events.append((H.SEARCH_POST_PROCESS_FAILED, kwargs)) + + handler._apply_relativity_threshold = fail_post_processing + register_hook(H.SEARCH_POST_PROCESS_FAILED, record_failure) + register_hook(H.SEARCH_CONTEXT_RENDER, lambda **_kwargs: events.append(("render", {}))) + register_hook(H.SEARCH_AFTER, lambda **_kwargs: events.append(("after", {}))) + + with pytest.raises(RuntimeError, match="post processing failed") as exc_info: + handler.handle_search_memories(search_req) + + assert exc_info.value is error + assert [name for name, _kwargs in events] == [H.SEARCH_POST_PROCESS_FAILED] + failure = events[0][1] + assert failure["error"] is error + assert failure["handler"] is handler + assert failure["search_req"].query == "query" + context = failure["hook_context"] + assert context.trace_id == "trace-1" + assert context.user_id == "user" diff --git a/tests/api/test_search_pipeline_hooks.py b/tests/api/test_search_pipeline_hooks.py index 329a03350..bfdad25a0 100644 --- a/tests/api/test_search_pipeline_hooks.py +++ b/tests/api/test_search_pipeline_hooks.py @@ -1,20 +1,6 @@ -import pytest - -from memos.api.handlers.base_handler import HandlerDependencies from memos.api.handlers.formatters_handler import rerank_knowledge_mem -from memos.api.handlers.search_handler import SearchHandler from memos.api.product_models import APISearchRequest -from memos.multi_mem_cube.single_cube import SingleCubeView -from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H, get_hook_spec -from memos.plugins.hooks import _hooks, register_hook - - -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() def _memory(memory_id: str, memory: str, memory_type: str = "LongTermMemory") -> dict: @@ -41,39 +27,6 @@ def _file_memory(memory_id: str, memory: str, source_content: str) -> dict: } -def _empty_search_results() -> dict: - return { - "text_mem": [], - "act_mem": [], - "para_mem": [], - "pref_mem": [], - "pref_note": "", - "tool_mem": [], - "skill_mem": [], - } - - -def _search_handler(results: dict | None = None) -> tuple[SearchHandler, SingleCubeView]: - handler = SearchHandler( - HandlerDependencies( - naive_mem_cube=object(), - mem_scheduler=object(), - searcher=object(), - deepsearch_agent=object(), - ) - ) - cube_view = SingleCubeView.__new__(SingleCubeView) - search_results = results if results is not None else _empty_search_results() - cube_view.search_memories = lambda _request: search_results - # Preserve the existing _build_cube_view(search_req) call contract. - handler._build_cube_view = lambda _search_req: cube_view - return handler, cube_view - - -def _raise(error: Exception): - raise error - - def test_search_request_passes_context_format_through_to_plugins(): req = APISearchRequest( user_id="user", @@ -85,136 +38,16 @@ def test_search_request_passes_context_format_through_to_plugins(): def test_search_pipeline_hook_specs_are_registered(): - for hook_name in (H.SEARCH_RESULTS_AFTER_RERANK, H.SEARCH_CONTEXT_RENDER): - spec = get_hook_spec(hook_name) - assert spec is not None - assert spec.pipe_key == "results" - assert spec.params == ["hook_context", "handler", "search_req", "results"] - - -@pytest.mark.parametrize( - ("hook_name", "params", "pipe_key"), - [ - (H.SEARCH_BEFORE, ["hook_context", "request"], "request"), - (H.SEARCH_AFTER, ["hook_context", "request", "result"], "result"), - ( - H.SEARCH_MEMORY_RESULTS, - ["hook_context", "handler", "search_req", "results"], - "results", - ), - ( - H.SEARCH_RESULTS_AFTER_THRESHOLD, - ["hook_context", "handler", "search_req", "results"], - "results", - ), - ( - H.SEARCH_RESULTS_AFTER_DEDUP, - ["hook_context", "handler", "search_req", "results"], - "results", - ), - ( - H.SEARCH_POST_PROCESS_FAILED, - ["hook_context", "handler", "search_req", "error"], - None, - ), - ], -) -def test_search_hook_specs_match_trigger_contracts(hook_name, params, pipe_key): - spec = get_hook_spec(hook_name) - - assert spec is not None - assert spec.params == params - assert spec.pipe_key == pipe_key - - -def test_search_success_hooks_run_in_order_and_share_one_context(monkeypatch): - events: list[tuple[str, dict]] = [] - handler, cube_view = _search_handler() - search_req = APISearchRequest( - user_id="user", - query="query", - session_id="session-1", - readable_cube_ids=["cube-1", "cube-2", "cube-1"], - ) - monkeypatch.setattr( - "memos.api.handlers.search_handler.get_current_trace_id", - lambda: "trace-1", - ) - - def record(hook_name): - def callback(**kwargs): - events.append((hook_name, kwargs)) - - return callback - - ordered_hooks = [ - H.SEARCH_BEFORE, - H.SEARCH_MEMORY_RESULTS, - H.SEARCH_RESULTS_AFTER_THRESHOLD, - H.SEARCH_RESULTS_AFTER_DEDUP, - H.SEARCH_RESULTS_AFTER_RERANK, - H.SEARCH_CONTEXT_RENDER, - H.SEARCH_AFTER, - ] - for hook_name in ordered_hooks: - register_hook(hook_name, record(hook_name)) - - handler.handle_search_memories(search_req) - - assert [name for name, _kwargs in events] == ordered_hooks - contexts = [kwargs["hook_context"] for _name, kwargs in events] - assert all(context is contexts[0] for context in contexts) - context = contexts[0] - assert isinstance(context, HookContext) - assert context.trace_id == "trace-1" - assert context.user_id == "user" - assert context.session_id == "session-1" - assert context.cube_ids == ("cube-1", "cube-2") - assert context.source == "api.search" - assert context.attributes == {"mode": "fast"} - assert context.operation_id is not None - assert not hasattr(cube_view, "search_hook_context") - + after_rerank = get_hook_spec(H.SEARCH_RESULTS_AFTER_RERANK) + render = get_hook_spec(H.SEARCH_CONTEXT_RENDER) -def test_search_context_render_hook_can_replace_results(): - handler, _cube_view = _search_handler() - search_req = APISearchRequest(user_id="user", query="query") - - def render(*, results, **_kwargs): - return {**results, "rendered_by_plugin": True} - - register_hook(H.SEARCH_CONTEXT_RENDER, render) - - response = handler.handle_search_memories(search_req) - - assert response.data["rendered_by_plugin"] is True - - -def test_search_post_processing_failure_triggers_only_post_process_hook(monkeypatch): - error = RuntimeError("post processing failed") - post_process_failures = [] - handler, _cube_view = _search_handler() - handler._apply_relativity_threshold = lambda _results, _relativity: _raise(error) - search_req = APISearchRequest(user_id="user", query="query") - monkeypatch.setattr( - "memos.api.handlers.search_handler.get_current_trace_id", - lambda: "trace-1", - ) - - register_hook( - H.SEARCH_POST_PROCESS_FAILED, - lambda **kwargs: post_process_failures.append(kwargs), - ) - with pytest.raises(RuntimeError, match="post processing failed"): - handler.handle_search_memories(search_req) + assert after_rerank is not None + assert after_rerank.pipe_key == "results" + assert after_rerank.params == ["hook_context", "handler", "search_req", "results"] - assert len(post_process_failures) == 1 - assert post_process_failures[0]["error"] is error - context = post_process_failures[0]["hook_context"] - assert isinstance(context, HookContext) - assert context.trace_id == "trace-1" - assert context.user_id == "user" - assert context.operation_id is not None + assert render is not None + assert render.pipe_key == "results" + assert render.params == ["hook_context", "handler", "search_req", "results"] def test_rerank_knowledge_mem_preserves_conversation_sources_by_default(): diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..509111b2b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.fixture +def clean_hooks(): + """Keep Hook registrations isolated for tests that opt into this fixture.""" + from memos.plugins.hooks import _hooks + + _hooks.clear() + yield + _hooks.clear() diff --git a/tests/mem_reader/test_fine_transfer_order.py b/tests/mem_reader/test_fine_transfer_order.py new file mode 100644 index 000000000..2beacfbe2 --- /dev/null +++ b/tests/mem_reader/test_fine_transfer_order.py @@ -0,0 +1,44 @@ +"""Fine-transfer groups retain their input index when tasks finish out of order.""" + +from types import SimpleNamespace + +import pytest + +from memos.mem_reader.simple_struct import SimpleStructMemReader + + +@pytest.mark.parametrize("failed_result", [None, ValueError("synthetic failure")]) +def test_fine_transfer_keeps_failed_slots_and_input_order(monkeypatch, failed_result): + monkeypatch.setattr( + "memos.mem_reader.simple_struct.concurrent.futures.as_completed", + lambda futures: iter(reversed(list(futures))), + ) + + def process(source, custom_tags, **kwargs): + if source == "missing": + if isinstance(failed_result, Exception): + raise failed_result + return failed_result + return [source + "-enhanced"] + + reader = SimpleNamespace(_process_transfer_chat_data=process) + result = SimpleStructMemReader.fine_transfer_simple_mem( + reader, ["first", "missing", "last"], "chat" + ) + assert result == [["first-enhanced"], [], ["last-enhanced"]] + + +def test_multimodal_fine_transfer_returns_one_group_for_the_batch(): + from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader + + sources = ["first", "last"] + enhanced = ["combined-enhanced"] + + def process(memories, custom_tags, **kwargs): + assert memories is sources + return enhanced + + reader = SimpleNamespace(_process_transfer_multi_modal_data=process) + result = MultiModalStructMemReader.fine_transfer_simple_mem(reader, sources, "chat") + assert result == [enhanced] + assert result[0] is enhanced diff --git a/tests/mem_reader/test_hook_pipeline.py b/tests/mem_reader/test_hook_pipeline.py index 986fcffda..d912f0ab2 100644 --- a/tests/mem_reader/test_hook_pipeline.py +++ b/tests/mem_reader/test_hook_pipeline.py @@ -1,60 +1,37 @@ -"""MemReader success/failure Hook contracts and result piping.""" - -from __future__ import annotations +"""MemReader Hook contracts at the extraction boundary.""" from types import MethodType -from uuid import UUID import pytest from memos.mem_reader.simple_struct import SimpleStructMemReader -from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H -from memos.plugins.hooks import _hooks, register_hook +from memos.plugins.hooks import register_hook -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() +pytestmark = pytest.mark.usefixtures("clean_hooks") -def test_extract_after_hook_can_replace_reader_result(): +def test_extract_after_hook_receives_operation_and_can_replace_result(monkeypatch): reader = object.__new__(SimpleStructMemReader) original_result = [["original"]] plugin_result = [["plugin"]] - - def fake_read_memory(self, *args, **kwargs): - return original_result - - reader._read_memory = MethodType(fake_read_memory, reader) - register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **_kwargs: plugin_result) - - result = reader.get_memory( - [{"role": "user", "content": "remember this"}], - type="chat", - info={"user_id": "alice", "session_id": "session-1"}, - ) - - assert result == plugin_result - - -def test_extract_after_hook_receives_context_and_business_values(monkeypatch): - reader = object.__new__(SimpleStructMemReader) - result = [["memory"]] scene_data = [{"role": "user", "content": "remember this"}] info = {"user_id": "alice", "session_id": "session-1"} captured = {} - def fake_read_memory(self, *args, **kwargs): - return result + def fake_read_memory(self, *_args, **_kwargs): + return original_result + + def replace_result(**kwargs): + captured.update(kwargs) + return plugin_result reader._read_memory = MethodType(fake_read_memory, reader) monkeypatch.setattr("memos.mem_reader.simple_struct.get_current_trace_id", lambda: "trace-1") - register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **kwargs: captured.update(kwargs)) + register_hook(H.MEM_READER_EXTRACT_AFTER, replace_result) - actual = reader.get_memory( + result = reader.get_memory( scene_data, type="chat", info=info, @@ -63,17 +40,13 @@ def fake_read_memory(self, *args, **kwargs): chat_history=[], ) - assert actual is result context = captured.pop("hook_context") - assert isinstance(context, HookContext) + assert result == plugin_result assert context.trace_id == "trace-1" assert context.user_id == "alice" assert context.session_id == "session-1" assert context.cube_ids == ("cube-1",) assert context.source == "mem_reader.extract" - assert context.attributes == {"mode": "fine", "type": "chat"} - assert context.operation_id is not None - assert UUID(context.operation_id).hex == context.operation_id assert captured == { "mem_reader": reader, "scene_data": scene_data, @@ -82,40 +55,39 @@ def fake_read_memory(self, *args, **kwargs): "mode": "fine", "user_name": "cube-1", "kwargs": {"chat_history": []}, - "result": result, + "result": original_result, } -def test_extract_failure_hook_preserves_original_error_and_skips_after(monkeypatch): +def test_extract_failure_hook_reports_and_preserves_original_error(monkeypatch): reader = object.__new__(SimpleStructMemReader) error = ValueError("reader failed") - failed_events = [] - after_events = [] + events = [] - def fake_read_memory(self, *args, **kwargs): + def fake_read_memory(self, *_args, **_kwargs): raise error - def on_failed(**kwargs): - failed_events.append(kwargs) - raise RuntimeError("plugin failed") + def record(stage): + def callback(**kwargs): + events.append((stage, kwargs)) + + return callback reader._read_memory = MethodType(fake_read_memory, reader) monkeypatch.setattr("memos.mem_reader.simple_struct.get_current_trace_id", lambda: "trace-1") - register_hook(H.MEM_READER_EXTRACT_FAILED, on_failed) - register_hook(H.MEM_READER_EXTRACT_AFTER, lambda **kwargs: after_events.append(kwargs)) + register_hook(H.MEM_READER_EXTRACT_FAILED, record("failed")) + register_hook(H.MEM_READER_EXTRACT_AFTER, record("after")) with pytest.raises(ValueError, match="reader failed") as exc_info: reader.get_memory( [{"role": "user", "content": "remember this"}], type="chat", info={"user_id": "alice", "session_id": "session-1"}, - mode="fast", user_name="cube-1", ) assert exc_info.value is error - assert len(failed_events) == 1 - assert failed_events[0]["error"] is error - assert failed_events[0]["hook_context"].source == "mem_reader.extract" - assert failed_events[0]["hook_context"].operation_id is not None - assert after_events == [] + assert [stage for stage, _kwargs in events] == ["failed"] + failure = events[0][1] + assert failure["error"] is error + assert failure["hook_context"].source == "mem_reader.extract" diff --git a/tests/mem_scheduler/test_mem_read_grouping.py b/tests/mem_scheduler/test_mem_read_grouping.py new file mode 100644 index 000000000..c6d0e003d --- /dev/null +++ b/tests/mem_scheduler/test_mem_read_grouping.py @@ -0,0 +1,132 @@ +"""Fine-transfer Hooks describe the actual Reader result grouping.""" + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.mem_reader.simple_struct import SimpleStructMemReader +from memos.mem_reader.strategy_struct import StrategyStructMemReader +from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import ( + MemReadMessageHandler, +) +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.plugins.hook_defs import H +from memos.plugins.hooks import register_hook + + +class CustomTransferReader(SimpleStructMemReader): + def fine_transfer_simple_mem(self, *args, **kwargs): + return self.transfer_result + + +@pytest.fixture +def captured_operations(clean_hooks): + captured = [] + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, lambda **kwargs: captured.append(kwargs)) + return captured + + +def memory(text): + return TextualMemoryItem( + memory=text, + metadata=TreeNodeTextualMemoryMetadata(memory_type="LongTermMemory"), + ) + + +@pytest.mark.parametrize( + "reader_class,expected_grouping", + [ + (SimpleStructMemReader, "per_input"), + (StrategyStructMemReader, "per_input"), + (MultiModalStructMemReader, "batch"), + (CustomTransferReader, "unknown"), + ], +) +def test_scheduler_hook_reports_reader_grouping_without_changing_results( + monkeypatch, captured_operations, reader_class, expected_grouping +): + sources = [memory("first"), memory("missing"), memory("last")] + enhanced = [memory("first enhanced"), memory("last enhanced")] + reader = object.__new__(reader_class) + reader.save_rawfile = False + reader.graph_db = None + reader.memory_version_switch = "off" + reader.transfer_result = [enhanced] + + def process(source, custom_tags, **kwargs): + if source is sources[1]: + return None + return [enhanced[0] if source is sources[0] else enhanced[1]] + + reader._process_transfer_chat_data = process + reader._process_transfer_multi_modal_data = lambda *args, **kwargs: enhanced + monkeypatch.setattr( + "memos.mem_reader.simple_struct.concurrent.futures.as_completed", + lambda futures: iter(reversed(list(futures))), + ) + text_mem = Mock() + text_mem.get.side_effect = sources + text_mem.add.return_value = [item.id for item in enhanced] + text_mem.memory_manager.reorganizer.is_reorganize = False + handler = MemReadMessageHandler( + SimpleNamespace(get_mem_reader=lambda: reader, services=Mock(), get_mem_cube=Mock()) + ) + handler._process_memories_with_reader( + mem_ids=[item.id for item in sources], + user_id="test-user", + mem_cube_id="test-cube", + text_mem=text_mem, + user_name="test-cube", + operation_context={ + "trace_id": "test-trace", + "user_id": "test-user", + "cube_id": "test-cube", + }, + ) + + transfer = next( + event + for event in captured_operations + if event["hook_context"].source == "scheduler.mem_read.fine_transfer_simple_mem" + ) + assert transfer["operation_input"]["result_grouping"] == expected_grouping + assert transfer["operation_input"]["memories"] == sources + if reader_class in (SimpleStructMemReader, StrategyStructMemReader): + assert transfer["result"] == [[enhanced[0]], [], [enhanced[1]]] + else: + assert transfer["result"] == [enhanced] + text_mem.add.assert_called_once_with(enhanced, user_name="test-cube") + text_mem.delete.assert_called_once_with([item.id for item in sources], user_name="test-cube") + text_mem.memory_manager.remove_and_refresh_memory.assert_called_once_with(user_name="test-cube") + + +def test_scheduler_failed_hook_keeps_grouping_and_original_error(captured_operations): + error = ValueError("synthetic transfer failure") + reader = CustomTransferReader.__new__(CustomTransferReader) + + def fail(*args, **kwargs): + raise error + + reader.fine_transfer_simple_mem = fail + failed = [] + register_hook(H.SCHEDULER_MEMORY_OPERATION_FAILED, lambda **kwargs: failed.append(kwargs)) + text_mem = Mock() + text_mem.get.return_value = memory("source") + handler = MemReadMessageHandler(SimpleNamespace(get_mem_reader=lambda: reader)) + handler._process_memories_with_reader( + mem_ids=[text_mem.get.return_value.id], + user_id="test-user", + mem_cube_id="test-cube", + text_mem=text_mem, + user_name="test-cube", + ) + assert len(failed) == 1 + assert failed[0]["error"] is error + assert failed[0]["operation_input"]["result_grouping"] == "unknown" + assert all( + event["hook_context"].source != "scheduler.mem_read.fine_transfer_simple_mem" + for event in captured_operations + ) + text_mem.add.assert_not_called() diff --git a/tests/mem_scheduler/test_operation_observer.py b/tests/mem_scheduler/test_operation_observer.py index b8513e5d5..6b4beaa5e 100644 --- a/tests/mem_scheduler/test_operation_observer.py +++ b/tests/mem_scheduler/test_operation_observer.py @@ -1,33 +1,126 @@ -"""Scheduler operation Hook contracts at existing exception boundaries.""" +"""Scheduler Hook contracts and their MemRead integration points.""" -from __future__ import annotations - -from uuid import UUID +from unittest.mock import MagicMock import pytest +from memos.mem_scheduler.schemas.message_schemas import ScheduleMessageItem +from memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler import ( + MemReadMessageHandler, +) from memos.mem_scheduler.task_schedule_modules.operation_observer import ( SchedulerOperationObserver, ) -from memos.plugins.hook_context import HookContext +from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata from memos.plugins.hook_defs import H -from memos.plugins.hooks import _hooks, register_hook +from memos.plugins.hooks import register_hook + + +pytestmark = pytest.mark.usefixtures("clean_hooks") + +RAW_MEMORY_ID = "00000000-0000-0000-0000-000000000001" +ENHANCED_MEMORY_ID = "00000000-0000-0000-0000-000000000002" +SCHEDULER_ITEM_ID = "00000000-0000-0000-0000-000000000003" +OPERATION_SOURCE = { + "fine": "fine_transfer_simple_mem", + "add": "add_enhanced_memories", + "archive": "archive_merged_memories", + "delete": "remove_memories", + "soft_delete": "remove_memories", +} + + +def _memory(memory_id, memory_type, *, merged_from=None): + return TextualMemoryItem( + id=memory_id, + memory=f"{memory_type} content", + metadata=TreeNodeTextualMemoryMetadata( + user_id="alice", + session_id="session-1", + memory_type=memory_type, + sources=[], + info={"merged_from": merged_from} if merged_from else {}, + ), + ) -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() +def _mem_read_dependencies(memory_version_switch): + raw_memory = _memory(RAW_MEMORY_ID, "WorkingMemory") + enhanced_memory = _memory( + ENHANCED_MEMORY_ID, + "LongTermMemory", + merged_from=RAW_MEMORY_ID, + ) + reader = MagicMock() + reader.fine_transfer_simple_mem.return_value = [[enhanced_memory]] + reader.save_rawfile = False + reader.memory_version_switch = memory_version_switch + + text_memory = MagicMock() + text_memory.get.return_value = raw_memory + text_memory.add.return_value = [ENHANCED_MEMORY_ID] + text_memory.memory_manager.reorganizer.is_reorganize = False + + scheduler_context = MagicMock() + scheduler_context.get_mem_reader.return_value = reader + scheduler_context.services.create_event_log.return_value = MagicMock() + + message = ScheduleMessageItem( + item_id=SCHEDULER_ITEM_ID, + user_id="alice", + trace_id="trace-1", + mem_cube_id="cube-1", + session_id="session-1", + label="mem_read", + content=f'["{RAW_MEMORY_ID}"]', + user_name="cube-1", + task_id="task-1", + ) + return MemReadMessageHandler(scheduler_context), reader, text_memory, message -def test_scheduler_observer_emits_context_and_business_values_as_hook_kwargs(): - captured = {} - register_hook( - H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **kwargs: captured.update(kwargs), + +def test_mem_read_handler_uses_replaced_transfer_results_and_stored_ids(monkeypatch): + handler, _reader, text_memory, message = _mem_read_dependencies("off") + replacement = _memory("00000000-0000-0000-0000-000000000004", "LongTermMemory") + stored_id = "00000000-0000-0000-0000-000000000005" + text_memory.memory_manager.reorganizer.is_reorganize = True + monkeypatch.setattr( + "memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler.is_playground_api", + lambda: False, ) + def replace_result(*, operation, result, **_kwargs): + if operation == "fine": + return [[replacement]] + if operation == "add": + return [stored_id] + return result + + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, replace_result) + handler._process_memories_with_reader( + mem_ids=[RAW_MEMORY_ID], + user_id="alice", + mem_cube_id="cube-1", + text_mem=text_memory, + user_name="cube-1", + operation_context=message, + ) + + text_memory.add.assert_called_once_with([replacement], user_name="cube-1") + submitted = handler.scheduler_context.services.submit_messages.call_args.args[0] + assert submitted[0].content == f'["{stored_id}"]' + + +def test_scheduler_observer_pipes_result_and_exposes_operation_contract(): + captured = {} + + def replace_result(**kwargs): + captured.update(kwargs) + return ["plugin-result"] + + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, replace_result) + with SchedulerOperationObserver( source="scheduler.mem_read.add_enhanced_memories", operation="add", @@ -37,42 +130,30 @@ def test_scheduler_observer_emits_context_and_business_values_as_hook_kwargs(): ) as observer: observer.result(["stored-1"]) - assert isinstance(captured["hook_context"], HookContext) - assert captured["hook_context"].trace_id == "trace-1" + assert observer.value == ["plugin-result"] assert captured["operation"] == "add" assert captured["target"] == "textual_memory" assert captured["operation_input"] == {"memory_ids": ["memory-1"]} assert captured["result"] == ["stored-1"] - assert "event" not in captured + assert captured["hook_context"].source == "scheduler.mem_read.add_enhanced_memories" -def test_scheduler_observer_failure_hook_preserves_original_error_and_skips_after(): +def test_scheduler_observer_reports_failure_and_preserves_error(): error = ValueError("scheduler operation failed") - failed_events = [] - after_events = [] - - def on_failed(**kwargs): - failed_events.append(kwargs) - raise RuntimeError("plugin failed") - - register_hook(H.SCHEDULER_MEMORY_OPERATION_FAILED, on_failed) + events = [] + register_hook( + H.SCHEDULER_MEMORY_OPERATION_FAILED, + lambda **kwargs: events.append(("failed", kwargs)), + ) register_hook( H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **kwargs: after_events.append(kwargs), + lambda **kwargs: events.append(("after", kwargs)), ) observer = SchedulerOperationObserver( source="scheduler.mem_read.fine_transfer_simple_mem", operation="fine", target="textual_memory", - operation_context={ - "trace_id": "trace-1", - "user_id": "alice", - "session_id": "session-1", - "task_id": "task-1", - "mem_cube_id": "cube-1", - "item_id": "item-1", - "label": "mem_read", - }, + operation_context={"trace_id": "trace-1", "mem_cube_id": "cube-1"}, operation_input={"memory_ids": ["memory-1"]}, capture_failure=True, ) @@ -81,70 +162,87 @@ def on_failed(**kwargs): raise error assert exc_info.value is error - assert len(failed_events) == 1 - event = failed_events[0] - context = event["hook_context"] - assert isinstance(context, HookContext) - assert context.trace_id == "trace-1" - assert context.user_id == "alice" - assert context.session_id == "session-1" - assert context.task_id == "task-1" - assert context.cube_ids == ("cube-1",) - assert context.source == "scheduler.mem_read.fine_transfer_simple_mem" - assert context.attributes == {"scheduler_item_id": "item-1", "label": "mem_read"} - assert context.operation_id is not None - assert UUID(context.operation_id).hex == context.operation_id - assert event["operation"] == "fine" - assert event["target"] == "textual_memory" - assert event["operation_input"] == {"memory_ids": ["memory-1"]} - assert event["error"] is error - assert after_events == [] - assert observer.value is None - - -def test_scheduler_observer_emits_explicit_failure_only_once(): - failed_events = [] - after_events = [] - error = ValueError("scheduler operation failed") - - register_hook( - H.SCHEDULER_MEMORY_OPERATION_FAILED, - lambda **kwargs: failed_events.append(kwargs), - ) - register_hook( - H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **kwargs: after_events.append(kwargs), + assert [stage for stage, _kwargs in events] == ["failed"] + failure = events[0][1] + assert failure["error"] is error + assert failure["hook_context"].cube_ids == ("cube-1",) + + +@pytest.mark.parametrize( + ("memory_version_switch", "expected_operations"), + [ + ("off", ["fine", "add", "archive", "delete"]), + ("on", ["fine", "add", "soft_delete"]), + ], +) +def test_mem_read_handler_exposes_each_memory_operation( + memory_version_switch, + expected_operations, + monkeypatch, +): + handler, reader, text_memory, message = _mem_read_dependencies(memory_version_switch) + events = [] + monkeypatch.setattr( + "memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler.is_playground_api", + lambda: False, ) - with SchedulerOperationObserver( - source="scheduler.mem_read.remove_memories", - operation="delete", - target="textual_memory", - operation_context={"trace_id": "trace-1"}, - operation_input={"memory_ids": ["memory-1"]}, - ) as observer: - observer.failed(error) - observer.failed(error) - observer.result(None) - - assert len(failed_events) == 1 - assert failed_events[0]["error"] is error - assert after_events == [] + def record_operation(**kwargs): + events.append(kwargs) + return kwargs["result"] + register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, record_operation) -def test_scheduler_observer_exposes_piped_result_after_successful_exit(): - register_hook( - H.SCHEDULER_MEMORY_OPERATION_AFTER, - lambda **_kwargs: ["plugin-result"], + handler._process_memories_with_reader( + mem_ids=[RAW_MEMORY_ID], + user_id="alice", + mem_cube_id="cube-1", + text_mem=text_memory, + user_name="cube-1", + task_id="task-1", + operation_context=message, ) - with SchedulerOperationObserver( - source="scheduler.mem_read.add_enhanced_memories", - operation="add", - target="textual_memory", - operation_context={"trace_id": "trace-1"}, - operation_input={}, - ) as observer: - observer.result(["original-result"]) - - assert observer.value == ["plugin-result"] + assert [event["operation"] for event in events] == expected_operations + assert [event["hook_context"].source for event in events] == [ + f"scheduler.mem_read.{OPERATION_SOURCE[operation]}" for operation in expected_operations + ] + for event in events: + context = event["hook_context"] + assert event["target"] == "textual_memory" + assert context.trace_id == "trace-1" + assert context.user_id == "alice" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube-1",) + assert context.attributes["scheduler_item_id"] == SCHEDULER_ITEM_ID + assert context.attributes["label"] == "mem_read" + + inputs = {event["operation"]: event["operation_input"] for event in events} + assert inputs["fine"]["memories"][0].id == RAW_MEMORY_ID + assert inputs["fine"]["type"] == "chat" + assert inputs["add"]["memories"][0].id == ENHANCED_MEMORY_ID + assert inputs["add"]["user_name"] == "cube-1" + + if memory_version_switch == "off": + assert inputs["archive"] == {"memory_ids": [RAW_MEMORY_ID]} + assert inputs["delete"] == {"memory_ids": [RAW_MEMORY_ID]} + reader.graph_db.update_node.assert_called_once_with( + RAW_MEMORY_ID, + {"status": "archived"}, + user_name="cube-1", + ) + text_memory.delete.assert_called_once_with([RAW_MEMORY_ID], user_name="cube-1") + text_memory.soft_delete.assert_not_called() + else: + assert inputs["soft_delete"] == { + "memory_ids": [RAW_MEMORY_ID], + "preserved_memory_ids": [ENHANCED_MEMORY_ID], + } + reader.graph_db.update_node.assert_not_called() + text_memory.delete.assert_not_called() + text_memory.soft_delete.assert_called_once_with( + [RAW_MEMORY_ID], + "cube-1", + [ENHANCED_MEMORY_ID], + ) diff --git a/tests/multi_mem_cube/test_single_cube_hooks.py b/tests/multi_mem_cube/test_single_cube_hooks.py index b384157ba..f639bec8e 100644 --- a/tests/multi_mem_cube/test_single_cube_hooks.py +++ b/tests/multi_mem_cube/test_single_cube_hooks.py @@ -3,23 +3,17 @@ from __future__ import annotations import logging -import uuid from importlib import import_module from unittest.mock import MagicMock import pytest -from memos.plugins.hook_context import HookContext from memos.plugins.hook_defs import H -from memos.plugins.hooks import _hooks, register_hook +from memos.plugins.hooks import register_hook -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() +pytestmark = pytest.mark.usefixtures("clean_hooks") def _make_add_request(**overrides): @@ -48,7 +42,7 @@ def single_cube_view(): from memos.multi_mem_cube.single_cube import SingleCubeView memory = TextualMemoryItem( - id=str(uuid.uuid4()), + id="00000000-0000-0000-0000-000000000001", memory="hello world", metadata=TreeNodeTextualMemoryMetadata( user_id="u1", @@ -79,15 +73,38 @@ def single_cube_view(): return view, memory -def test_text_memory_after_hook_can_replace_memory_ids(single_cube_view): - view, _memory = single_cube_view - register_hook(H.TEXT_MEMORY_ADD_AFTER, lambda **_kwargs: ["plugin-memory-id"]) +def test_text_memory_after_hook_observes_write_and_replaces_memory_ids(single_cube_view): + view, memory = single_cube_view + captured = {} + + def replace_memory_ids(**kwargs): + captured.update(kwargs) + return ["plugin-memory-id"] - results = view.add_memories(_make_add_request(async_mode="async")) + register_hook(H.TEXT_MEMORY_ADD_AFTER, replace_memory_ids) + + results = view.add_memories( + _make_add_request( + async_mode="async", + session_id="session-1", + task_id="task-1", + ) + ) assert results[0]["memory_id"] == "plugin-memory-id" + context = captured.pop("hook_context") + assert context.user_id == "test_user" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube_test",) + assert context.source == "cube_view.text_mem.add" + assert captured == { + "text_memory": view.naive_mem_cube.text_mem, + "memories": [memory], + "kwargs": {"user_name": "cube_test"}, + "result": [memory.id], + } submitted_message = view.mem_scheduler.submit_messages.call_args.kwargs["messages"][0] - assert submitted_message.mem_cube_id == "cube_test" assert submitted_message.content == '["plugin-memory-id"]' @@ -97,20 +114,21 @@ def test_text_memory_failure_hook_preserves_original_error_and_skips_after( ): view, memory = single_cube_view error = ValueError("text memory add failed") - failed_calls = [] - after_calls = [] + events = [] view.naive_mem_cube.text_mem.add.side_effect = error monkeypatch.setattr( "memos.multi_mem_cube.single_cube.get_current_trace_id", lambda: "trace-1", ) - def on_failed(**kwargs): - failed_calls.append(kwargs) - raise RuntimeError("plugin failed") + def record(stage): + def callback(**kwargs): + events.append((stage, kwargs)) - register_hook(H.TEXT_MEMORY_ADD_FAILED, on_failed) - register_hook(H.TEXT_MEMORY_ADD_AFTER, lambda **kwargs: after_calls.append(kwargs)) + return callback + + register_hook(H.TEXT_MEMORY_ADD_FAILED, record("failed")) + register_hook(H.TEXT_MEMORY_ADD_AFTER, record("after")) with pytest.raises(ValueError, match="text memory add failed") as exc_info: view.add_memories( @@ -122,19 +140,13 @@ def on_failed(**kwargs): ) assert exc_info.value is error - assert after_calls == [] - assert len(failed_calls) == 1 - call = failed_calls[0] + assert [stage for stage, _kwargs in events] == ["failed"] + call = events[0][1] context = call["hook_context"] - assert isinstance(context, HookContext) assert context.trace_id == "trace-1" assert context.user_id == "test_user" - assert context.session_id == "session-1" - assert context.task_id == "task-1" assert context.cube_ids == ("cube_test",) assert context.source == "cube_view.text_mem.add" - assert context.attributes == {"sync_mode": "async", "extract_mode": "fast"} - assert context.operation_id is not None assert call["text_memory"] is view.naive_mem_cube.text_mem assert call["memories"] == [memory] assert call["kwargs"] == {"user_name": "cube_test"} diff --git a/tests/plugins/test_hook_context.py b/tests/plugins/test_hook_context.py index c1fdad0bd..ad06e2f98 100644 --- a/tests/plugins/test_hook_context.py +++ b/tests/plugins/test_hook_context.py @@ -1,41 +1,44 @@ -"""HookContext values, callback compatibility, and @hookable propagation.""" - -from __future__ import annotations +"""Public HookContext contracts and context-aware callback behavior.""" import asyncio -from collections.abc import Mapping -from typing import Any, get_type_hints -from uuid import UUID - import pytest -from memos.plugins.hook_context import HookContext, build_hook_context +from memos.plugins.hook_context import build_hook_context from memos.plugins.hook_defs import H, define_hook, get_hook_spec -from memos.plugins.hooks import ( - _hooks, - hookable, - register_hook, - trigger_hook, - trigger_single_hook, +from memos.plugins.hooks import hookable, register_hook, trigger_hook, trigger_single_hook + + +pytestmark = pytest.mark.usefixtures("clean_hooks") + + +@pytest.mark.parametrize( + ("hook_name", "pipe_key"), + [ + (H.SEARCH_BEFORE, "request"), + (H.SEARCH_AFTER, "result"), + (H.SEARCH_MEMORY_RESULTS, "results"), + (H.SEARCH_RESULTS_AFTER_THRESHOLD, "results"), + (H.SEARCH_RESULTS_AFTER_DEDUP, "results"), + (H.SEARCH_RESULTS_AFTER_RERANK, "results"), + (H.SEARCH_CONTEXT_RENDER, "results"), + (H.SEARCH_POST_PROCESS_FAILED, None), + (H.TEXT_MEMORY_ADD_AFTER, "result"), + (H.TEXT_MEMORY_ADD_FAILED, None), + (H.SCHEDULER_MEMORY_OPERATION_AFTER, "result"), + (H.SCHEDULER_MEMORY_OPERATION_FAILED, None), + (H.MEM_READER_EXTRACT_AFTER, "result"), + (H.MEM_READER_EXTRACT_FAILED, None), + ], ) +def test_integration_hook_specs_are_registered(hook_name, pipe_key): + spec = get_hook_spec(hook_name) + assert spec is not None + assert spec.pipe_key == pipe_key -@pytest.fixture(autouse=True) -def _reset_registered_hooks(): - _hooks.clear() - yield - _hooks.clear() - - -def test_hook_context_type_hints_are_runtime_resolvable(): - hints = get_type_hints(HookContext) - assert hints["attributes"] == Mapping[str, Any] - assert hints["operation_id"] == str | None - - -def test_build_hook_context_populates_correlation_metadata(): +def test_build_hook_context_copies_correlation_metadata(): cube_ids = ["cube-1", "cube-2"] attributes = {"mode": "fine"} @@ -46,95 +49,26 @@ def test_build_hook_context_populates_correlation_metadata(): session_id="session-1", task_id="task-1", cube_ids=cube_ids, - operation_id="operation-1", attributes=attributes, ) cube_ids.append("cube-3") - attributes["mode"] = "mutated" + attributes["mode"] = "fast" - assert context == HookContext( - trace_id="trace-1", - user_id="user-1", - session_id="session-1", - task_id="task-1", - cube_ids=("cube-1", "cube-2"), - operation_id="operation-1", - source="mem_reader.extract", - attributes={"mode": "fine"}, - ) + assert context.trace_id == "trace-1" + assert context.user_id == "user-1" + assert context.session_id == "session-1" + assert context.task_id == "task-1" + assert context.cube_ids == ("cube-1", "cube-2") + assert context.source == "mem_reader.extract" + assert context.attributes == {"mode": "fine"} + assert context.operation_id -def test_build_hook_context_generates_unique_operation_ids(): - first = build_hook_context(source="text_memory.add") - second = build_hook_context(source="text_memory.add") - - assert first.operation_id is not None - assert second.operation_id is not None - assert UUID(first.operation_id).hex == first.operation_id - assert UUID(second.operation_id).hex == second.operation_id - assert first.operation_id != second.operation_id - - -def test_context_aware_hook_specs_match_trigger_contracts(): - expected = { - H.TEXT_MEMORY_ADD_AFTER: ( - ["hook_context", "text_memory", "memories", "kwargs", "result"], - "result", - ), - H.TEXT_MEMORY_ADD_FAILED: ( - ["hook_context", "text_memory", "memories", "kwargs", "error"], - None, - ), - H.SCHEDULER_MEMORY_OPERATION_AFTER: ( - ["hook_context", "operation", "target", "operation_input", "result"], - "result", - ), - H.SCHEDULER_MEMORY_OPERATION_FAILED: ( - ["hook_context", "operation", "target", "operation_input", "error"], - None, - ), - H.MEM_READER_EXTRACT_AFTER: ( - [ - "hook_context", - "mem_reader", - "scene_data", - "type", - "info", - "mode", - "user_name", - "kwargs", - "result", - ], - "result", - ), - H.MEM_READER_EXTRACT_FAILED: ( - [ - "hook_context", - "mem_reader", - "scene_data", - "type", - "info", - "mode", - "user_name", - "kwargs", - "error", - ], - None, - ), - } - - for hook_name, (params, pipe_key) in expected.items(): - spec = get_hook_spec(hook_name) - assert spec is not None - assert spec.params == params - assert spec.pipe_key == pipe_key - - -def test_hook_context_is_omitted_only_for_legacy_exact_signature_callbacks(): - hook_name = "test.hook-context.compat" +def test_hook_context_is_additive_for_new_and_legacy_callbacks(): + hook_name = "test.hook-context.compatibility" define_hook( hook_name, - description="hook context compatibility test", + description="HookContext compatibility", params=["hook_context", "result"], pipe_key="result", ) @@ -149,164 +83,106 @@ def context_callback(*, hook_context, result): calls.append(("context", hook_context, result)) return result + 1 - def kwargs_callback(*, result, **kwargs): - calls.append(("kwargs", kwargs["hook_context"], result)) - return result + 1 - register_hook(hook_name, legacy_callback) register_hook(hook_name, context_callback) - register_hook(hook_name, kwargs_callback) result = trigger_hook(hook_name, hook_context=context, result=1) - assert result == 4 - assert calls == [ - ("legacy", 1), - ("context", context, 2), - ("kwargs", context, 3), - ] - - -def test_hook_callback_type_error_is_not_retried(): - hook_name = "test.hook-context.type-error" - define_hook( - hook_name, - description="hook callback type error test", - params=["hook_context", "value"], - ) - calls = [] - - def callback(*, hook_context, value): - calls.append((hook_context, value)) - raise TypeError("raised inside callback") - - register_hook(hook_name, callback) - - trigger_hook(hook_name, hook_context="context", value="value") - - assert calls == [("context", "value")] + assert result == 3 + assert calls == [("legacy", 1), ("context", context, 2)] -def test_trigger_single_hook_supports_legacy_exact_signature(): +def test_single_provider_hook_supports_legacy_callback(): hook_name = "test.single.hook-context" define_hook( hook_name, - description="single hook context compatibility test", + description="Single-provider HookContext compatibility", params=["hook_context", "value"], ) + calls = [] - def handler(*, value): + def callback(*, value): + calls.append(value) return value + 1 - register_hook(hook_name, handler) + register_hook(hook_name, callback) - result = trigger_single_hook(hook_name, hook_context=object(), value=1) + assert trigger_single_hook(hook_name, hook_context=object(), value=1) == 2 + assert calls == [1] - assert result == 2 +def test_hookable_shares_and_reuses_context_for_sync_operation(): + built_context = object() + supplied_context = object() + builder_calls = [] + observed = [] -def test_hookable_shares_context_with_sync_operation_and_legacy_callback(): - context = object() - calls = [] + def build_context(_handler, request): + builder_calls.append(request) + return built_context class Handler: - @hookable( - "context_sync", - context_builder=lambda _self, _request: context, - ) + @hookable("context_sync", context_builder=build_context) def run(self, request, *, hook_context=None): - calls.append(("operation", hook_context, request)) - return f"processed:{request}" - - def legacy_before(*, request): - calls.append(("legacy_before", request)) + observed.append(("operation", hook_context)) + return request - def context_before(*, hook_context, request): - calls.append(("context_before", hook_context, request)) + def before(*, hook_context, **_kwargs): + observed.append(("before", hook_context)) - def context_after(*, hook_context, request, result): - calls.append(("context_after", hook_context, request, result)) + def after(*, hook_context, **_kwargs): + observed.append(("after", hook_context)) - register_hook("context_sync.before", legacy_before) - register_hook("context_sync.before", context_before) - register_hook("context_sync.after", context_after) + register_hook("context_sync.before", before) + register_hook("context_sync.after", after) - result = Handler().run("request") + Handler().run("built") + Handler().run("supplied", hook_context=supplied_context) - assert result == "processed:request" - assert calls == [ - ("legacy_before", "request"), - ("context_before", context, "request"), - ("operation", context, "request"), - ("context_after", context, "request", "processed:request"), - ] - before_spec = get_hook_spec("context_sync.before") - after_spec = get_hook_spec("context_sync.after") - assert before_spec is not None - assert after_spec is not None - assert before_spec.params == ["hook_context", "request"] - assert after_spec.params == [ - "hook_context", - "request", - "result", + assert builder_calls == ["built"] + assert observed == [ + ("before", built_context), + ("operation", built_context), + ("after", built_context), + ("before", supplied_context), + ("operation", supplied_context), + ("after", supplied_context), ] -def test_hookable_reuses_explicit_context_without_calling_builder(): +def test_hookable_shares_and_reuses_context_for_async_operation(): + built_context = object() supplied_context = object() builder_calls = [] - observed_contexts = [] + observed = [] - def build_context(_self, request): + def build_context(_handler, request): builder_calls.append(request) - return object() + return built_context class Handler: - @hookable("explicit_context", context_builder=build_context) - def run(self, request, *, hook_context=None): - observed_contexts.append(hook_context) - return request - - def observe_context(*, hook_context, **_kwargs): - observed_contexts.append(hook_context) - - register_hook("explicit_context.before", observe_context) - register_hook("explicit_context.after", observe_context) - - result = Handler().run("request", hook_context=supplied_context) - - assert result == "request" - assert builder_calls == [] - assert observed_contexts == [supplied_context] * 3 - - -def test_hookable_shares_context_with_async_operation(): - context = object() - calls = [] - - class Handler: - @hookable( - "context_async", - context_builder=lambda _self, _request: context, - ) + @hookable("context_async", context_builder=build_context) async def run(self, request, *, hook_context=None): - calls.append(("operation", hook_context, request)) - return "async_result" - - def on_before(*, hook_context, request): - calls.append(("before", hook_context, request)) - - def on_after(*, hook_context, request, result): - calls.append(("after", hook_context, request, result)) - - register_hook("context_async.before", on_before) - register_hook("context_async.after", on_after) - - result = asyncio.run(Handler().run("request")) + observed.append(("operation", hook_context)) + return request - assert result == "async_result" - assert calls == [ - ("before", context, "request"), - ("operation", context, "request"), - ("after", context, "request", "async_result"), + def observe(stage): + def callback(*, hook_context, **_kwargs): + observed.append((stage, hook_context)) + + return callback + + register_hook("context_async.before", observe("before")) + register_hook("context_async.after", observe("after")) + + assert asyncio.run(Handler().run("built")) == "built" + assert asyncio.run(Handler().run("supplied", hook_context=supplied_context)) == "supplied" + assert builder_calls == ["built"] + assert observed == [ + ("before", built_context), + ("operation", built_context), + ("after", built_context), + ("before", supplied_context), + ("operation", supplied_context), + ("after", supplied_context), ] From a070ac5a9935eb358212e6d52f04d75ceccdafa6 Mon Sep 17 00:00:00 2001 From: qzs Date: Fri, 16 Oct 2026 14:51:34 +0800 Subject: [PATCH 12/22] docs(plugins): clarify hook documentation and comments --- src/memos/plugins/README.md | 18 +----------------- src/memos/plugins/hook_defs.py | 4 ++-- src/memos/plugins/hooks.py | 2 +- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/memos/plugins/README.md b/src/memos/plugins/README.md index 859fe76d7..3832bca1e 100644 --- a/src/memos/plugins/README.md +++ b/src/memos/plugins/README.md @@ -286,23 +286,7 @@ Hook callbacks receive keyword arguments. Some Hooks define a `pipe_key`; when a callback returns a non-`None` value, that value replaces the named argument for later callbacks and for the caller. -`hook_context` is additive correlation metadata, not a separate event model. -Context-aware callbacks can declare it explicitly or accept `**kwargs`. Legacy -callbacks with an exact signature remain compatible: the runtime omits only an -unsupported `hook_context` argument and still invokes the callback once. New -callbacks should accept `**kwargs` so later additive metadata remains backward -compatible. - -All Search Hooks triggered for one request—from `SEARCH_BEFORE` through -`SEARCH_AFTER`, including result and post-processing Hooks—receive the same -`HookContext` instance and `operation_id`. - -The MemReader extraction, Textual Memory add, and successful Scheduler -operation Hooks pipe their `result` value. Their matching failure Hooks are -notifications and therefore do not define a `pipe_key`. - -`SEARCH_POST_PROCESS_FAILED` reports a result post-processing error that -continues to propagate to the Search API caller. +`hook_context` is additive correlation metadata. Context-aware callbacks can declare it explicitly or accept `**kwargs`. Legacy callbacks with an exact signature remain compatible: the runtime omits only an unsupported `hook_context` argument and still invokes the callback once. New callbacks should accept `**kwargs` so later additive metadata remains backward compatible. For example, `add.after` can replace `result`: diff --git a/src/memos/plugins/hook_defs.py b/src/memos/plugins/hook_defs.py index 87375e942..95cc493f6 100644 --- a/src/memos/plugins/hook_defs.py +++ b/src/memos/plugins/hook_defs.py @@ -109,7 +109,7 @@ class H: # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# CE Hook declarations +# CE custom Hook declarations (@hookable-generated ones need not be declared here) # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ define_hook( @@ -237,7 +237,7 @@ class H: ) # Operation-boundary Hooks. Their business data remains explicit in the -# normal Hook keyword-argument contract; HookContext only carries correlation +# normal keyword-argument contract. HookContext only carries correlation # metadata. define_hook( H.TEXT_MEMORY_ADD_AFTER, diff --git a/src/memos/plugins/hooks.py b/src/memos/plugins/hooks.py index 4128da3b7..084f0b66b 100644 --- a/src/memos/plugins/hooks.py +++ b/src/memos/plugins/hooks.py @@ -46,7 +46,7 @@ def _invoke_hook_callback(callback: Callable, kwargs: dict[str, Any]) -> Any: ``hook_context`` is additive framework metadata. Older callbacks that use an exact keyword-only signature continue to receive the original business - arguments; context-aware callbacks and callbacks with ``**kwargs`` receive + arguments. Context-aware callbacks and callbacks with ``**kwargs`` receive the complete contract. """ if "hook_context" not in kwargs or _accepts_hook_context(callback): From 2cbb3a8685c50b5f440fdf186ffbcde23c035136 Mon Sep 17 00:00:00 2001 From: qzs Date: Fri, 16 Oct 2026 17:16:03 +0800 Subject: [PATCH 13/22] docs(plugins): clarify hook documentation in README.md and HOOKS.md --- src/memos/plugins/HOOKS.md | 290 ++++++++++++++++++++++++++++++++++++ src/memos/plugins/README.md | 66 ++++++-- 2 files changed, 344 insertions(+), 12 deletions(-) create mode 100644 src/memos/plugins/HOOKS.md diff --git a/src/memos/plugins/HOOKS.md b/src/memos/plugins/HOOKS.md new file mode 100644 index 000000000..c28a63b9b --- /dev/null +++ b/src/memos/plugins/HOOKS.md @@ -0,0 +1,290 @@ +# Pipeline Hook Reference + +Use this reference when implementing a plugin that observes or modifies MemOS +search, extraction, or memory operations. For plugin registration and lifecycle, +start with the [plugin guide](README.md#using-hooks). + +Hook names and keyword parameters are declared in [hook_defs.py](hook_defs.py). +The runtime lives in [hooks.py](hooks.py). + +--- + +## How hooks work + +Every hook falls into one of two patterns: + +| Pattern | Return value matters? | API | Exception handling | +| --- | --- | --- | --- | +| **Pipeline hook** — has a `pipe_key` | Yes — non-`None` replaces that argument for subsequent callbacks | `trigger_hook` | Logged and swallowed; later callbacks still run | +| **Observer hook** — no `pipe_key` | No | `trigger_hook` | Logged and swallowed | +| **Single-provider hook** | Returned directly to caller | `trigger_single_hook` | Re-raised | + +Memory-version and Dream business hooks use the single-provider API; their declarations +are in [hook_defs.py](hook_defs.py). All hooks documented below use `trigger_hook`. + +### Quick example + +```python +from memos.plugins import H + +class MyPlugin(BasePlugin): + def on_enable(self): + # Observe results without changing them + self.register_hook(H.SEARCH_AFTER, self._on_search_done) + + # Replace results mid-pipeline + self.register_hook(H.SEARCH_RESULTS_AFTER_RERANK, self._rerank) + + def _on_search_done(self, request, result, **kwargs): + print(f"Search for '{request.query}' returned {len(result.memories)} results") + + def _rerank(self, results, **kwargs): + # pipe_key=results — return non-None to propagate the change + for bucket in results.values(): + bucket.sort(key=lambda m: my_score(m), reverse=True) + return results +``` + +--- + +## Registering callbacks + +- Call `self.register_hook(H.NAME, callback)` **once** per plugin lifecycle. Registration is + additive — calling it twice registers the callback twice. +- Callbacks run synchronously, in registration order. `@hookable`-decorated handlers support + async, but hook callbacks themselves are not awaited. +- Accept `**kwargs` in any callback to stay forward-compatible when a hook gains new arguments. + +--- + +## Pipeline callbacks (`pipe_key` present) + +The `pipe_key` column tells you which argument a return value replaces: + +- **Return a non-`None` value** → replaces that argument for all later callbacks and the caller. +- **Return `None`** → keeps the current value unchanged. +- `False`, `[]`, `{}`, and `0` are all **replacements** — only `None` means "pass-through". + +Because arguments are live objects, in-place mutations are visible to later callbacks even when +your callback returns `None`. Return a copy when isolation is needed, and preserve the shape +downstream code expects. + +--- + +## Observer callbacks (no `pipe_key`) + +Return values are ignored. These hooks are for side effects only — logging, metrics, +notifications, etc. The same live-object caveat applies: mutating an argument in-place +is visible downstream even though the callback's return is discarded. + +--- + +## Exception handling + +`trigger_hook` logs any `Exception` raised by a callback and continues to subsequent +callbacks. Mutations made before the exception are **not** rolled back. A crashing observer +does not, by itself, fail the business operation. + +The single-provider API (`trigger_single_hook`) propagates exceptions — see each hook's +documentation for which API it uses. + +--- + +## HookContext + +Context-aware hooks receive `hook_context` alongside their business arguments. +It carries correlation metadata, not the request, result, or exception payload. + +| Field | Meaning | +| --- | --- | +| `trace_id` | Trace identifier available at the operation boundary. | +| `user_id` | User identifier from the request, reader input, or scheduler message. | +| `session_id` | Session identifier when available. Search uses `default_session` when omitted. | +| `task_id` | Task identifier when available. | +| `cube_ids` | Tuple of relevant cube identifiers; empty when unavailable. | +| `operation_id` | UUID hex string identifying one observed operation. | +| `source` | Integration point that created the context; see the tables below. | +| `attributes` | Integration-specific metadata such as mode or scheduler item ID. | + +Fields unavailable at a boundary are `None`. Treat the context as read-only — the +dataclass is frozen, but `attributes` is not deeply immutable. The builder copies the +outer attributes mapping and converts cube IDs to a tuple. + +### Receiving `hook_context` in your callback + +Declare `hook_context` as a keyword parameter, or pull it from `**kwargs`: + +```python +# Option A — explicit parameter +def my_callback(self, results, hook_context, **kwargs): + print(hook_context.trace_id) + +# Option B — via kwargs (useful when the same handler is registered on hooks +# both with and without context) +def my_callback(self, results, **kwargs): + ctx = kwargs.get("hook_context") # None if hook does not provide it +``` + +The runtime omits `hook_context` when the callback's signature accepts neither form. +Hooks that do **not** provide context will never pass it, even if you declare the parameter. + +### Context scope + +- All hooks in one **search** invocation share the same context object. It is built before + `search.before`, so replacing the request does not rebuild the context. Use the callback's + `request` argument for the current business input. +- **Reader extraction**, **cube writes**, and **scheduler operations** each create a separate + context. Do not assume they share an `operation_id`; use `trace_id` or `task_id` for + broader correlation. + +--- + +## Search pipeline + +The successful `SearchHandler.handle_search_memories` path is: + +```text +search.before + └─► retrieval + └─► search.memory_results + └─► threshold filtering ──► search.results.after_threshold + └─► deduplication ──► search.results.after_dedup + └─► reranking ──► search.results.after_rerank + └─► search.context.render + └─► SearchResponse construction + └─► search.after +``` + +Every hook receives `hook_context`. Context source: `api.search`; attributes contain `mode`. + +| Constant / Hook name | Additional arguments | `pipe_key` | Typical use | +| --- | --- | --- | --- | +| `H.SEARCH_BEFORE` / `search.before` | `request` | `request` | Rewrite the query or inject filters before retrieval. | +| `H.SEARCH_MEMORY_RESULTS` / `search.memory_results` | `handler`, `search_req`, `results` | `results` | Inspect or augment raw retrieval output. | +| `H.SEARCH_RESULTS_AFTER_THRESHOLD` / `search.results.after_threshold` | `handler`, `search_req`, `results` | `results` | Remove or re-score results after score filtering. | +| `H.SEARCH_RESULTS_AFTER_DEDUP` / `search.results.after_dedup` | `handler`, `search_req`, `results` | `results` | Post-dedup cleanup; runs even when no dedup mode is selected. | +| `H.SEARCH_RESULTS_AFTER_RERANK` / `search.results.after_rerank` | `handler`, `search_req`, `results` | `results` | Apply a custom final ranking. | +| `H.SEARCH_CONTEXT_RENDER` / `search.context.render` | `handler`, `search_req`, `results` | `results` | Last chance to modify the result dictionary before response construction. | +| `H.SEARCH_AFTER` / `search.after` | `request`, `result` | `result` | Post-process or log the final `SearchResponse`. | +| `H.SEARCH_POST_PROCESS_FAILED` / `search.post_process.failed` | `handler`, `search_req`, `error` | — | Observe an exception escaping threshold/dedup/reranking. | + +**Argument notes:** +- `request` at `search.before` and `search.after` is an `APISearchRequest`. The `search.after` + hook receives the request *after* any `search.before` replacement. +- Intermediate hooks receive `search_req` (the handler's working copy, which may include + internal adjustments such as expanded `top_k` for deduplication) and `results` (the + result-bucket dictionary). When returning a replacement, preserve its required buckets and + item structures. + +--- + +## MemReader extraction + +These hooks wrap `SimpleStructMemReader.get_memory`, including subclasses that inherit +that implementation. + +```text +get_memory() + └─► validation + input coercion + _read_memory() + ├─► [success] mem_reader.extract.after + └─► [failure] mem_reader.extract.failed ──► exception re-raised +``` + +Both hooks receive `hook_context`, `mem_reader`, `scene_data`, `type`, `info`, `mode`, +`user_name`, and `kwargs`. + +| Constant / Hook name | Additional argument | Timing | `pipe_key` | +| --- | --- | --- | --- | +| `H.MEM_READER_EXTRACT_AFTER` / `mem_reader.extract.after` | `result` | After validation, input coercion, and `_read_memory` return successfully. | `result` | +| `H.MEM_READER_EXTRACT_FAILED` / `mem_reader.extract.failed` | `error` | When validation, input coercion, or `_read_memory` raises an exception. | — | + +- `scene_data` is the raw input passed to `get_memory`, before coercion. +- `kwargs` is the dictionary of extra reader arguments. +- `result` has the shape `list[list[TextualMemoryItem]]`. +- The failure hook observes the original exception, which is then re-raised. + +Context: source `mem_reader.extract`; user/session IDs from `info`; cube IDs from +`user_name` when supplied; attributes contain `mode` and `type`. + +--- + +## Textual memory writes + +These hooks surround the `text_memory.add` call in `SingleCubeView._process_text_mem`. + +```text +_process_text_mem() + └─► text_memory.add(memories, user_name=...) + ├─► [success] text_memory.add.after + └─► [failure] text_memory.add.failed ──► exception re-raised +``` + +Both hooks receive `hook_context`, `text_memory`, `memories`, and `kwargs`. +- `memories` is the group submitted to the backend, excluding `RawFileMemory` items. +- `kwargs` contains the `user_name` for the write. + +| Constant / Hook name | Additional argument | Timing | `pipe_key` | +| --- | --- | --- | --- | +| `H.TEXT_MEMORY_ADD_AFTER` / `text_memory.add.after` | `result` | After the backend add call returns. | `result` | +| `H.TEXT_MEMORY_ADD_FAILED` / `text_memory.add.failed` | `error` | When the backend add call raises an exception. | — | + +- The success `result` is a list of stored memory IDs. A replacement is consumed by subsequent + processing, including scheduling and response construction — keep IDs consistent with the + stored memories. +- Backend exceptions are re-raised after observation. + +Context: source `cube_view.text_mem.add`; attributes contain `sync_mode` and `extract_mode`; +user, session, task, and cube metadata come from the add operation. + +--- + +## Scheduler memory operations + +`MemReadMessageHandler` observes selected operations through `SchedulerOperationObserver`. +Both hooks receive `hook_context`, `operation`, `target`, and `operation_input`. + +```text +SchedulerOperationObserver + └─► operation (fine | add | archive | delete | soft_delete) + ├─► [success] scheduler.memory.operation.after + └─► [failure] scheduler.memory.operation.failed +``` + +| Constant / Hook name | Additional argument | `pipe_key` | +| --- | --- | --- | +| `H.SCHEDULER_MEMORY_OPERATION_AFTER` / `scheduler.memory.operation.after` | `result` | `result` | +| `H.SCHEDULER_MEMORY_OPERATION_FAILED` / `scheduler.memory.operation.failed` | `error` | — | + +For the current MemRead integration, `target` is `textual_memory`. Context source has the +prefix `scheduler.mem_read.` followed by the operation suffix below. + +| `operation` | Source suffix | `operation_input` keys | Successful result | Notes | +| --- | --- | --- | --- | --- | +| `fine` | `fine_transfer_simple_mem` | `memories`, `type`, `result_grouping` | Nested reader output; replacements consumed before flattening and storage. | See [Fine-transfer result grouping](#fine-transfer-result-grouping). | +| `add` | `add_enhanced_memories` | `memories`, `user_name` | Stored IDs; replacements consumed by subsequent processing. | | +| `archive` | `archive_merged_memories` | `memory_ids` | Backend update return value; handler does not consume a replacement. | May emit one observation per merged memory. | +| `delete` | `remove_memories` | `memory_ids` | Backend delete return value; handler does not consume a replacement. | | +| `soft_delete` | `remove_memories` | `memory_ids`, `preserved_memory_ids` | Backend soft-delete return value; handler does not consume a replacement. | | + +Context attributes include available `scheduler_item_id`, `redis_message_id`, and `label`. + +The observer emits a success hook only after the operation block exits successfully — a +recorded `None` is still a successful result. Failure is reported at most once per observer. +These operations run only when the corresponding handler branch executes. + +### Fine-transfer result grouping + +Check `operation_input["result_grouping"]` on `fine` operations before matching output +groups to input memories: + +| Value | Interpretation | +| --- | --- | +| `per_input` | One group per input, in input order (built-in `SimpleStructMemReader`). A worker exception or `None` result leaves an empty group at that position. Empty input → `[]`. | +| `batch` | One group for the entire batch (built-in `MultiModalStructMemReader`). Not one-to-one with inputs — **do not zip**. Empty input → `[]`. | +| `unknown` | Implementation not recognized (e.g. a method override). Do not assume positional correspondence. | + +The label describes the reader's contract, not your plugin's replacement. Preserve the +original grouping in a replacement so downstream callbacks can interpret it consistently. + +Inheriting a recognized built-in method preserves its grouping label. An override receives +`unknown`, even if it belongs to a subclass of a built-in reader. diff --git a/src/memos/plugins/README.md b/src/memos/plugins/README.md index 3832bca1e..41904d293 100644 --- a/src/memos/plugins/README.md +++ b/src/memos/plugins/README.md @@ -27,6 +27,7 @@ src/memos/plugins/ ├── base.py # MemOSPlugin base class ├── manager.py # Plugin discovery, loading, and initialization ├── hooks.py # Hook registration and trigger runtime +├── hook_context.py # Operation correlation metadata ├── hook_defs.py # Core Hook declarations and constants └── component_bootstrap.py # Runtime component context bootstrap helpers ``` @@ -264,38 +265,79 @@ Core Hook names are defined in `memos.plugins.hook_defs.H`. Common extension points include: ```python +# Memory add H.ADD_BEFORE H.ADD_AFTER + +# Search pipeline H.SEARCH_BEFORE H.SEARCH_AFTER H.SEARCH_MEMORY_RESULTS -H.SEARCH_RESULTS_AFTER_THRESHOLD -H.SEARCH_RESULTS_AFTER_DEDUP -H.SEARCH_RESULTS_AFTER_RERANK +H.SEARCH_RESULTS_AFTER_THRESHOLD # after score threshold filtering +H.SEARCH_RESULTS_AFTER_DEDUP # after deduplication +H.SEARCH_RESULTS_AFTER_RERANK # after reranking H.SEARCH_CONTEXT_RENDER -H.SEARCH_POST_PROCESS_FAILED +H.SEARCH_POST_PROCESS_FAILED # threshold / dedup / rerank exception escaped + +# MemReader extraction H.MEM_READER_PRE_EXTRACT H.MEMORY_ITEMS_AFTER_FINE_EXTRACT H.MEM_READER_EXTRACT_AFTER + +# Textual memory writes H.TEXT_MEMORY_ADD_AFTER + +# Scheduler operations H.SCHEDULER_MEMORY_OPERATION_AFTER + +# Dream H.DREAM_EXECUTE ``` -Hook callbacks receive keyword arguments. Some Hooks define a `pipe_key`; when a -callback returns a non-`None` value, that value replaces the named argument for -later callbacks and for the caller. +Hook callbacks receive keyword arguments and run synchronously in registration +order. Some Hooks define a `pipe_key`; when a callback returns a non-`None` value, +that value replaces the named argument for later callbacks and for the caller. +Return `None` to leave the current value unchanged. -`hook_context` is additive correlation metadata. Context-aware callbacks can declare it explicitly or accept `**kwargs`. Legacy callbacks with an exact signature remain compatible: the runtime omits only an unsupported `hook_context` argument and still invokes the callback once. New callbacks should accept `**kwargs` so later additive metadata remains backward compatible. +Some Hooks carry a `hook_context` argument with operation correlation metadata +(trace, user, session, task, and cube IDs). Declare `hook_context` as a keyword +parameter to access it, or read it from `**kwargs`. The runtime silently omits +`hook_context` for callbacks that accept neither — existing callbacks with a +fixed signature stay compatible. Always include `**kwargs` in new callbacks so +future additive arguments do not require a signature change. -For example, `add.after` can replace `result`: +For example, a plugin can observe a search operation and cap its requested result +count through the `search.before` request pipeline: ```python -def on_add_after(*, request, result, **kwargs): - result.metadata["handled_by"] = "foo" - return result +from memos.log import get_logger +from memos.plugins import H, MemOSPlugin + + +logger = get_logger(__name__) + + +def on_search_before(*, hook_context, request, **kwargs): + logger.info("Search operation: %s", hook_context.operation_id) + return request.model_copy(update={"top_k": min(request.top_k, 20)}) + + +class SearchLimitPlugin(MemOSPlugin): + name = "search_limit" + + def on_load(self) -> None: + self.register_hook(H.SEARCH_BEFORE, on_search_before) ``` +Expose the plugin through an entry point as described in +[Registering a Plugin](#registering-a-plugin). The callback above changes the +request's `top_k`; subsequent pipeline stages and plugins still control the final +response. + +For trigger order, keyword parameters, replacement rules, failure boundaries, +`HookContext` fields, and so on, see the +[Pipeline Hook Reference](HOOKS.md). + ### Defining Plugin-Owned Hooks Plugins may define their own Hook names when they need internal extension From ad2ff52ba042f4edd29f454f731ed70b3c9b3173 Mon Sep 17 00:00:00 2001 From: qzs Date: Mon, 19 Oct 2026 17:05:33 +0800 Subject: [PATCH 14/22] refactor(hooks): simplify scheduler operation contracts - Normalize scheduler context to dictionaries and remove target parameter - Include reader and deletion arguments in operation inputs - Remove archive observations and redundant search context initialization - Update hook documentation and regression tests --- src/memos/api/handlers/search_handler.py | 3 - .../task_schedule_modules/base_handler.py | 4 +- .../handlers/mem_read_handler.py | 36 +++++------ .../operation_observer.py | 38 +++++------ src/memos/plugins/HOOKS.md | 63 +++---------------- src/memos/plugins/README.md | 3 + src/memos/plugins/hook_defs.py | 6 +- .../mem_scheduler/test_operation_observer.py | 54 +++++++++------- 8 files changed, 75 insertions(+), 132 deletions(-) diff --git a/src/memos/api/handlers/search_handler.py b/src/memos/api/handlers/search_handler.py index c95a89270..a94318a7e 100644 --- a/src/memos/api/handlers/search_handler.py +++ b/src/memos/api/handlers/search_handler.py @@ -98,8 +98,6 @@ def handle_search_memories( # Use deepcopy to avoid modifying the original request object search_req_local = copy.deepcopy(search_req) - if hook_context is None: - hook_context = self._build_search_hook_context(search_req_local) # Expand top_k for deduplication (5x to ensure enough candidates) if search_req_local.dedup in ("sim", "mmr"): @@ -108,7 +106,6 @@ def handle_search_memories( # Search and deduplicate cube_view = self._build_cube_view(search_req_local) results = cube_view.search_memories(search_req_local) - hooked_results = trigger_hook( H.SEARCH_MEMORY_RESULTS, hook_context=hook_context, diff --git a/src/memos/mem_scheduler/task_schedule_modules/base_handler.py b/src/memos/mem_scheduler/task_schedule_modules/base_handler.py index 438c30a0b..67c39aa0f 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/base_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/base_handler.py @@ -42,8 +42,7 @@ def observe_operation( operation_name: str, *, operation: str, - target: str, - operation_context: Any, + operation_context: dict[str, Any], operation_input: dict[str, Any], capture_failure: bool = False, ) -> SchedulerOperationObserver: @@ -52,7 +51,6 @@ def observe_operation( return SchedulerOperationObserver( source=source, operation=operation, - target=target, operation_context=operation_context, operation_input=operation_input, capture_failure=capture_failure, diff --git a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py index f203c2346..743627d19 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py +++ b/src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py @@ -96,7 +96,7 @@ def process_message(self, message: ScheduleMessageItem): info=info, chat_history=chat_history, user_context=user_context, - operation_context=message, + operation_context=message.model_dump(), ) logger.info( @@ -120,7 +120,7 @@ def _process_memories_with_reader( info: dict | None = None, chat_history: list | None = None, user_context: UserContext | None = None, - operation_context: Any | None = None, + operation_context: dict | None = None, ) -> None: logger.info( "[DIAGNOSTIC] mem_read_handler._process_memories_with_reader called. mem_ids: %s, user_id: %s, mem_cube_id: %s, task_id: %s", @@ -180,11 +180,15 @@ def _process_memories_with_reader( with self.observe_operation( "fine_transfer_simple_mem", operation="fine", - target="textual_memory", operation_context=operation_context, operation_input={ "memories": memory_items, "type": "chat", + "custom_tags": custom_tags, + "user_name": user_name, + "chat_history": chat_history, + "user_context": user_context, + "is_upload_skill": is_upload_skill, "result_grouping": self._fine_transfer_result_grouping(mem_reader), }, capture_failure=True, @@ -221,7 +225,6 @@ def _process_memories_with_reader( with self.observe_operation( "add_enhanced_memories", operation="add", - target="textual_memory", operation_context=operation_context, operation_input={"memories": mem_group, "user_name": user_name}, capture_failure=True, @@ -276,21 +279,11 @@ def _process_memories_with_reader( ) for old_id in old_ids: try: - with self.observe_operation( - "archive_merged_memories", - operation="archive", - target="textual_memory", - operation_context=operation_context, - operation_input={"memory_ids": [str(old_id)]}, - capture_failure=True, - ) as observation: - observation.result( - mem_reader.graph_db.update_node( - str(old_id), - {"status": "archived"}, - user_name=user_name, - ) - ) + mem_reader.graph_db.update_node( + str(old_id), + {"status": "archived"}, + user_name=user_name, + ) logger.info( "[Scheduler] Archived merged_from memory: %s", old_id, @@ -450,9 +443,8 @@ def _process_memories_with_reader( with self.observe_operation( "remove_memories", operation="delete", - target="textual_memory", operation_context=operation_context, - operation_input={"memory_ids": delete_ids}, + operation_input={"memory_ids": delete_ids, "user_name": user_name}, capture_failure=True, ) as observation: observation.result(text_mem.delete(delete_ids, user_name=user_name)) @@ -476,10 +468,10 @@ def _process_memories_with_reader( with self.observe_operation( "remove_memories", operation="soft_delete", - target="textual_memory", operation_context=operation_context, operation_input={ "memory_ids": delete_ids, + "user_name": user_name, "preserved_memory_ids": preserved_memory_ids, }, capture_failure=True, diff --git a/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py b/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py index 7a151d4ab..89a737663 100644 --- a/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py +++ b/src/memos/mem_scheduler/task_schedule_modules/operation_observer.py @@ -1,14 +1,17 @@ from __future__ import annotations -from collections.abc import Mapping from contextlib import AbstractContextManager -from typing import Any +from typing import TYPE_CHECKING, Any from memos.plugins.hook_context import build_hook_context from memos.plugins.hook_defs import H from memos.plugins.hooks import trigger_hook +if TYPE_CHECKING: + from collections.abc import Mapping + + _MISSING = object() @@ -26,24 +29,20 @@ def __init__( *, source: str, operation: str, - target: str, - operation_context: Any, + operation_context: dict[str, Any], operation_input: Mapping[str, Any], capture_failure: bool = False, ) -> None: self._hook_kwargs = { "operation": operation, - "target": target, "operation_input": dict(operation_input), } - cube_id = self._context_value(operation_context, "cube_id") or self._context_value( - operation_context, "mem_cube_id" - ) + cube_id = operation_context.get("cube_id") or operation_context.get("mem_cube_id") self._hook_context = build_hook_context( - trace_id=self._context_value(operation_context, "trace_id"), - user_id=self._context_value(operation_context, "user_id"), - session_id=self._context_value(operation_context, "session_id"), - task_id=self._context_value(operation_context, "task_id"), + trace_id=operation_context.get("trace_id"), + user_id=operation_context.get("user_id"), + session_id=operation_context.get("session_id"), + task_id=operation_context.get("task_id"), cube_ids=[str(cube_id)] if cube_id else None, source=source, attributes=self._attributes(operation_context), @@ -95,17 +94,10 @@ def failed(self, error: BaseException) -> None: ) @staticmethod - def _context_value(operation_context: Any, name: str) -> Any: - if isinstance(operation_context, Mapping): - return operation_context.get(name) - return getattr(operation_context, name, None) - - @classmethod - def _attributes(cls, operation_context: Any) -> dict[str, Any]: + def _attributes(operation_context: dict[str, Any]) -> dict[str, Any]: attributes = { - "scheduler_item_id": cls._context_value(operation_context, "scheduler_item_id") - or cls._context_value(operation_context, "item_id"), - "redis_message_id": cls._context_value(operation_context, "redis_message_id"), - "label": cls._context_value(operation_context, "label"), + "scheduler_item_id": operation_context.get("scheduler_item_id") + or operation_context.get("item_id"), + "label": operation_context.get("label"), } return {key: value for key, value in attributes.items() if value is not None} diff --git a/src/memos/plugins/HOOKS.md b/src/memos/plugins/HOOKS.md index c28a63b9b..6a3276507 100644 --- a/src/memos/plugins/HOOKS.md +++ b/src/memos/plugins/HOOKS.md @@ -11,7 +11,7 @@ The runtime lives in [hooks.py](hooks.py). ## How hooks work -Every hook falls into one of two patterns: +Every hook falls into one of three patterns: | Pattern | Return value matters? | API | Exception handling | | --- | --- | --- | --- | @@ -19,32 +19,6 @@ Every hook falls into one of two patterns: | **Observer hook** — no `pipe_key` | No | `trigger_hook` | Logged and swallowed | | **Single-provider hook** | Returned directly to caller | `trigger_single_hook` | Re-raised | -Memory-version and Dream business hooks use the single-provider API; their declarations -are in [hook_defs.py](hook_defs.py). All hooks documented below use `trigger_hook`. - -### Quick example - -```python -from memos.plugins import H - -class MyPlugin(BasePlugin): - def on_enable(self): - # Observe results without changing them - self.register_hook(H.SEARCH_AFTER, self._on_search_done) - - # Replace results mid-pipeline - self.register_hook(H.SEARCH_RESULTS_AFTER_RERANK, self._rerank) - - def _on_search_done(self, request, result, **kwargs): - print(f"Search for '{request.query}' returned {len(result.memories)} results") - - def _rerank(self, results, **kwargs): - # pipe_key=results — return non-None to propagate the change - for bucket in results.values(): - bucket.sort(key=lambda m: my_score(m), reverse=True) - return results -``` - --- ## Registering callbacks @@ -71,25 +45,6 @@ downstream code expects. --- -## Observer callbacks (no `pipe_key`) - -Return values are ignored. These hooks are for side effects only — logging, metrics, -notifications, etc. The same live-object caveat applies: mutating an argument in-place -is visible downstream even though the callback's return is discarded. - ---- - -## Exception handling - -`trigger_hook` logs any `Exception` raised by a callback and continues to subsequent -callbacks. Mutations made before the exception are **not** rolled back. A crashing observer -does not, by itself, fail the business operation. - -The single-provider API (`trigger_single_hook`) propagates exceptions — see each hook's -documentation for which API it uses. - ---- - ## HookContext Context-aware hooks receive `hook_context` alongside their business arguments. @@ -241,11 +196,11 @@ user, session, task, and cube metadata come from the add operation. ## Scheduler memory operations `MemReadMessageHandler` observes selected operations through `SchedulerOperationObserver`. -Both hooks receive `hook_context`, `operation`, `target`, and `operation_input`. +Both hooks receive `hook_context`, `operation`, and `operation_input`. ```text SchedulerOperationObserver - └─► operation (fine | add | archive | delete | soft_delete) + └─► operation (fine | add | delete | soft_delete) ├─► [success] scheduler.memory.operation.after └─► [failure] scheduler.memory.operation.failed ``` @@ -255,18 +210,14 @@ SchedulerOperationObserver | `H.SCHEDULER_MEMORY_OPERATION_AFTER` / `scheduler.memory.operation.after` | `result` | `result` | | `H.SCHEDULER_MEMORY_OPERATION_FAILED` / `scheduler.memory.operation.failed` | `error` | — | -For the current MemRead integration, `target` is `textual_memory`. Context source has the -prefix `scheduler.mem_read.` followed by the operation suffix below. +Context source has the prefix `scheduler.mem_read.` followed by the operation suffix below. | `operation` | Source suffix | `operation_input` keys | Successful result | Notes | | --- | --- | --- | --- | --- | -| `fine` | `fine_transfer_simple_mem` | `memories`, `type`, `result_grouping` | Nested reader output; replacements consumed before flattening and storage. | See [Fine-transfer result grouping](#fine-transfer-result-grouping). | +| `fine` | `fine_transfer_simple_mem` | `memories`, `type`, `custom_tags`, `user_name`, `chat_history`, `user_context`, `is_upload_skill`, `result_grouping` | Nested reader output; replacements consumed before flattening and storage. | See [Fine-transfer result grouping](#fine-transfer-result-grouping). | | `add` | `add_enhanced_memories` | `memories`, `user_name` | Stored IDs; replacements consumed by subsequent processing. | | -| `archive` | `archive_merged_memories` | `memory_ids` | Backend update return value; handler does not consume a replacement. | May emit one observation per merged memory. | -| `delete` | `remove_memories` | `memory_ids` | Backend delete return value; handler does not consume a replacement. | | -| `soft_delete` | `remove_memories` | `memory_ids`, `preserved_memory_ids` | Backend soft-delete return value; handler does not consume a replacement. | | - -Context attributes include available `scheduler_item_id`, `redis_message_id`, and `label`. +| `delete` | `remove_memories` | `memory_ids`, `user_name` | Backend delete return value; handler does not consume a replacement. | | +| `soft_delete` | `remove_memories` | `memory_ids`, `user_name`, `preserved_memory_ids` | Backend soft-delete return value; handler does not consume a replacement. | | The observer emits a success hook only after the operation block exits successfully — a recorded `None` is still a successful result. Failure is reported at most once per observer. diff --git a/src/memos/plugins/README.md b/src/memos/plugins/README.md index 41904d293..aecb11890 100644 --- a/src/memos/plugins/README.md +++ b/src/memos/plugins/README.md @@ -283,12 +283,15 @@ H.SEARCH_POST_PROCESS_FAILED # threshold / dedup / rerank exception escape H.MEM_READER_PRE_EXTRACT H.MEMORY_ITEMS_AFTER_FINE_EXTRACT H.MEM_READER_EXTRACT_AFTER +H.MEM_READER_EXTRACT_FAILED # Textual memory writes H.TEXT_MEMORY_ADD_AFTER +H.TEXT_MEMORY_ADD_FAILED # Scheduler operations H.SCHEDULER_MEMORY_OPERATION_AFTER +H.SCHEDULER_MEMORY_OPERATION_FAILED # Dream H.DREAM_EXECUTE diff --git a/src/memos/plugins/hook_defs.py b/src/memos/plugins/hook_defs.py index 95cc493f6..90b6052a2 100644 --- a/src/memos/plugins/hook_defs.py +++ b/src/memos/plugins/hook_defs.py @@ -103,7 +103,7 @@ class H: SCHEDULER_MEMORY_OPERATION_AFTER = "scheduler.memory.operation.after" SCHEDULER_MEMORY_OPERATION_FAILED = "scheduler.memory.operation.failed" - # mem_reader — generic extension point before LLM extraction + # mem_reader — memory extraction MEM_READER_EXTRACT_AFTER = "mem_reader.extract.after" MEM_READER_EXTRACT_FAILED = "mem_reader.extract.failed" @@ -255,14 +255,14 @@ class H: define_hook( H.SCHEDULER_MEMORY_OPERATION_AFTER, description="Observe or replace a successful Scheduler memory operation result", - params=["hook_context", "operation", "target", "operation_input", "result"], + params=["hook_context", "operation", "operation_input", "result"], pipe_key="result", ) define_hook( H.SCHEDULER_MEMORY_OPERATION_FAILED, description="Observe a failed Scheduler memory operation and its error", - params=["hook_context", "operation", "target", "operation_input", "error"], + params=["hook_context", "operation", "operation_input", "error"], ) define_hook( diff --git a/tests/mem_scheduler/test_operation_observer.py b/tests/mem_scheduler/test_operation_observer.py index 6b4beaa5e..52c24a0a1 100644 --- a/tests/mem_scheduler/test_operation_observer.py +++ b/tests/mem_scheduler/test_operation_observer.py @@ -12,7 +12,8 @@ SchedulerOperationObserver, ) from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata -from memos.plugins.hook_defs import H +from memos.memories.textual.tree import TreeTextMemory +from memos.plugins.hook_defs import H, get_hook_spec from memos.plugins.hooks import register_hook @@ -24,7 +25,6 @@ OPERATION_SOURCE = { "fine": "fine_transfer_simple_mem", "add": "add_enhanced_memories", - "archive": "archive_merged_memories", "delete": "remove_memories", "soft_delete": "remove_memories", } @@ -58,12 +58,14 @@ def _mem_read_dependencies(memory_version_switch): reader.memory_version_switch = memory_version_switch text_memory = MagicMock() + text_memory.__class__ = TreeTextMemory text_memory.get.return_value = raw_memory text_memory.add.return_value = [ENHANCED_MEMORY_ID] text_memory.memory_manager.reorganizer.is_reorganize = False scheduler_context = MagicMock() scheduler_context.get_mem_reader.return_value = reader + scheduler_context.get_mem_cube.return_value.text_mem = text_memory scheduler_context.services.create_event_log.return_value = MagicMock() message = ScheduleMessageItem( @@ -104,7 +106,7 @@ def replace_result(*, operation, result, **_kwargs): mem_cube_id="cube-1", text_mem=text_memory, user_name="cube-1", - operation_context=message, + operation_context=message.model_dump(), ) text_memory.add.assert_called_once_with([replacement], user_name="cube-1") @@ -124,7 +126,6 @@ def replace_result(**kwargs): with SchedulerOperationObserver( source="scheduler.mem_read.add_enhanced_memories", operation="add", - target="textual_memory", operation_context={"trace_id": "trace-1", "task_id": "task-1"}, operation_input={"memory_ids": ["memory-1"]}, ) as observer: @@ -132,7 +133,7 @@ def replace_result(**kwargs): assert observer.value == ["plugin-result"] assert captured["operation"] == "add" - assert captured["target"] == "textual_memory" + assert set(captured) == set(get_hook_spec(H.SCHEDULER_MEMORY_OPERATION_AFTER).params) assert captured["operation_input"] == {"memory_ids": ["memory-1"]} assert captured["result"] == ["stored-1"] assert captured["hook_context"].source == "scheduler.mem_read.add_enhanced_memories" @@ -152,7 +153,6 @@ def test_scheduler_observer_reports_failure_and_preserves_error(): observer = SchedulerOperationObserver( source="scheduler.mem_read.fine_transfer_simple_mem", operation="fine", - target="textual_memory", operation_context={"trace_id": "trace-1", "mem_cube_id": "cube-1"}, operation_input={"memory_ids": ["memory-1"]}, capture_failure=True, @@ -165,22 +165,29 @@ def test_scheduler_observer_reports_failure_and_preserves_error(): assert [stage for stage, _kwargs in events] == ["failed"] failure = events[0][1] assert failure["error"] is error + assert set(failure) == set(get_hook_spec(H.SCHEDULER_MEMORY_OPERATION_FAILED).params) assert failure["hook_context"].cube_ids == ("cube-1",) @pytest.mark.parametrize( ("memory_version_switch", "expected_operations"), [ - ("off", ["fine", "add", "archive", "delete"]), + ("off", ["fine", "add", "delete"]), ("on", ["fine", "add", "soft_delete"]), ], ) +@pytest.mark.parametrize("task_id", ["task-1", None]) def test_mem_read_handler_exposes_each_memory_operation( memory_version_switch, expected_operations, + task_id, monkeypatch, ): handler, reader, text_memory, message = _mem_read_dependencies(memory_version_switch) + message.task_id = task_id + message.redis_message_id = "synthetic-redis-id" + message.info = {"custom_tags": ["synthetic-tag"], "is_upload_skill": True} + message.chat_history = [{"role": "user", "content": "synthetic history"}] events = [] monkeypatch.setattr( "memos.mem_scheduler.task_schedule_modules.handlers.mem_read_handler.is_playground_api", @@ -193,15 +200,7 @@ def record_operation(**kwargs): register_hook(H.SCHEDULER_MEMORY_OPERATION_AFTER, record_operation) - handler._process_memories_with_reader( - mem_ids=[RAW_MEMORY_ID], - user_id="alice", - mem_cube_id="cube-1", - text_mem=text_memory, - user_name="cube-1", - task_id="task-1", - operation_context=message, - ) + handler.process_message(message) assert [event["operation"] for event in events] == expected_operations assert [event["hook_context"].source for event in events] == [ @@ -209,24 +208,34 @@ def record_operation(**kwargs): ] for event in events: context = event["hook_context"] - assert event["target"] == "textual_memory" assert context.trace_id == "trace-1" assert context.user_id == "alice" assert context.session_id == "session-1" - assert context.task_id == "task-1" + assert context.task_id == task_id assert context.cube_ids == ("cube-1",) - assert context.attributes["scheduler_item_id"] == SCHEDULER_ITEM_ID - assert context.attributes["label"] == "mem_read" + assert context.attributes == {"scheduler_item_id": SCHEDULER_ITEM_ID, "label": "mem_read"} inputs = {event["operation"]: event["operation_input"] for event in events} assert inputs["fine"]["memories"][0].id == RAW_MEMORY_ID assert inputs["fine"]["type"] == "chat" + fine_input = inputs["fine"] + reader.fine_transfer_simple_mem.assert_called_once_with( + fine_input["memories"], + type=fine_input["type"], + custom_tags=fine_input["custom_tags"], + user_name=fine_input["user_name"], + chat_history=fine_input["chat_history"], + user_context=fine_input["user_context"], + is_upload_skill=fine_input["is_upload_skill"], + ) + assert fine_input["custom_tags"] == ["synthetic-tag"] + assert fine_input["chat_history"] == message.chat_history + assert fine_input["is_upload_skill"] is True assert inputs["add"]["memories"][0].id == ENHANCED_MEMORY_ID assert inputs["add"]["user_name"] == "cube-1" if memory_version_switch == "off": - assert inputs["archive"] == {"memory_ids": [RAW_MEMORY_ID]} - assert inputs["delete"] == {"memory_ids": [RAW_MEMORY_ID]} + assert inputs["delete"] == {"memory_ids": [RAW_MEMORY_ID], "user_name": "cube-1"} reader.graph_db.update_node.assert_called_once_with( RAW_MEMORY_ID, {"status": "archived"}, @@ -237,6 +246,7 @@ def record_operation(**kwargs): else: assert inputs["soft_delete"] == { "memory_ids": [RAW_MEMORY_ID], + "user_name": "cube-1", "preserved_memory_ids": [ENHANCED_MEMORY_ID], } reader.graph_db.update_node.assert_not_called() From 30b0ec9493f3bdb77b929292494c665730f5514f Mon Sep 17 00:00:00 2001 From: qzs Date: Wed, 21 Oct 2026 14:47:53 +0800 Subject: [PATCH 15/22] refactor(hooks): simplify search hook context --- src/memos/api/handlers/search_handler.py | 16 ++---- src/memos/plugins/HOOKS.md | 9 +-- src/memos/plugins/hooks.py | 9 ++- tests/api/test_search_hook_context.py | 70 ++++++++++++++++++++++-- tests/plugins/test_hook_context.py | 23 ++++++++ 5 files changed, 102 insertions(+), 25 deletions(-) diff --git a/src/memos/api/handlers/search_handler.py b/src/memos/api/handlers/search_handler.py index a94318a7e..392c2b601 100644 --- a/src/memos/api/handlers/search_handler.py +++ b/src/memos/api/handlers/search_handler.py @@ -74,8 +74,6 @@ def __init__(self, dependencies: HandlerDependencies): def handle_search_memories( self, search_req: APISearchRequest, - *, - hook_context: HookContext | None = None, ) -> SearchResponse: """ Main handler for search memories endpoint. @@ -85,8 +83,6 @@ def handle_search_memories( Args: search_req: Search request containing query and parameters - hook_context: Internal correlation context shared by all Hooks for - this search operation. A context is created when omitted. Returns: SearchResponse with formatted results @@ -108,7 +104,7 @@ def handle_search_memories( results = cube_view.search_memories(search_req_local) hooked_results = trigger_hook( H.SEARCH_MEMORY_RESULTS, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -122,7 +118,7 @@ def handle_search_memories( results = self._apply_relativity_threshold(results, search_req_local.relativity) hooked_results = trigger_hook( H.SEARCH_RESULTS_AFTER_THRESHOLD, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -139,7 +135,7 @@ def handle_search_memories( self._strip_embeddings(results) hooked_results = trigger_hook( H.SEARCH_RESULTS_AFTER_DEDUP, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -157,7 +153,7 @@ def handle_search_memories( ) hooked_results = trigger_hook( H.SEARCH_RESULTS_AFTER_RERANK, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -167,7 +163,7 @@ def handle_search_memories( except Exception as error: trigger_hook( H.SEARCH_POST_PROCESS_FAILED, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, error=error, @@ -176,7 +172,7 @@ def handle_search_memories( hooked_results = trigger_hook( H.SEARCH_CONTEXT_RENDER, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, diff --git a/src/memos/plugins/HOOKS.md b/src/memos/plugins/HOOKS.md index 6a3276507..e2895e14d 100644 --- a/src/memos/plugins/HOOKS.md +++ b/src/memos/plugins/HOOKS.md @@ -85,11 +85,10 @@ Hooks that do **not** provide context will never pass it, even if you declare th ### Context scope -- All hooks in one **search** invocation share the same context object. It is built before - `search.before`, so replacing the request does not rebuild the context. Use the callback's - `request` argument for the current business input. +- **Search** builds a context for `search.before` and `search.after`. Each intermediate + hook creates its own context. - **Reader extraction**, **cube writes**, and **scheduler operations** each create a separate - context. Do not assume they share an `operation_id`; use `trace_id` or `task_id` for + context. Use `trace_id` or `task_id` for broader correlation. --- @@ -110,8 +109,6 @@ search.before └─► search.after ``` -Every hook receives `hook_context`. Context source: `api.search`; attributes contain `mode`. - | Constant / Hook name | Additional arguments | `pipe_key` | Typical use | | --- | --- | --- | --- | | `H.SEARCH_BEFORE` / `search.before` | `request` | `request` | Rewrite the query or inject filters before retrieval. | diff --git a/src/memos/plugins/hooks.py b/src/memos/plugins/hooks.py index 084f0b66b..f3dc6a67d 100644 --- a/src/memos/plugins/hooks.py +++ b/src/memos/plugins/hooks.py @@ -148,7 +148,8 @@ def hookable( Auto-declares before/after Hooks (idempotent); no need to manually define_hook in hook_defs.py. Supports piped return values: before can modify request, after can modify result. When ``context_builder`` is provided, one context is built before the first - Hook and passed to both Hooks and the decorated method as ``hook_context``. + Hook and passed to both Hooks as ``hook_context``. It is also passed to the + decorated method when that method accepts it. Compatible with both sync and async methods. """ from memos.plugins.hook_defs import define_hook @@ -184,7 +185,8 @@ async def async_wrapper(self, request, *args, **kwargs): call_kwargs = dict(kwargs) if context_builder is not None: before_kwargs["hook_context"] = hook_context - call_kwargs["hook_context"] = hook_context + if _accepts_hook_context(func): + call_kwargs["hook_context"] = hook_context rv = trigger_hook(f"{name}.before", **before_kwargs) request = rv if rv is not None else request @@ -207,7 +209,8 @@ def sync_wrapper(self, request, *args, **kwargs): call_kwargs = dict(kwargs) if context_builder is not None: before_kwargs["hook_context"] = hook_context - call_kwargs["hook_context"] = hook_context + if _accepts_hook_context(func): + call_kwargs["hook_context"] = hook_context rv = trigger_hook(f"{name}.before", **before_kwargs) request = rv if rv is not None else request diff --git a/tests/api/test_search_hook_context.py b/tests/api/test_search_hook_context.py index a5d39b3e8..3a7df0698 100644 --- a/tests/api/test_search_hook_context.py +++ b/tests/api/test_search_hook_context.py @@ -1,3 +1,5 @@ +import inspect + import pytest from memos.api.handlers.base_handler import HandlerDependencies @@ -47,7 +49,7 @@ def _search_handler() -> tuple[SearchHandler, _CubeView]: return handler, cube_view -def test_search_success_hooks_run_in_order_and_share_context(monkeypatch): +def test_search_success_hooks_run_in_order_with_distinct_stage_operations(monkeypatch): events = [] result_hook_calls = [] handler, cube_view = _search_handler() @@ -97,7 +99,8 @@ def replace_response(*, hook_context, result, **_kwargs): H.SEARCH_AFTER, ] contexts = [context for _name, context in events] - assert all(context is contexts[0] for context in contexts) + assert contexts[-1] is contexts[0] + assert len({context.operation_id for context in contexts}) == 6 context = contexts[0] assert isinstance(context, HookContext) assert context.trace_id == "trace-1" @@ -107,6 +110,13 @@ def replace_response(*, hook_context, result, **_kwargs): assert context.source == "api.search" assert context.attributes == {"mode": "fast"} assert context.operation_id + for stage_context in contexts[1:-1]: + assert stage_context.source == "api.search" + assert stage_context.operation_id != context.operation_id + assert stage_context.attributes == {"mode": "fast"} + assert stage_context.attributes is not context.attributes + for field in ("trace_id", "user_id", "session_id", "task_id", "cube_ids"): + assert getattr(stage_context, field) == getattr(context, field) assert cube_view.request.query == "hooked query" assert all(call["handler"] is handler for call in result_hook_calls) assert all(call["search_req"] is cube_view.request for call in result_hook_calls) @@ -143,23 +153,42 @@ def replace_memories(*, results, **_kwargs): assert response.data["text_mem"][0]["memories"] == [memory] -def test_search_post_processing_failure_triggers_only_failure_hook(monkeypatch): +@pytest.mark.parametrize("failed_stage", ["threshold", "dedup", "rerank"]) +def test_search_post_processing_failure_identifies_the_failed_operation(monkeypatch, failed_stage): error = RuntimeError("post processing failed") events = [] handler, _cube_view = _search_handler() - search_req = APISearchRequest(user_id="user", query="query") + search_req = APISearchRequest(user_id="user", query="query", dedup="sim") + observed_contexts = [] + register_hook( + H.SEARCH_BEFORE, lambda **kwargs: observed_contexts.append(kwargs["hook_context"]) + ) + for name in ( + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + ): + register_hook(name, lambda **kwargs: observed_contexts.append(kwargs["hook_context"])) monkeypatch.setattr( "memos.api.handlers.search_handler.get_current_trace_id", lambda: "trace-1", ) - def fail_post_processing(_results, _relativity): + def fail_post_processing(*_args, **_kwargs): raise error def record_failure(**kwargs): events.append((H.SEARCH_POST_PROCESS_FAILED, kwargs)) - handler._apply_relativity_threshold = fail_post_processing + if failed_stage == "threshold": + handler._apply_relativity_threshold = fail_post_processing + elif failed_stage == "dedup": + handler._dedup_text_memories = fail_post_processing + else: + monkeypatch.setattr( + "memos.api.handlers.search_handler.rerank_knowledge_mem", fail_post_processing + ) register_hook(H.SEARCH_POST_PROCESS_FAILED, record_failure) register_hook(H.SEARCH_CONTEXT_RENDER, lambda **_kwargs: events.append(("render", {}))) register_hook(H.SEARCH_AFTER, lambda **_kwargs: events.append(("after", {}))) @@ -176,3 +205,32 @@ def record_failure(**kwargs): context = failure["hook_context"] assert context.trace_id == "trace-1" assert context.user_id == "user" + assert context.source == "api.search" + assert context.attributes == {"mode": "fast"} + assert context.operation_id not in {prior.operation_id for prior in observed_contexts} + + +def test_search_handler_does_not_accept_hook_context(): + assert list(inspect.signature(SearchHandler.handle_search_memories).parameters) == [ + "self", + "search_req", + ] + + +def test_search_stage_operations_stay_isolated_between_searches_in_one_trace(monkeypatch): + handler, _cube_view = _search_handler() + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", lambda: "one-trace" + ) + contexts = [] + for name in ( + H.SEARCH_BEFORE, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + ): + register_hook(name, lambda **kwargs: contexts.append(kwargs["hook_context"])) + for query in ("first", "second"): + handler.handle_search_memories(APISearchRequest(user_id="user", query=query)) + assert len({context.operation_id for context in contexts}) == 8 + assert all(context.attributes == {"mode": "fast"} for context in contexts) diff --git a/tests/plugins/test_hook_context.py b/tests/plugins/test_hook_context.py index ad06e2f98..453c80fd2 100644 --- a/tests/plugins/test_hook_context.py +++ b/tests/plugins/test_hook_context.py @@ -186,3 +186,26 @@ def callback(*, hook_context, **_kwargs): ("operation", supplied_context), ("after", supplied_context), ] + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_hookable_supports_method_without_context_parameter(is_async): + context = build_hook_context(source="test") + observed = [] + + async def async_run(self, request): + return request + + def sync_run(self, request): + return request + + class Handler: + run = hookable("without_context", context_builder=lambda _self, _request: context)( + async_run if is_async else sync_run + ) + + for name in ("without_context.before", "without_context.after"): + register_hook(name, lambda **kwargs: observed.append(kwargs["hook_context"])) + result = Handler().run("request") + assert (asyncio.run(result) if is_async else result) == "request" + assert observed == [context, context] From 00053753f32f10ba1d35a5ed8df745878291e337 Mon Sep 17 00:00:00 2001 From: qzs Date: Wed, 21 Oct 2026 15:14:28 +0800 Subject: [PATCH 16/22] fix: Adjusted the link direction and comments when merging search and add, removed the node for nodes filtered out during the search phase, and aligned some handler changes with the corresponding hook changes. --- apps/memtrace-memos-integration/.gitignore | 3 +- apps/memtrace-memos-integration/FEATURES.md | 242 ++++++++++++++++++ apps/memtrace-memos-integration/README.md | 68 +++-- .../memos_smartcomment/CONVENTIONS.md | 78 +++--- .../handlers/add_handler.py | 20 +- .../handlers/search_handler.py | 122 +++------ .../memos_smartcomment/lineage.py | 48 ++-- src/memos/api/handlers/search_handler.py | 16 +- src/memos/plugins/HOOKS.md | 9 +- src/memos/plugins/hooks.py | 9 +- tests/api/test_search_hook_context.py | 70 ++++- tests/plugins/test_hook_context.py | 23 ++ 12 files changed, 491 insertions(+), 217 deletions(-) create mode 100644 apps/memtrace-memos-integration/FEATURES.md diff --git a/apps/memtrace-memos-integration/.gitignore b/apps/memtrace-memos-integration/.gitignore index 131eab0ee..2739346ef 100644 --- a/apps/memtrace-memos-integration/.gitignore +++ b/apps/memtrace-memos-integration/.gitignore @@ -16,11 +16,10 @@ __pycache__/ *.py[cod] # Tests and tool output -tests/* +tests test_*.py *_test.py conftest.py -!tests/test_search_lineage.py .pytest_cache/ .ruff_cache/ .tests/ diff --git a/apps/memtrace-memos-integration/FEATURES.md b/apps/memtrace-memos-integration/FEATURES.md new file mode 100644 index 000000000..450ee9d4d --- /dev/null +++ b/apps/memtrace-memos-integration/FEATURES.md @@ -0,0 +1,242 @@ +# MemOS × MemTrace 集成:产品功能说明 + +## 1. 概览 + +| 产品 | 一句话定位 | 归属 | +| --- | --- | --- | +| **本插件**(`memos_smartcomment`) | 面向开发者的调试工具。把每次记忆写入和检索都记录成一张执行图,记录本身不影响业务结果。 | MemOS 可选插件,默认关闭,按需开启 | +| **MemTrace** | 读取执行图,沿着数据依赖往回查,找出导致答案出错的那一步操作,并给出诊断报告。 | 浙大 ZJUNLP 开源项目,[独立仓库](https://github.com/zjunlp/MemTrace),不随本插件提供 | +| **smartcomment** | 两者之间的「通用语言」,定义执行图的数据格式,负责导出 JSON 和可视化。 | 公开 PyPI 包,本插件的依赖 | + +简单说,本插件负责记录,MemTrace 负责分析,smartcomment 是两者共用的数据格式。 + +### 1.1 定位:开发者内部迭代的调试工具 + +本插件服务于开发者的开发与调试。开发者在本地或测试环境中通过环境变量按需开启,用来排查 bad case、分析评测失败、验证改动效果。调试结束后关闭即可。使用 MemOS 的用户对它完全无感,生产部署也无需安装或开启。 + +| 关注点 | 说明 | +| --- | --- | +| 面向用户 | 开发者和算法 / 评测同学,用于内部迭代时定位问题。 | +| 使用场景 | 复现和排查 bad case、跑基准评测并做错误归因、验证提取 / 增强 / 检索策略的改动是否生效。 | +| 如何开启 | 启动服务前设置 `MEMOS_ENABLED_PLUGINS=smartcomment`。需要检索溯源时再加 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true`。不设置时插件不会加载,去掉变量后重启即关闭。 | +| 对用户的影响 | 无感。插件只旁路订阅 Hook,不改变任何请求、响应和业务结果,API 行为与未开启时完全一致。 | +| 生产部署 | 无需安装,也无需开启。线上服务保持默认配置即可,不会产生任何执行图文件或额外开销。 | + +## 2. 要解决的问题 + +MemOS 回答错了,或者「应该记住的事没记住」,原因往往不在最后一步,而是藏在中间某个环节: + +- **写入时**:MemReader 没有从对话里提取出这条事实; +- **后台增强时**:MemRead 调度器把记忆合并、改写或删除了; +- **检索时**:记忆其实还在,但被相关度阈值、去重或重排过滤掉了。 + +MemOS 的日志只能看到零散的事件,看不出「这条记忆从哪条消息来、经过了哪些步骤、在哪一步消失了」。本插件和 MemTrace 合在一起,目的是在开发调试阶段帮助开发者把这条链路看清楚。 + +## 3. 整体关系 + +```text + ┌──────────────────── 本插件负责 ────────────────────┐ ┌──────── MemTrace 负责 ────────┐ + │ │ │ │ + ┌───────────┐ Hook │ ┌──────────────┐ ┌──────────────┐ ┌──────────┐ │ │ ┌───────────┐ ┌───────────┐ │ + │ MemOS │ ───▶ │ │ 观测与快照 │ ─▶│ 建图与关联 │ ─▶│ 落盘与索引 │ │ ─▶│ │ 回溯探查 │ ─▶│ 归因与报告 │ │ + │ Add/Search│ │ └──────────────┘ └──────────────┘ └──────────┘ │ │ └───────────┘ └───────────┘ │ + └───────────┘ │ │ │ │ + └──────────────────────────┬─────────────────────────┘ └───────────────▲───────────────┘ + │ graph.json(smartcomment 执行图格式) │ + └────────────────────────────────────────────────┘ +``` + +- **数据流方向是单向的**:MemOS → 本插件 → 执行图文件 → MemTrace。MemTrace 不会回写 MemOS,本插件也不会调用 MemTrace。 +- **连接点只有一个**:本插件输出的 `graph.json`。它采用 smartcomment 格式,和 MemTrace 已支持的 Mem0、EverMemOS、RAG 等系统的执行图格式相同,所以 MemTrace 可以直接加载,不需要额外转换。 +- **两者可以单独使用**:不装 MemTrace 时,本插件的产物也能直接查看、可视化或用脚本分析。 + +## 4. 本插件的功能 + +开发者开启插件后,它运行在 MemOS 服务进程内,通过 MemOS 已有的 Hook 机制工作,**不修改 MemOS 的任何业务代码**。以下功能都只在插件开启时生效。 + +### 4.1 记录写入链路 + +记录一次 `/product/add` 中,记忆从原始对话到写入存储的全过程: + +```text +原始消息批次 ──提取──▶ 提取出的候选记忆 ──写入──▶ 已写入存储的记忆(带 memory_id) +``` + +功能:监测这批消息被提取成了哪几条记忆,每条记忆写入后的 ID 是什么,提取或写入有没有失败 + +### 4.2 记录后台增强 + +MemOS 的 MemRead 调度器会在请求返回**之后**继续异步处理记忆:精炼增强、写入增强结果、删除旧记忆。本插件把这些迟到的操作**追加到原来那张 Add 图上**,一张图就能看完一次写入的前因后果: + +```text +已写入的记忆 ──增强──▶ 增强后的记忆 ──写入──▶ 新的已写入记忆 + └──删除──▶ 调度结果 +``` + +功能:监测 MemRead 调度器将记忆后来被改成了什么 + +### 4.3 记录检索管线 + +记录一次 `/product/search` 的每个阶段,图中**只保留各阶段处理后仍存在的记忆列表**: + +```text +查询 → 记忆类型 → cube → 原始召回 → 阈值过滤 → 去重 → 重排(最终结果) +``` + +功能:通过比较相邻阶段的结果列表,监测记忆有没有被召回、在哪个阶段消失,以及最终返回结果的顺序和分数。被剔除的记忆不再单独生成列表节点,阶段 metadata 保留过滤数量统计。结果列表的 category 按 raw、threshold、dedup、rerank 区分,DOT 图以不同颜色展示各阶段。 + +### 4.4 检索结果溯源 + +开启 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true` 后,本插件会把 Search 的最终结果**连回它们在历史 Add 图中的来源**,并把来源链路复制进这张 Search 图: + +```text +原始消息批次 → 提取 → 写入 → 增强 → 写入 ◀──溯源── 最终检索结果 +``` + +连接边由最终检索结果列表指向记忆单元,comment 展示该结果在列表内从 1 开始的 rank 和实际 score。这样只看一张 Search 图,就能知道「这条结果最初来自哪批消息、中间经历了什么」。 + +这是本插件最关键的产品能力:它把原本分散在多次请求里的写入图和检索图串成一条完整的证据链,MemTrace 拿到这张图后,可以从检索结果一路追溯到原始对话。 + +### 4.5 请求目录,开箱即读 + +输出文件按「用户 / 日期 / 请求类型」组织,每个用户自动生成一份 `index.md`,按时间倒序列出每次请求的类型、查询或消息摘要、trace ID 和图文件链接。不需要任何工具,打开 Markdown 就能找到想看的那次请求。 + +| 时间 | 类型 | 请求摘要 | Trace ID | 图 | +| --- | --- | --- | --- | --- | +| 2026-10-15T14:36:10+08:00 | search | 我什么时候去游泳? | 462f7ef9... | graph | +| 2026-10-15T14:35:22+08:00 | add | 2 messages: 我上周开始学游泳了… | 0ba7563d... | graph | + +通过请求头 `x-trace-id` 可以指定 trace ID,方便把执行图和业务日志对应起来。 + +### 4.6 可视化与脚本分析 + +本插件的产物是标准 smartcomment 执行图,因此可以直接使用 smartcomment 自带的能力: + +- `smartcomment-visualize` 一键生成交互式 HTML、PNG 图片或 DOT 源文件; +- 用 `ExecNetwork` 写几行 Python,查询祖先 / 后代节点、按类别筛选,例如「这条消息最后变成了哪些记忆」「哪些记忆在去重阶段被剔除」。 + +> 严格来说,可视化和 `ExecNetwork` 是 **smartcomment 的能力**,本插件负责产出能被它们直接读取的文件。 + +### 4.7 调试时不干扰服务 + +本插件不面向生产部署,但开发者常常需要在接近真实的环境中复现问题。下面这些设计保证:即使在调试时开启插件,MemOS 的正常服务也不受影响。 + +| 特性 | 说明 | +| --- | --- | +| **默认关闭** | 安装后不会生效,必须显式设置 `MEMOS_ENABLED_PLUGINS=smartcomment` 才会启动插件,去掉后重启即可关闭。生产部署保持默认即可,无需任何额外配置。 | +| **非侵入** | 只订阅 Hook,不改变任何请求、响应和业务结果。插件出错不会影响 MemOS 请求,调用方对插件是否开启无感。 | +| **不阻塞请求** | Hook 中只做数据快照,建图、溯源和写文件都在后台线程完成。队列满时丢弃事件并告警,不会为了记录而拖慢业务。 | +| **可靠落盘** | 原子写文件,服务重启或缓存淘汰后,迟到的事件会从磁盘恢复,继续追加到原来那张图上。 | + +## 5. MemTrace 中与本插件相关的功能 + +MemTrace 是一个完整的记忆系统错误分析框架,下面与本插件相关的能力由 MemTrace 提供,需要单独安装 [MemTrace 仓库](https://github.com/zjunlp/MemTrace)并配置 OpenAI 兼容的模型。本插件不包含也不修改 MemTrace 的任何功能。 + +### 5.1 共同基础:operation-variable 执行图 + +MemTrace 把记忆系统的运行过程表示为一张「操作—变量」执行图:**变量**是运行中产生的具体数据,**操作**是产生或使用这些数据的步骤,**边**表示数据依赖。答案出错时,沿着依赖往回追,就能找到错误是在哪一步引入的。 + +本插件就是按这套模型来记录 MemOS 的,两边的概念一一对应: + +| MemTrace 概念 | 在本插件产物中的对应 | +| --- | --- | +| **变量** | 原始消息批次(`input_message_batch`)、提取出的记忆(`extracted_memory`)、已写入的记忆(`persisted_memory`)、增强后的记忆(`enhanced_memory`)、检索查询(`search_query`)、各阶段保留的检索结果列表(`search_memory_list_{stage}`) | +| **操作** | MemOS 中的提取、写入、后台增强、归档 / 删除、召回、阈值过滤、去重、重排,以及检索结果溯源,统一以 `memos.` 开头命名 | +| **依赖** | Add 内的「消息 → 提取 → 写入 → 增强」、Search 内的各阶段结果链,以及「最终检索结果 → 记忆单元」的溯源边;只记录确定的关系,不按时间先后猜测 | +| **说明文字** | 每个节点、边和操作都带一句英文 `comment`,MemTrace 探查时会展示给 Agent,帮助它理解每一步在做什么 | + +因此,本插件输出的 `graph.json` 可以用 `ExecNetwork.import_graph()` 直接加载,这也是 MemTrace 内部读取执行图的同一个入口,不需要格式转换。 + +### 5.2 交互式图探查:可直接使用 + +这是 MemTrace 中与本插件结合最紧密的能力,由两个组件组成: + +- **`GraphTraceNotebook`**:加载一张执行图,从指定的起点节点出发,向 Agent 提供「查看节点、展开上游 / 下游、查看操作」等图工具; +- **`GraphTraceAgent`**:一个 LLM Agent,根据你的问题调用这些工具逐步查看图,最后基于图中的证据给出结论。 + +用于本插件产物时: + +| 环节 | 做法 | +| --- | --- | +| **选哪张图** | 推荐用**开启溯源后的 Search 图**。它同时包含检索的各个阶段和最终结果的 Add 来源,Agent 在一张图里就能从检索结果一路追到原始对话。不开溯源时 Search 图只有检索部分,Add 图只有写入部分,Agent 无法跨图回溯。 | +| **从哪里开始** | 怀疑「检索出了问题」时,以最终结果列表(`search_memory_list_rerank`)为起点;怀疑「某句话没记住」时,以对应的消息批次节点为起点。 | +| **怎么提问** | 告诉 Agent 预期答案,让它逐步检查相关消息有没有被提取、写入、增强,最终有没有被检索到,在哪个操作丢失了。 | +| **能得到什么** | 一个基于图中证据的结论,例如「该事实在 MemReader 提取时未被抽出」或「该记忆被召回,但在阈值过滤阶段被剔除」。 | + +### 5.3 批量错误归因与诊断报告:需要补充问答数据 + +如果要分析的不是单个问题,而是一整轮评测中的大量失败用例,可以使用 MemTrace 的两个批量工具: + +- **批量错误归因**(`run_error_attribution.py`):对每个失败用例自动定位导致错误的关键操作,并判断错误类型; +- **诊断报告**(`generate_report.py`):汇总所有归因结果,用 LLM 总结目标系统的常见失败模式,输出系统级报告。 + +这两个工具不能直接读取本插件的产物,原因是它们需要一张**完整的问答图**,即除了写入和检索,还要有问题、模型回答、标准答案和标注。本插件只记录 MemOS **服务端内**的写入和检索;「根据检索结果生成回答」通常发生在客户端或评测脚本中,因此不会出现在图里。 + +| 批量归因需要的内容 | 本插件是否提供 | +| --- | --- | +| 记忆写入、后台增强的执行过程 | ✅ Add 图 | +| 检索各阶段的执行过程与结果 | ✅ Search 图 | +| 检索结果到原始消息的依赖链 | ✅ 开启溯源后的 Search 图 | +| 问题、模型回答、标准答案、错误标注 | ❌ 需要在评测侧补充 | + +也就是说,本插件提供了批量归因中最难获取的那部分数据,即 MemOS 内部的执行过程。剩下的问答数据需要和评测结果一起,整理成 MemTrace 批量工具要求的格式(MemTraceBench 格式)。 + +## 6. 典型使用场景 + +### 场景一:单个 bad case 排查(只用本插件) + +> 用户说「我告诉过你我周二周四游泳」,MemOS 却答不上来。 + +1. 打开该用户的 `index.md`,找到那次 Search 和之前的 Add; +2. 看 Add 图:这句话有没有被提取成记忆?后台增强时有没有被删掉或改写? +3. 看 Search 图:这条记忆有没有被召回?是被阈值、去重还是重排剔除的? +4. 开启溯源(即 Add 和 Search 图相连)后,只看 Search 图就能把整条链路看全。 + +**价值**:几分钟内判断问题出在提取、增强还是检索,不用翻日志,也不用加调试代码。 + +### 场景二:AI 分析(本插件 + MemTrace Agent) + +> 主要场景:链路很长、记忆很多,人工看图太累,使用 Agent 进行错误归因。 + +1. 本插件产出 Search 图(建议开启溯源); +2. 用 MemTrace 的 `GraphTraceNotebook` 加载这张图,以最终检索结果或可疑的消息批次为起点; +3. 告诉 `GraphTraceAgent` 预期答案,让它逐步检查「相关消息有没有被提取、写入、最终检索到,是哪个操作丢掉的」。 + +**价值**:由 Agent 基于图中证据给出结论,适合复杂 case 或需要可解释结论的场合。 + +### 场景三:系统级评测与归因(本插件 + MemTrace 批量工具,需要额外整理数据) + +> 在 LoCoMo 等基准上评测 MemOS,想知道失败主要集中在哪些环节。 + +1. 用独立的输出目录跑评测,本插件记录全部 Add / Search; +2. 在评测脚本中完成问答和打分,把失败用例的问题、答案、标注和对应执行图组织成 MemTraceBench 格式; +3. 运行 MemTrace 的批量归因,再用诊断报告汇总失败模式。 + +**价值**:从「某个 case 错了」上升到「MemOS 的哪个模块最常出错」,为优化提供方向。 + +## 7. 快速开始(开发 / 调试环境) + +以下步骤只用于本地或测试环境的调试,生产部署请跳过。 + +```bash +# 1. 在运行 MemOS 的同一环境中安装 +python -m pip install "smartcomment>=0.1.2" +python -m pip install -e apps/memtrace-memos-integration + +# 2. 按需开启插件进行调试(必须在启动服务前设置;不设置则插件不加载) +export MEMOS_ENABLED_PLUGINS=smartcomment +export MEMOS_SMARTCOMMENT_OUTPUT_DIR="$PWD/.memos/memtrace" +export MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true # 推荐:开启检索溯源 +python -m uvicorn memos.api.server_api:app --host 127.0.0.1 --port 8000 --workers 1 + +# 3. 正常调用 /product/add 和 /product/search 后,查看请求目录 +find .memos/memtrace -name index.md + +# 4. 调试结束:取消环境变量后重启服务,即恢复为未开启插件的状态 +unset MEMOS_ENABLED_PLUGINS +``` + +## 8. 相关链接 + +- [MemTrace](https://github.com/zjunlp/MemTrace) · [论文](https://arxiv.org/abs/2605.28732) · [MemTraceBench](https://huggingface.co/datasets/zjunlp/MemTraceBench) +- [smartcomment](https://github.com/zjunlp/smartcomment) · [PyPI](https://pypi.org/project/smartcomment/) diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md index 7f26ceffb..991bb091f 100644 --- a/apps/memtrace-memos-integration/README.md +++ b/apps/memtrace-memos-integration/README.md @@ -132,7 +132,7 @@ Set these environment variables before loading the plugin. Restart MemOS after c | `MEMOS_SMARTCOMMENT_MAX_CACHED_TRACES` | `64` | Maximum graphs retained in memory. | | `MEMOS_SMARTCOMMENT_MAX_CACHED_GRAPH_ITEMS` | `50000` | Maximum cached nodes, edges, operations, and sessions. | | `MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS` | `300` | Idle cache expiration in seconds. | -| `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` | `false` | Include the Add ancestry of final Search hits and link persisted memory nodes to the final result lists. | +| `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` | `false` | Include the Add ancestry of final Search hits and link final result lists to persisted memory nodes. | Snapshots are written atomically. An idle or full in-memory cache can be evicted; a later event restores the saved graph and appends to it. Cache limits do not delete JSON files. Use a different output directory for each writer process, including when running multiple server workers. @@ -152,13 +152,16 @@ message batch -> extracted memory -> persisted memory -> enhanced memory | v persisted memory + ^ + memory_search_lineage (rank, score) | - memory_search_lineage - v final search memory list ``` -The new edge points **from the persisted memory to the final Search list**. Only +The new edge points **from the final Search list to the persisted memory**. Its comment +shows `rank` (1-based position within that final list) and the observed `score`, for +example `rank=2, score=0.8`. Both fields are also stored in edge metadata. Zero scores +are preserved; missing scores are shown as `unknown` and stored as `null`. Only explicit incoming dependencies are followed; sibling outputs from the same Add, unrelated memories, downstream archive/delete branches, and prior Search graphs are not imported. A shared ancestor is included once. If multiple source memories @@ -166,8 +169,9 @@ actually contributed to a merged memory, all those ancestors remain relevant. Message batches remain the original captured units; the plugin does not infer which individual messages within a batch contributed to a memory. -Raw candidates and filtered lists do not trigger Add imports. Repeated hits in one -list share a single link; the same memory in different final lists links to each. +Only final results trigger Add imports. Repeated hits in one list share a single link +using their first occurrence's rank and score; each final list links to the same memory +independently. Unresolved or anonymous entries still occupy their original list positions. Missing IDs, unavailable Add history, and ambiguous identities produce no guessed links. `memos.search.link_add_lineage` reports linked/unresolved counts for distinct identifiable hits per list. Imported items retain `lineage_source_trace_id` metadata; @@ -232,15 +236,14 @@ The asynchronous MemRead scheduler also emits: - `enhanced_memory`: a fine-transfer result reused as the input to the enhanced-memory database write; -- `scheduler_result`: a string result for operations such as archive, delete, +- `scheduler_result`: a string result for operations such as delete, soft-delete, or failure that do not otherwise expose a graphable output value. Scheduler fine-transfer inputs reuse the `memory::` identity created by the earlier TextMemory persistence event. This connects the original persisted memory -to its enhanced successor without creating a duplicate memory node. Archive, delete, +to its enhanced successor without creating a duplicate memory node. Delete, soft-delete, and failed MemRead operations are also recorded; failures include only the -error type, not the error message or an error-value node. The current MemRead refresh -call is not wrapped by an operation Hook and is not recorded separately. +error type, not the error message or an error-value node. A persisted memory unit is one immutable graph anchor per cube and memory ID. Its first complete snapshot is retained; different later observations log a warning without replacing @@ -268,6 +271,18 @@ Successful MemReader, TextMemory, and scheduler callbacks observe the piped `res their matching `failed` callbacks record a status node without changing MemOS exception handling. MemReader and TextMemory failures also omit exception messages. +Scheduler callbacks receive `hook_context`, `operation`, `operation_input`, and either +`result` or `error`. Scheduler message dictionaries are converted into `HookContext` by MemOS; the +plugin uses that context for trace, user, session, task, and cube correlation. + +Fine-transfer `operation_input` includes `memories`, `type`, `custom_tags`, `user_name`, +`chat_history`, `user_context`, `is_upload_skill`, and `result_grouping`. Delete and +soft-delete inputs include `memory_ids` and `user_name`; soft-delete also includes +`preserved_memory_ids`. The plugin uses `user_name` only as a fallback cube identifier, +never as the user identity. Reader arguments do not become memory nodes, and preserved +IDs do not become deletion edges. See the [MemOS Hook reference](../../src/memos/plugins/HOOKS.md#scheduler-memory-operations) +for the upstream contract. + A successful Hook means the observed method returned; the plugin does not independently verify database persistence. MemOS may catch and log lower-level write errors before they reach the Hook boundary, so a `persisted_memory` node alone is not proof of a @@ -280,10 +295,6 @@ pipeline for each populated type/cube bucket: ```text query -> memory type -> cube -> raw list -> threshold list -> dedup list -> rerank list - | | | - v v v - threshold's dedup's rerank's - filtered list filtered list filtered list ``` Raw retrieval creates `search_memory_type` and `search_cube` branch nodes followed @@ -292,6 +303,9 @@ omitted. The three retrieval links use `memory_type_branch`, `memory_cube_branch and `memory_retrieval`, respectively. Every enabled, observed stage creates one new list node per observed bucket and connects the last recorded list directly to it. A fully removed bucket continues as an empty list, preserving the stage chain. +The list categories are `search_memory_list_raw`, `search_memory_list_threshold`, +`search_memory_list_dedup`, and `search_memory_list_rerank`, so DOT colors each stage +separately. Their `class_name` remains `search_memory_list`. The plugin follows the API's current stage conditions even though the API emits Hooks for disabled stages: threshold is recorded only when `relativity > 0`, and @@ -312,26 +326,26 @@ the list node's metadata. Individual memories no longer create graph nodes, rank or pairwise node matches. Snapshots are taken per entry so the outer list retains all results, including lists longer than 200 entries; field truncation and redaction still apply within each entry. With `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true`, -final lists additionally receive incoming links from matching persisted memory units. +final lists additionally link to matching persisted memory units, with per-list rank +and score on each edge. -When a stage removes entries, its last recorded list also connects to a -`search_filter_result` node containing the removed entries, rather than a status -string. One filtered list is created per affected bucket. Differences preserve -duplicate counts and prefer identical entries before falling back to memory ID -(or original text for anonymous entries), so reordered results and expanded text -with the same ID are not treated as removals. `filtered_count` counts removed -memories across buckets. No filtered node is created when nothing was removed. +Each stage records only its retained result lists. Removed entries do not create +separate list nodes or edges. `filtered_count` still counts removed occurrences across +buckets by memory ID (or original text for anonymous entries), including duplicates. +Reordering, expanded text, and score changes for the same ID do not count as removals. -Branches and lists use `HookContext.operation_id` as their search scope. List links +Branches and lists use `trace_id` for stage connections. List links use result type, cube ID, and the occurrence of that cube within the type; buckets without IDs use their position. Reordered unique cube IDs remain correctly linked. Repeated or missing cube IDs rely on stable occurrence order or position. A new bucket without a previous-stage list is retained without a guessed source edge. -Search observation ends at `search.results.after_rerank`. Each search caches only -its latest recorded lists; skipped stages leave that cache intact. Rerank, -post-processing failure, and shutdown release -remaining caches in their respective scopes. The current rerank event describes +The plugin copies `HookContext.operation_id` unchanged into event, operation, and +edge metadata. Optional Add-history links retain the rerank Hook's operation ID. + +Search observation ends at `search.results.after_rerank`. Each trace caches only +its latest recorded lists; skipped stages leave that cache intact. Rerank and +post-processing failure clear that trace's cache; shutdown clears all caches. The current rerank event describes the API's descending `metadata.relativity` sort; the tracing handler preserves observed order and does not sort results itself. diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md index e3746b8cb..cf63e2442 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -39,12 +39,12 @@ Recorder 用 `id_strategy` 返回 `TraceValue.identity`,因此映射如下: | 标识 | 规则 | | --- | --- | | 图的隔离键 | Adapter 按 `(user_id, trace_id)` 管理图;`graph_id = trace_id`,`project_id` 来自配置,默认 `memos` | -| `operation_id` | 来自 Hook 业务上下文,用于关联一次业务操作;存在时由 `_event()` 补入 `TraceEvent.metadata` | -| `event_id` | 每次创建事件都生成一个 `uuid4().hex`,标识本次观测;不是业务 `operation_id` | +| `operation_id` | 由 `_event()` 从 HookContext 原样复制到事件 metadata,再由 recorder 传入 operation 和 edge metadata | +| `event_id` | 每次创建事件都生成一个 `uuid4().hex`,标识本次观测;与 `operation_id` 分别生成 | | `created_at` | 事件创建时的 UTC ISO 时间 | | SmartComment `session_id` | 每个事件使用 `event-{event_id}`,session 名称是 `event.operation` | | MemOS `session_id` | 业务会话 ID,导出时放在 session/operation metadata 的 `memos_session_id` 中 | -| SmartComment `op_id` | 由 SmartComment 创建 operation 时分配;业务 `operation_id` 仍在 metadata 中 | +| SmartComment `op_id` | 由 SmartComment 创建 operation 时分配;事件的 `operation_id` 保存在 metadata 中 | 默认情况下,即使两个 trace 出现相同 `identity`,也不会跨图合并。开启 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` 后,writer 会按最终 Search 命中的持久化记忆导入其 Add 上游子图,见第 4.1 节;不是全局合并 trace。 @@ -79,7 +79,7 @@ cube 列表取第一个非空字段:`writable_cube_ids → readable_cube_ids | 节点 `category` | 值在流程中的业务角色,使用 snake_case 名词,如 `extracted_memory`、`persisted_memory`。相同正文在不同阶段可有不同角色。 | | 节点 `class_name` | SmartComment 类型/身份命名空间,不是阶段名;`memory` 共用于提取、增强和持久化记忆,`str` 用于查询和状态。改名会改变 `full_node_id`。 | | `TraceEvent.operation` | 稳定的操作标识,以 `memos.` 开头,点号分隔流程和动作;Scheduler 保留调用点及 `after/failed` 后缀。与业务关联 ID `operation_id` 不同。 | -| 事件/边 `category` | 操作或依赖关系的类别,如 `memory_persistence`、`memory_filtered`,不要求与节点类别相同。 | +| 事件/边 `category` | 操作或依赖关系的类别,如 `memory_persistence`、`memory_threshold_passed`,不要求与节点类别相同。 | | 节点 `comment` | 说明捕获了什么值及其观测边界;搜索列表注明阶段并保留返回顺序,状态节点不冒充业务输出。 | | 事件 `comment` | 说明本次操作的业务意图和实际观测结果,写入导出 operation/session 的 `comment`。自定义事件未提供时保留按 operation 生成的默认说明。 | | 连线 `comment` | 解释为什么从来源值连向目标值;所有内置流程显式提供,不用操作名代替依赖含义。 | @@ -90,7 +90,7 @@ cube 列表取第一个非空字段:`writable_cube_ids → readable_cube_ids ## 2. `TraceValue` 节点总表 -下面的 `{cube}` 在 Add 链中表示 `cube_id or 'unknown'`;`{trace}` 为解析后的 trace。`{search_scope}` 优先使用非 `None` 的 `operation_id`,否则使用 trace。`{status}` 在调度链中为 `after` 或 `failed`。 +下面的 `{cube}` 在 Add 链中表示 `cube_id or 'unknown'`;`{trace}` 为解析后的 trace,也是 Search 各阶段的关联依据。`{status}` 在调度链中为 `after` 或 `failed`。 ### 2.1 AddHandler:输入、提取、持久化与后台增强 @@ -125,15 +125,14 @@ Recorder 对导出的 `persisted_memory.metadata` 额外添加 `snapshot_status= | 业务含义 | `category` | `class_name` | `identity` | `value` | | --- | --- | --- | --- | --- | -| 搜索查询 | `search_query` | `str` | `search_query:{search_scope}` | 请求 query;无字段时默认空字符串 | -| 召回类型分支 | `search_memory_type` | `search_memory_type` | `search_memory_type:{search_scope}:{result_type}` | results 顶层类型字符串,例如 `text_mem`、`tool_mem` | -| 召回 cube 分支 | `search_cube` | `search_cube` | `search_cube:{search_scope}:{result_type}:{bucket_index}` | cube ID;缺失时为 `bucket-{bucket_index}` | -| 阶段结果列表 | `search_memory_list` | `search_memory_list` | `search_memory_list:{search_scope}:{stage}:{bucket_key}` | 当前桶的记忆列表,每项包含 `memory_id`、`memory`、`score` | -| 被过滤记忆列表 | `search_filter_result` | `search_memory_list` | `search_filter_result:{search_scope}:{stage}:{bucket_key}` | 当前阶段从前一阶段移除的记忆列表,条目形状与结果列表一致 | +| 搜索查询 | `search_query` | `str` | `search_query:{trace}` | 请求 query;无字段时默认空字符串 | +| 召回类型分支 | `search_memory_type` | `search_memory_type` | `search_memory_type:{trace}:{result_type}` | results 顶层类型字符串,例如 `text_mem`、`tool_mem` | +| 召回 cube 分支 | `search_cube` | `search_cube` | `search_cube:{trace}:{result_type}:{bucket_index}` | cube ID;缺失时为 `bucket-{bucket_index}` | +| 阶段结果列表 | `search_memory_list_{stage}` | `search_memory_list` | `search_memory_list:{trace}:{stage}:{bucket_key}` | 当前桶的记忆列表,每项包含 `memory_id`、`memory`、`score` | | 搜索身份/位置字段 | 规则 | | --- | --- | -| `stage` | `raw`、`threshold`、`dedup`、`rerank` | +| `stage` | `raw`、`threshold`、`dedup`、`rerank`;对应 category 为 `search_memory_list_raw`、`search_memory_list_threshold`、`search_memory_list_dedup`、`search_memory_list_rerank`,使 DOT 按阶段区分颜色 | | `result_type` | results 顶层 key 转字符串,例如 `text_mem`、`tool_mem`;handler 没有限定枚举,也不是源记忆的 `memory_type` | | `bucket_index` | 同一 result type 内的桶序号,从 0 开始 | | `bucket_key` | 紧凑 JSON 数组 `[result_type, cube_id, occurrence]` 经 URL 百分号编码后的字符串(`quote(..., safe="")`),避免引号和反斜杠破坏 DOT 节点 ID;cube ID 非 `None` 时 occurrence 为同 type 下该 cube 的出现序号,从 0 开始;缺失 ID 时使用 bucket_index | @@ -147,8 +146,7 @@ Recorder 对导出的 `persisted_memory.metadata` 额外添加 `snapshot_status= | `search_query` | `memos_stage="search.before"` | | `search_memory_type` | `stage="raw"`、`result_type` | | `search_cube` | `stage="raw"`、`result_type`、`cube_id`、`bucket_index` | -| `search_memory_list` | `stage`、`result_type`、`cube_id`、`bucket_index`、`memory_count` | -| `search_filter_result` | 与结果列表相同,另加 `status="filtered"` | +| `search_memory_list_{stage}` | `stage`、`result_type`、`cube_id`、`bucket_index`、`memory_count` | Search 消费的形状是 `result_type → 桶列表 → memories 列表`,例如: @@ -169,11 +167,11 @@ results = { raw 阶段显式建立 `search_query → search_memory_type → search_cube → search_memory_list`。每个非空桶只创建一个记忆列表节点;空 type 和 cube 桶不产生分支。后续仅为已启用且收到 Hook 的阶段创建列表节点,连接最近实际记录的列表。关闭的阶段不产生事件、节点或边;已启用阶段即使结果没有改变也保留。若桶被全部过滤或从返回结构中消失,后续结果列表为 `[]`,链路继续保留。首次在后续阶段出现的非空桶保留独立列表,不猜测来源边。 -列表按 Hook 实际返回顺序保存;handler 不生成逐条 memory 节点,不计算 rank、rank_scope 或 global_position,也不做节点级配对。开启 Add 关联时,writer 额外导入命中记忆的 `persisted_memory` 及祖先,并连接到最终列表,见第 4.1 节。 +列表按 Hook 实际返回顺序保存;handler 不生成逐条 memory 节点,不计算 rank、rank_scope 或 global_position,也不做节点级配对。开启 Add 关联时,writer 额外导入命中记忆的 `persisted_memory` 及祖先,从最终列表连向记忆单元,并在边上记录列表内 rank 和 score,见第 4.1 节。 -被过滤的内容只通过相邻列表的多重集差集计算,不用于决定阶段连线。先扣除完全相同的条目,再按 memory ID 扣除剩余项;没有 ID 时使用原始正文。这样既保留重复项的数量,又不会将同 ID 的正文扩展或分数变化误报为过滤。无法区分的同 ID 剩余项按原列表次序扣除。过滤列表保留前一阶段的内容和顺序,仅在实际有移除项时生成。 +被过滤的记忆不生成独立列表节点或边。`filtered_count` 通过相邻已记录列表的多重集差集计数,按 memory ID 匹配;没有 ID 时使用截断前的原始正文。这样既保留重复项的数量,又不会将同 ID 的正文扩展或分数变化误报为过滤。该计数不用于决定阶段连线。 -内部保留脱离业务对象的完整条目用于差集计算,显示时逐条调用快照处理;外层结果/过滤列表不受通用 200 项容器截断限制,条目内部仍遵守文本长度、脱敏及向量省略规则。`memory_count` 为该列表实际条目数。 +内部保留脱离业务对象的完整条目用于差集计算,显示时逐条调用快照处理;外层结果列表不受通用 200 项容器截断限制,条目内部仍遵守文本长度、脱敏及向量省略规则。`memory_count` 为该列表实际条目数。 ### 2.3 BaseHandler:通用失败状态 @@ -187,7 +185,7 @@ raw 阶段显式建立 `search_query → search_memory_type → search_cube → MemRead 调度失败使用上一节的 `scheduler_result`,不使用这个节点。异常类型 `error_type` 放在事件/操作 metadata 中;异常对象和异常消息不作为失败节点内容。 -当前 handler 共使用 **6 种 class_name**:`message_batch`、`memory`、`str`、`search_memory_type`、`search_cube`、`search_memory_list`;共使用 **11 种节点 category**:以上表格中的五种 Add、五种 Search 和一种通用失败状态。`TraceValue` 自身的默认 `category="variable"`、`class_name=None` 供自定义调用使用,当前 handler 显式指定这两个字段。 +当前 handler 共使用 **6 种 class_name**:`message_batch`、`memory`、`str`、`search_memory_type`、`search_cube`、`search_memory_list`;共使用 **13 种节点 category**:以上表格中的五种 Add、七种 Search 和一种通用失败状态。`TraceValue` 自身的默认 `category="variable"`、`class_name=None` 供自定义调用使用,当前 handler 显式指定这两个字段。 ## 3. AddHandler 的 operation 与连线 @@ -246,7 +244,7 @@ flowchart LR flowchart LR P[persisted_memory] -->|memory_refinement| E[enhanced_memory] E -->|memory_persistence| P2[persisted_memory] - P -->|归档、删除或其他操作结果| S[scheduler_result] + P -->|删除或其他操作结果| S[scheduler_result] E -->|scheduler_operation_failed| SF[scheduler_result: failed] ``` @@ -257,20 +255,23 @@ flowchart LR | 订阅 Hook | `scheduler.memory.operation.after` / `scheduler.memory.operation.failed` | | `event.operation` | `memos.scheduler.mem_read.{operation_name}.{status}` | | `event.category` | 固定 `scheduler_memory_operation` | -| 共同 metadata | `memos_stage="scheduler.mem_read.{operation_name}.{status}"`、`handler="mem_read"`、`operation_name`、`operation`、`target`、`status` | -| `operation / target` | 直接记录 Hook 参数,例如 `fine/add/archive/delete/soft_delete` 和 `textual_memory`;不是插件内强制枚举 | +| 共同 metadata | `memos_stage="scheduler.mem_read.{operation_name}.{status}"`、`handler="mem_read"`、`operation_name`、`operation`、`status` | +| `operation` | 直接记录 Hook 参数,当前 MemRead 使用 `fine/add/delete/soft_delete`;不是插件内强制枚举 | | 失败额外 metadata | `error_type`,仅异常类名 | | `operation_name` / 情况 | 输入、输出与边 category | | --- | --- | | `fine_transfer_simple_mem` 成功且有记忆输出 | operation_input.memories 中的持久化记忆引用 → enhanced_memory;`memory_refinement` | | `add_enhanced_memories` 成功且有返回 ID | enhanced_memory → persisted_memory;`memory_persistence`,沿用唯一 ID 配对 | -| `archive_merged_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_archival` | -| `remove_memories` / `remove_source_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_deletion` | +| `remove_memories` 成功 | memory_ids 对应持久化记忆 → scheduler_result;`memory_deletion` | | 其他符合 source 前缀的操作成功 | memory_ids 对应持久化记忆 → scheduler_result;`scheduler_operation_result` | | 失败 | 当前操作输入 → scheduler_result;`scheduler_operation_failed`;增强写入失败保留 enhanced_memory 输入 | -handler 并没有封闭的 operation_name 枚举:上述特殊分支之外,其他符合 source 前缀的名称按通用结果分支处理。这里描述的是插件收到 Hook 后的处理能力,不代表每个调度调用都有 Hook;当前 MemRead 的 `remove_and_refresh_memory` 没有独立观测点,不生成刷新事件。成功的 fine-transfer/增强写入如果没有记忆输出,也会生成 scheduler_result;没有输入时保留独立结果节点和 operation,不补造边。 +handler 并没有封闭的 operation_name 枚举:上述特殊分支之外,其他符合 source 前缀的名称按通用结果分支处理。这里描述的是插件收到 Hook 后的处理能力,不代表每个调度调用都有 Hook;当前 MemRead 的 `remove_and_refresh_memory` 和 merged-memory 归档调用没有独立观测点,不生成刷新或归档事件。成功的 fine-transfer/增强写入如果没有记忆输出,也会生成 scheduler_result;没有输入时保留独立结果节点和 operation,不补造边。 + +当前回调参数为 `hook_context`、`operation`、`operation_input` 和成功时的 `result` / 失败时的 `error`。MemOS 将字典形式的调度消息转换为 `HookContext`,插件只从该 context 获取 trace、user、session、task 和 cube 关联信息;`operation_input.user_name` 仅作为 cube 缺失时的补充,不作为 user 身份。 + +fine-transfer 输入现包含 `memories`、`type`、`custom_tags`、`user_name`、`chat_history`、`user_context`、`is_upload_skill`、`result_grouping`。删除输入包含 `memory_ids`、`user_name`,软删除额外包含 `preserved_memory_ids`。插件只用记忆输入及 grouping 构造来源边,不将其他 reader 参数绘成记忆节点,也不为 preserved IDs 生成删除边。上游契约见 [HOOKS.md](../../../src/memos/plugins/HOOKS.md#scheduler-memory-operations)。 fine-transfer 额外记录 `result_grouping`,取自 `operation_input`,缺省 `unknown`: @@ -295,12 +296,9 @@ flowchart LR R -->|memory_threshold_passed| T[threshold] T -->|memory_deduplicated| D[dedup] D -->|memory_reranked| RR[rerank] - R -->|memory_filtered| F1[threshold: filtered list] - T -->|memory_filtered| F2[dedup: filtered list] - D -->|memory_filtered| F3[rerank: filtered list] ``` -上图展示全部阶段启用时的链路。raw、threshold、dedup、rerank 框都表示一个桶的 `search_memory_list` 节点;filtered list 表示该桶在当前阶段移除的记忆列表。 +上图展示全部阶段启用时的链路。raw、threshold、dedup、rerank 框都表示一个桶在该阶段保留的 `search_memory_list` 节点。 插件按当前业务条件判断阶段是否启用:`relativity > 0` 才记录 threshold,`dedup` 为 `sim` 或 `mmr` 才记录 dedup。业务层即使跳过处理仍会发出这两个 Hook,插件会直接忽略它们,并保留最近记录的列表。只关闭 threshold 时链路为 `raw → dedup → rerank`;只关闭 dedup 时为 `raw → threshold → rerank`;两者都关闭时为 `raw → rerank`。中间 Hook 未触发时,后续阶段同样连接最近实际记录的列表。 @@ -315,7 +313,7 @@ API 后处理当前始终调用 `rerank_knowledge_mem`,因此收到 rerank Hoo | `search.results.after_rerank` | `memos.search.rerank` | `memory_reranking` | 最近记录列表 → rerank;`memory_reranked` | | `search.post_process.failed` | `memos.search.post_process.failed` | `search_failure` | query → operation_status;`operation_failed` | -已记录的 threshold、dedup、rerank 中,每个桶最近实际记录的结果列表连接当前结果列表;若有条目移除,再以一条 `memory_filtered` 边连接该桶的 `search_filter_result` 列表。差集和来源边均以最近记录的列表为准,因此跳过阶段不会丢失过滤内容。被过滤列表也包括去重及 rerank 阶段移除的记忆,且不再使用固定字符串状态节点。 +已记录的 threshold、dedup、rerank 中,每个桶最近实际记录的结果列表连接当前保留的结果列表;不为阈值过滤、去重或 rerank 移除的条目生成独立列表或状态节点。过滤数量统计和来源边均以最近实际记录的列表为准。 | 操作 | 由 handler 添加的 `TraceEvent.metadata`,不含公共补充字段 | | --- | --- | @@ -326,24 +324,26 @@ API 后处理当前始终调用 `rerank_knowledge_mem`,因此收到 rerank Hoo | rerank | `memos_stage="search.results.after_rerank"`、`algorithm="relativity_sort"`、`order_by="metadata.relativity"`、`direction="descending"`、`top_k` | | post_process.failed | `memos_stage="search.post_process.failed"`、`status="failed"`、`error_type`、`handler`(API handler 的类名) | -四个结果阶段另加 `link_strategy="bucket_lists"` 和 `filtered_count`。raw 阶段 filtered_count 为 0;其余阶段为所有桶的过滤列表条目数之和,不是过滤节点数。rerank 的 algorithm/order_by/direction 是插件当前固定写入的描述,不是对实际后端算法的动态识别;插件也不自己重新过滤、去重或排序。`top_k` 是当前 Hook 请求中的值,可能已经被业务层扩充。 +四个结果阶段另加 `link_strategy="bucket_lists"` 和 `filtered_count`。raw 阶段 filtered_count 为 0;其余阶段为相对最近记录阶段、所有桶移除的条目数之和。rerank 的 algorithm/order_by/direction 是插件当前固定写入的描述,不是对实际后端算法的动态识别;插件也不自己重新过滤、去重或排序。`top_k` 是当前 Hook 请求中的值,可能已经被业务层扩充。 + +缓存以 trace_id 为键,只保存该 trace 最近实际记录阶段的列表快照,按 bucket_key 索引。已记录阶段替换前一份快照;关闭的阶段不触碰缓存,raw 开始时丢弃该 trace 的旧快照;rerank 和 post_process.failed 清理该 trace 的缓存,shutdown 清理全部。 -缓存键为 `(trace_id, search_scope_id)`,每次搜索只保存最近实际记录阶段的列表快照,按 bucket_key 索引。已记录阶段替换前一份快照;关闭的阶段不触碰缓存,raw 开始时丢弃同 scope 的旧快照;rerank 和 post_process.failed 清理本次搜索的缓存,shutdown 清理全部。若缺少 `operation_id`,search_scope 退回 trace,此时同 trace 多次搜索不再具有独立身份,调用方应提供一次搜索各 Hook 共享的 operation_id。 +各阶段事件使用对应 HookContext.operation_id,同一事件中的所有桶和连线共用该 ID。节点身份和缓存使用 trace_id;插件只消费 Hook 已提供的上下文。 观测终点为 `search.results.after_rerank`;插件没有订阅 `search.context.render` 或 `search.after`。 -### 4.1 可选的 Add → Search 记忆流转 +### 4.1 可选的 Search → Add 记忆溯源 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD` 默认 `false`,在插件加载时读取。 开启后,异步 writer 只在 `memos.search.rerank` 事件落盘前进行关联: -1. 从本次事件的最终 `search_memory_list` 读取 `(cube_id, memory_id)`,不读取 raw 或 filtered list 作为命中集合。 +1. 从本次事件的 `search_memory_list_rerank` 节点读取 `(cube_id, memory_id)`,不读取早期阶段结果作为命中集合。 2. 在相同 user/project 的已落盘图中匹配持久化锚点。没有 ID、没有 Add 观测、身份有歧义时不猜测、不按正文匹配。 3. 仅逆向遍历 `memory_extraction`、`memory_persistence`、`memory_refinement` 入边。共同祖先去重,可跨 trace 追溯 MemRead 增强的原始来源;不扩展兄弟输出、归档/删除分支或其他 Search。 4. 导入选中的节点、边及其所需 operation/session,保留原 ID,graph_id 改为当前 Search trace,metadata 中以 `lineage_source_trace_id` 保存原 trace。源文件不变;已导入的副本不会成为后续 Search 的新来源。 -5. 用显式 `TraceLink` 创建 `persisted_memory → 最终 search_memory_list`,operation 为 `memos.search.link_add_lineage`,边 category 为 `memory_search_lineage`。一份列表中同一 memory 只连一次,不同列表各自连线。 +5. 用显式 `TraceLink` 创建 `最终 search_memory_list → persisted_memory`,operation 为 `memos.search.link_add_lineage`,边 category 为 `memory_search_lineage`。comment 展示该结果的 `rank` 和 `score`,例如 `rank=2, score=0.8`。rank 从 1 开始,以所属最终列表的实际返回顺序为准,不是跨桶的全局排名;未关联或无 ID 的条目仍占据原位置。一份列表中同一 memory 只连一次,使用第一次出现的 rank 和 score;不同列表各自连线。 -关联事件 metadata 中的 `requested_count`、`linked_count`、`unresolved_count` 按每个最终列表内去重后的、具有 cube/memory ID 的条目计数,`lineage_status` 为 `resolved/partial/unresolved`。边额外记录 `cube_id`、`memory_id`、`source_trace_id`。 +关联事件沿用 rerank Hook 的 `operation_id`。metadata 中的 `requested_count`、`linked_count`、`unresolved_count` 按每个最终列表内去重后的、具有 cube/memory ID 的条目计数,`lineage_status` 为 `resolved/partial/unresolved`。边额外记录 `cube_id`、`memory_id`、`source_trace_id`、`rank`、`score`。score 取结果条目中的实际分数,保留 0;缺失时 comment 显示 `score=unknown`,metadata 保存 `null`。 图切片以已观测节点为单位:共享 message batch 保留完整捕获内容,不推测批次内单条消息的贡献;合并记忆真实依赖的多个来源均属于相关祖先。当前 trace 原有节点不被裁剪,开关关闭后也不删除历史连线。 @@ -354,7 +354,7 @@ writer 每次最终 Search 扫描当前用户的 JSON 历史,临时索引大 | 层级 | 内容 | 自动继承情况 | | --- | --- | --- | | 节点 `TraceValue.metadata` | 第 2 节列出的值身份、阶段、列表条目数或输入参数 | 不自动继承事件 metadata、HookContext.attributes 或源记忆的完整 metadata | -| 事件 `TraceEvent.metadata` | 第 3/4 节列出的本次操作参数和结果说明 | `_event()` 在 subject.operation_id 非 `None` 时用 setdefault 补 `operation_id` | +| 事件 `TraceEvent.metadata` | 第 3/4 节列出的本次操作参数和结果说明 | `_event()` 从 Hook subject 用 setdefault 补 `operation_id` | | 导出 operation metadata | 事件 metadata,加关联公共字段 | 公共字段覆盖事件 metadata 同名项 | | 导出 edge metadata | operation metadata,加 `TraceLink.metadata` | link 字段优先;持久化边额外有 pair_index/input_index | | 导出 session metadata | 单独构造的事件关联字段 | 不自动复制完整 event metadata | @@ -385,7 +385,7 @@ session_metadata = { 内置 handler 为事件及每条边分别提供说明;自定义 `TraceLink.comment=None` 仍继承操作说明。说明文字不放进 metadata,也不参与节点身份或跨阶段匹配。 -持久化配对边和可选的 Add → Search 关联边显式添加 link metadata,其他 handler 边使用空字典并继承 operation metadata。 +持久化配对边和可选的 Search → Add 关联边显式添加 link metadata,其他 handler 边使用空字典并继承 operation metadata。 ## 6. 快照、导出与边界 @@ -394,7 +394,7 @@ session_metadata = { | 规则 | 当前行为 | | --- | --- | | 字符串长度 | 默认保留前 20,000 字符,超出追加 `…`;限制按每个字符串计算 | -| 容器大小 | 默认每层最多 200 项,附截断提示;Search 结果和过滤列表逐项快照,保留全部外层条目 | +| 容器大小 | 默认每层最多 200 项,附截断提示;Search 结果列表逐项快照,保留全部外层条目 | | 敏感字段 | 字段名规范化后包含 api_key/apikey/access_key/password/secret/authorization/cookie/credential/token 时,值替换为 `` | | 向量字段 | embedding/embeddings/vector/vectors 的值替换为 `` | | argument/arguments JSON 字符串 | 先解析并递归脱敏,再编码和截断;不能解析为 dict/list 则替换为 `` | @@ -416,7 +416,7 @@ session_metadata = { 阅读顺序: - `add_handler.py`:同步 Add 回调 → MemRead 回调及调度流程 → 消息关联状态 → 节点构造 → 来源配对。 -- `search_handler.py`:按阶段排列的回调 → 列表阶段链路 → raw 分桶分支 → 列表快照 → 过滤差集 → operation 清理。 +- `search_handler.py`:按阶段排列的回调 → 列表阶段链路 → raw 分桶分支 → 列表快照 → 过滤数量统计 → operation 清理。 - `base.py`:上下文字段适配 → 快照/事件构造与提交 → Hook 关联 → 公用失败节点与去重。 - `adapter.py`:请求侧 start/submit/flush/close → 工作线程的快照读写 → 缓存管理与处理循环。 - `storage.py`:用户身份与短哈希 → 固定请求目录 → 图和元数据原子写入 → 用户索引与历史图查找。 @@ -428,7 +428,7 @@ session_metadata = { | Add 身份与 metadata | [add_handler.py](handlers/add_handler.py):`_message_value`、`_extracted_memory_value`、`_persisted_memory_value`、`_enhanced_memory_value` | | 返回 ID 怎么连回输入 | [add_handler.py](handlers/add_handler.py):`_persistence_values`、`_persistence_lineage` | | 后台增强的来源关系 | [add_handler.py](handlers/add_handler.py):`_fine_transfer_values`、`_on_scheduler_memory_operation` | -| 搜索列表、阶段连线和过滤内容 | [search_handler.py](handlers/search_handler.py):`_on_search_results`、`_search_result_buckets`、`_search_memory_list_value`、`_removed_memories` | +| 搜索列表、阶段连线和过滤数量 | [search_handler.py](handlers/search_handler.py):`_on_search_results`、`_search_result_buckets`、`_search_memory_list_value`、`_removed_memory_count` | | 图中身份、metadata、占位补全 | [recorder.py](recorder.py):`_as_comment_item`、`record`、`_complete_memory_snapshots` | | 快照与落盘 | [serialization.py](serialization.py)、[adapter.py](adapter.py) | diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py index 9dbff3802..f5f29426a 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/add_handler.py @@ -520,23 +520,19 @@ def _persisted_memory_reference( ) # Background memory maintenance - # MemRead refines stored memories, writes enhanced candidates, and archives - # or removes source memories. The scheduler operation selects the relevant - # memory flow and records its inputs and outputs. Archival, deletion, and - # failures link the affected memories to an operation status. + # MemRead observes refinement, enhanced writes, and source-memory removal. + # Deletion and failures link the affected memories to an operation status. def on_scheduler_memory_operation_after( self, *, hook_context: HookContext, operation: Any, - target: Any, operation_input: Any, result: Any, ) -> None: self._on_scheduler_memory_operation( hook_context=hook_context, operation=operation, - target=target, operation_input=operation_input, result=result, status="after", @@ -547,14 +543,12 @@ def on_scheduler_memory_operation_failed( *, hook_context: HookContext, operation: Any, - target: Any, operation_input: Any, error: BaseException, ) -> None: self._on_scheduler_memory_operation( hook_context=hook_context, operation=operation, - target=target, operation_input=operation_input, error=error, status="failed", @@ -565,7 +559,6 @@ def _on_scheduler_memory_operation( *, hook_context: HookContext, operation: Any, - target: Any, operation_input: Any, result: Any | None = None, error: BaseException | None = None, @@ -580,9 +573,7 @@ def _on_scheduler_memory_operation( description = { "fine_transfer_simple_mem": "Refine source memories into enhanced memories", "add_enhanced_memories": "Write enhanced memories and observe returned memory IDs", - "archive_merged_memories": "Mark merged source memories as archived", "remove_memories": "deletion of source memories", - "remove_source_memories": "deletion of source memories", }.get(operation_name, f"Run MemRead Scheduler operation {operation_name}") outcome = "the method returned" if status == "after" else "an exception reached the Hook" operation_comment = f"{description} (backend operation: {operation}); {outcome}." @@ -623,11 +614,7 @@ def _on_scheduler_memory_operation( category = "scheduler_operation_failed" link_comment = "Raise an observed exception." value_comment = f"An observed exception {error!s} occurred." - elif operation_name == "archive_merged_memories": - category = "memory_archival" - link_comment = "Archive the memory unit." - value_comment = "The memory unit has been archived successfully." - elif operation_name in {"remove_memories", "remove_source_memories"}: + elif operation_name == "remove_memories": category = "memory_deletion" link_comment = f"Remove the memory unit through the {operation} operation." value_comment = ( @@ -661,7 +648,6 @@ def _on_scheduler_memory_operation( "handler": "mem_read", "operation_name": operation_name, "operation": operation, - "target": target, "status": status, } if operation_name == "fine_transfer_simple_mem": diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index d1077d300..46abe3430 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -1,4 +1,4 @@ -"""Capture Search result lists and the memories removed between stages.""" +"""Capture retained Search result lists at each observed stage.""" from __future__ import annotations @@ -78,7 +78,7 @@ def __init__( max_value_chars: int = 20_000, ) -> None: super().__init__(submit, max_value_chars=max_value_chars) - self._search_stages: dict[tuple[str, str], dict[_BucketKey, _SearchList]] = {} + self._search_stages: dict[str, dict[_BucketKey, _SearchList]] = {} self._search_stages_lock = Lock() def clear(self) -> None: @@ -88,8 +88,8 @@ def clear(self) -> None: # Query intake and search identity # A search starts with a query and the readable cubes from its request. - # Hook context and request fields place that query in a user trace and - # operation scope. Every result stage refers to the same query identity. + # Hook context and request fields place that query in a user trace. + # Every result stage refers to the same query identity. def on_search_before( self, *, @@ -125,12 +125,6 @@ def _search_subject( subject["mem_cube_ids"] = cube_ids return subject - @staticmethod - def _search_scope_id(subject: Any) -> str: - """Identify one search operation within a possibly shared trace.""" - operation_id = _get(subject, "operation_id") - return str(operation_id) if operation_id is not None else _current_trace_id(subject) - def _search_query_value( self, subject: Any, @@ -138,10 +132,10 @@ def _search_query_value( *, identity_only: bool, ) -> TraceValue: - search_scope_id = self._search_scope_id(subject) + trace_id = _current_trace_id(subject) return self._value( value=_get(request, "query", ""), - identity=f"search_query:{search_scope_id}", + identity=f"search_query:{trace_id}", category="search_query", class_name="str", identity_only=identity_only, @@ -152,7 +146,7 @@ def _search_query_value( # Retrieved candidates # Retrieval returns candidates grouped by memory type and storage cube. # Each bucket becomes one list node, preserving the observed memory order. - # Stage links connect entire lists; removed memories form a separate list. + # Stage links connect the retained lists at successive observed stages. def on_search_memory_results( self, *, @@ -273,7 +267,7 @@ def _on_search_results( return subject = self._search_subject(hook_context, request) buckets = self._search_result_buckets(results) - scope_key = self._search_scope_key(subject) + trace_id = _current_trace_id(subject) stage_label = _SEARCH_STAGE_LABELS.get(stage, stage) retained_comment = { @@ -300,7 +294,7 @@ def _on_search_results( ) with self._search_stages_lock: # Skipped or missing stages leave the last observed lists as predecessors. - previous = self._search_stages.pop(scope_key, {}) + previous = self._search_stages.pop(trace_id, {}) if stage == "raw": previous = {} @@ -339,33 +333,17 @@ def _on_search_results( source=source, target=result.node, category=_SEARCH_LINK_CATEGORIES[stage], - comment=f"Process memory candidates through the {stage_label} stage.", + comment=f"Filter memory candidates through the {stage_label} stage.", ) ) - removed = self._removed_memories(prior.bucket.memories, result.bucket.memories) - if removed: - filtered = self._search_memory_list_value( - subject, - stage=stage, - bucket=replace(prior.bucket, memories=removed), - filtered=True, - retained_comment=retained_comment, - ) - outputs += (filtered,) - filtered_count += len(removed) - stage_links.append( - TraceLink( - source=source, - target=filtered, - category="memory_filtered", - comment=f"Filter memory candidates through the {stage_label} stage.", - ) - ) + filtered_count += self._removed_memory_count( + prior.bucket.memories, result.bucket.memories + ) links = tuple(stage_links) inputs = self._unique_values([link.source for link in links]) with self._search_stages_lock: - self._search_stages[scope_key] = current + self._search_stages[trace_id] = current event_metadata = { **metadata, @@ -384,7 +362,7 @@ def _on_search_results( metadata=event_metadata, ) ) - # Rerank and failure are terminal stages for this operation's cache. + # Rerank and failure are terminal stages for this trace's cache. if stage == "rerank": self._clear_search_stages(subject) @@ -396,7 +374,7 @@ def _search_retrieval_branches( results: tuple[_SearchList, ...], ) -> tuple[tuple[TraceValue, ...], tuple[TraceLink, ...]]: """Expand raw retrieval into query -> result type -> cube -> memory list.""" - search_scope_id = self._search_scope_id(subject) + trace_id = _current_trace_id(subject) types: dict[str, TraceValue] = {} branches: list[TraceValue] = [] links: list[TraceLink] = [] @@ -406,7 +384,7 @@ def _search_retrieval_branches( if result_type not in types: type_node = self._value( value=result_type, - identity=f"search_memory_type:{search_scope_id}:{result_type}", + identity=f"search_memory_type:{trace_id}:{result_type}", category="search_memory_type", class_name="search_memory_type", identity_only=False, @@ -428,7 +406,7 @@ def _search_retrieval_branches( cube_label = cube_id if cube_id is not None else f"bucket-{bucket.bucket_index}" cube_node = self._value( value=cube_label, - identity=f"search_cube:{search_scope_id}:{result_type}:{bucket.bucket_index}", + identity=f"search_cube:{trace_id}:{result_type}:{bucket.bucket_index}", category="search_cube", class_name="search_cube", identity_only=False, @@ -500,7 +478,7 @@ def _search_result_buckets(results: Any) -> dict[_BucketKey, _SearchBucket]: cube_id=cube_id, bucket_index=bucket_index, occurrence=occurrence, - # Keep full content for list differences before display truncation. + # Count anonymous removals before display truncation. memories=deepcopy(items), ) buckets[bucket.key] = bucket @@ -513,9 +491,7 @@ def _search_memory_list_value( stage: str, bucket: _SearchBucket, retained_comment: str = "", - filtered: bool = False, ) -> TraceValue: - category = "search_filter_result" if filtered else "search_memory_list" # DOT renderers interpolate IDs inside quotes; encode the structured key # so its quotes and backslashes cannot split one node into multiple IDs. bucket_key = quote( @@ -529,66 +505,38 @@ def _search_memory_list_value( "memory_count": len(bucket.memories), } - if filtered: - metadata["status"] = "filtered" return TraceValue( # Bound each entry's fields without truncating the outer result list. value=[self._snapshot(memory) for memory in bucket.memories], - identity=f"{category}:{self._search_scope_id(subject)}:{stage}:{bucket_key}", - category=category, + identity=f"search_memory_list:{_current_trace_id(subject)}:{stage}:{bucket_key}", + category=f"search_memory_list_{stage}", class_name="search_memory_list", - comment=( - f"Memories filtered out during the {stage} stage." if filtered else retained_comment - ), + comment=retained_comment, metadata=self._snapshot(metadata), ) - def _search_scope_key(self, subject: Any) -> tuple[str, str]: - return ( - _current_trace_id(subject), - self._search_scope_id(subject), - ) - @staticmethod - def _removed_memories( - previous: list[dict[str, Any]], current: list[dict[str, Any]] - ) -> list[dict[str, Any]]: - """List multiset difference for reporting removals, never for graph links.""" - - def encode(value: Any) -> str: - return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + def _removed_memory_count(previous: list[dict[str, Any]], current: list[dict[str, Any]]) -> int: + """Count removed occurrences without creating a discarded-memory list.""" def identity(memory: dict[str, Any]) -> str: memory_id = memory["memory_id"] - return encode( - ["id", str(memory_id)] if memory_id is not None else ["text", memory["memory"]] + return json.dumps( + ["id", str(memory_id)] if memory_id is not None else ["text", memory["memory"]], + ensure_ascii=False, + sort_keys=True, + default=str, ) - remaining = Counter(identity(memory) for memory in current) - exact = Counter(encode(memory) for memory in current) - unmatched = [] - # Preserve the right occurrence when equal IDs have different scores/text. - for memory in previous: - snapshot = encode(memory) - if exact[snapshot]: - exact[snapshot] -= 1 - remaining[identity(memory)] -= 1 - else: - unmatched.append(memory) - removed = [] - for memory in unmatched: - key = identity(memory) - if remaining[key]: - # The same memory can gain expanded text or a new score during rerank. - remaining[key] -= 1 - else: - removed.append(memory) - return removed + # Scores and expanded text can change while the same memory is retained. + before = Counter(identity(memory) for memory in previous) + after = Counter(identity(memory) for memory in current) + return sum((before - after).values()) # Search completion # Reranking completes the search result flow. A post-processing exception # links the query to a failure status. Both outcomes release the cached - # result lists for that search operation. + # result lists for that trace. def on_search_post_process_failed( self, *, @@ -630,4 +578,4 @@ def on_search_post_process_failed( def _clear_search_stages(self, subject: Any) -> None: with self._search_stages_lock: - self._search_stages.pop(self._search_scope_key(subject), None) + self._search_stages.pop(_current_trace_id(subject), None) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/lineage.py b/apps/memtrace-memos-integration/memos_smartcomment/lineage.py index 9a0504d8c..b1c35493a 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/lineage.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/lineage.py @@ -157,18 +157,21 @@ def connect_search_lineage( event: TraceEvent, sources: Iterable[dict[str, Any]], ) -> tuple[dict[str, Any], TraceEvent | None]: - """Return a detached graph slice and an explicit persisted-memory -> list event.""" - targets: list[tuple[tuple[str, str], TraceValue]] = [] + """Return a detached Add slice and explicit final-list -> persisted-memory links.""" + hits: list[tuple[tuple[str, str], TraceValue, int, Any]] = [] for output in event.outputs: - if output.category != "search_memory_list" or output.metadata.get("stage") != "rerank": + if ( + output.category != "search_memory_list_rerank" + or output.metadata.get("stage") != "rerank" + ): continue seen = set() - for entry in output.value: + for rank, entry in enumerate(output.value, start=1): key = _memory_key(output.metadata.get("cube_id"), entry.get("memory_id")) if key is not None and key not in seen: - targets.append((key, output)) + hits.append((key, output, rank, entry.get("score"))) seen.add(key) - if not targets: + if not hits: return current, None ancestry = _Ancestry() @@ -177,8 +180,8 @@ def connect_search_lineage( ancestry.include(source) ancestry.include(current) roots = ancestry.roots() - matched = [(key, target) for key, target in targets if key in roots] - selected = ancestry.slice(roots[key] for key, _ in matched) + matched = [(key, result, rank, score) for key, result, rank, score in hits if key in roots] + selected = ancestry.slice(roots[key] for key, _result, _rank, _score in matched) merged = deepcopy(current) for field, id_field in _IDS.items(): items = {item[id_field]: item for item in merged["data"][field]} @@ -199,9 +202,12 @@ def connect_search_lineage( merged["data"][field] = list(items.values()) links = [] - for key, target in matched: + for key, result, rank, score in matched: node = ancestry.items["nodes"][roots[key]] - source = TraceValue( + + score_str = f"{float(score):.3f}" if score is not None else "unknown" + + memory = TraceValue( value=json.loads(node["value"]), identity=node["name"], category=node["category"], @@ -210,16 +216,22 @@ def connect_search_lineage( comment=node.get("comment"), metadata=deepcopy(node["metadata"]), ) + links.append( TraceLink( - source=source, - target=replace(target, identity_only=True), + source=replace(result, identity_only=True), + target=memory, category="memory_search_lineage", - comment="This persisted memory is retrieved in the final search result list.", + comment=( + f"Retrieve this memory at rank={rank}, " + f"score={score_str} in the final search result list." + ), metadata={ "cube_id": key[0], "memory_id": key[1], "source_trace_id": node["graph_id"], + "rank": rank, + "score": score, }, ) ) @@ -235,15 +247,11 @@ def connect_search_lineage( metadata={ "memos_stage": "search.results.after_rerank", "operation_id": event.metadata.get("operation_id"), - "requested_count": len(targets), + "requested_count": len(hits), "linked_count": len(matched), - "unresolved_count": len(targets) - len(matched), + "unresolved_count": len(hits) - len(matched), "lineage_status": ( - "resolved" - if len(matched) == len(targets) - else "partial" - if matched - else "unresolved" + "resolved" if len(matched) == len(hits) else "partial" if matched else "unresolved" ), }, ) diff --git a/src/memos/api/handlers/search_handler.py b/src/memos/api/handlers/search_handler.py index a94318a7e..392c2b601 100644 --- a/src/memos/api/handlers/search_handler.py +++ b/src/memos/api/handlers/search_handler.py @@ -74,8 +74,6 @@ def __init__(self, dependencies: HandlerDependencies): def handle_search_memories( self, search_req: APISearchRequest, - *, - hook_context: HookContext | None = None, ) -> SearchResponse: """ Main handler for search memories endpoint. @@ -85,8 +83,6 @@ def handle_search_memories( Args: search_req: Search request containing query and parameters - hook_context: Internal correlation context shared by all Hooks for - this search operation. A context is created when omitted. Returns: SearchResponse with formatted results @@ -108,7 +104,7 @@ def handle_search_memories( results = cube_view.search_memories(search_req_local) hooked_results = trigger_hook( H.SEARCH_MEMORY_RESULTS, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -122,7 +118,7 @@ def handle_search_memories( results = self._apply_relativity_threshold(results, search_req_local.relativity) hooked_results = trigger_hook( H.SEARCH_RESULTS_AFTER_THRESHOLD, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -139,7 +135,7 @@ def handle_search_memories( self._strip_embeddings(results) hooked_results = trigger_hook( H.SEARCH_RESULTS_AFTER_DEDUP, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -157,7 +153,7 @@ def handle_search_memories( ) hooked_results = trigger_hook( H.SEARCH_RESULTS_AFTER_RERANK, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, @@ -167,7 +163,7 @@ def handle_search_memories( except Exception as error: trigger_hook( H.SEARCH_POST_PROCESS_FAILED, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, error=error, @@ -176,7 +172,7 @@ def handle_search_memories( hooked_results = trigger_hook( H.SEARCH_CONTEXT_RENDER, - hook_context=hook_context, + hook_context=self._build_search_hook_context(search_req_local), handler=self, search_req=search_req_local, results=results, diff --git a/src/memos/plugins/HOOKS.md b/src/memos/plugins/HOOKS.md index 6a3276507..e2895e14d 100644 --- a/src/memos/plugins/HOOKS.md +++ b/src/memos/plugins/HOOKS.md @@ -85,11 +85,10 @@ Hooks that do **not** provide context will never pass it, even if you declare th ### Context scope -- All hooks in one **search** invocation share the same context object. It is built before - `search.before`, so replacing the request does not rebuild the context. Use the callback's - `request` argument for the current business input. +- **Search** builds a context for `search.before` and `search.after`. Each intermediate + hook creates its own context. - **Reader extraction**, **cube writes**, and **scheduler operations** each create a separate - context. Do not assume they share an `operation_id`; use `trace_id` or `task_id` for + context. Use `trace_id` or `task_id` for broader correlation. --- @@ -110,8 +109,6 @@ search.before └─► search.after ``` -Every hook receives `hook_context`. Context source: `api.search`; attributes contain `mode`. - | Constant / Hook name | Additional arguments | `pipe_key` | Typical use | | --- | --- | --- | --- | | `H.SEARCH_BEFORE` / `search.before` | `request` | `request` | Rewrite the query or inject filters before retrieval. | diff --git a/src/memos/plugins/hooks.py b/src/memos/plugins/hooks.py index 084f0b66b..f3dc6a67d 100644 --- a/src/memos/plugins/hooks.py +++ b/src/memos/plugins/hooks.py @@ -148,7 +148,8 @@ def hookable( Auto-declares before/after Hooks (idempotent); no need to manually define_hook in hook_defs.py. Supports piped return values: before can modify request, after can modify result. When ``context_builder`` is provided, one context is built before the first - Hook and passed to both Hooks and the decorated method as ``hook_context``. + Hook and passed to both Hooks as ``hook_context``. It is also passed to the + decorated method when that method accepts it. Compatible with both sync and async methods. """ from memos.plugins.hook_defs import define_hook @@ -184,7 +185,8 @@ async def async_wrapper(self, request, *args, **kwargs): call_kwargs = dict(kwargs) if context_builder is not None: before_kwargs["hook_context"] = hook_context - call_kwargs["hook_context"] = hook_context + if _accepts_hook_context(func): + call_kwargs["hook_context"] = hook_context rv = trigger_hook(f"{name}.before", **before_kwargs) request = rv if rv is not None else request @@ -207,7 +209,8 @@ def sync_wrapper(self, request, *args, **kwargs): call_kwargs = dict(kwargs) if context_builder is not None: before_kwargs["hook_context"] = hook_context - call_kwargs["hook_context"] = hook_context + if _accepts_hook_context(func): + call_kwargs["hook_context"] = hook_context rv = trigger_hook(f"{name}.before", **before_kwargs) request = rv if rv is not None else request diff --git a/tests/api/test_search_hook_context.py b/tests/api/test_search_hook_context.py index a5d39b3e8..3a7df0698 100644 --- a/tests/api/test_search_hook_context.py +++ b/tests/api/test_search_hook_context.py @@ -1,3 +1,5 @@ +import inspect + import pytest from memos.api.handlers.base_handler import HandlerDependencies @@ -47,7 +49,7 @@ def _search_handler() -> tuple[SearchHandler, _CubeView]: return handler, cube_view -def test_search_success_hooks_run_in_order_and_share_context(monkeypatch): +def test_search_success_hooks_run_in_order_with_distinct_stage_operations(monkeypatch): events = [] result_hook_calls = [] handler, cube_view = _search_handler() @@ -97,7 +99,8 @@ def replace_response(*, hook_context, result, **_kwargs): H.SEARCH_AFTER, ] contexts = [context for _name, context in events] - assert all(context is contexts[0] for context in contexts) + assert contexts[-1] is contexts[0] + assert len({context.operation_id for context in contexts}) == 6 context = contexts[0] assert isinstance(context, HookContext) assert context.trace_id == "trace-1" @@ -107,6 +110,13 @@ def replace_response(*, hook_context, result, **_kwargs): assert context.source == "api.search" assert context.attributes == {"mode": "fast"} assert context.operation_id + for stage_context in contexts[1:-1]: + assert stage_context.source == "api.search" + assert stage_context.operation_id != context.operation_id + assert stage_context.attributes == {"mode": "fast"} + assert stage_context.attributes is not context.attributes + for field in ("trace_id", "user_id", "session_id", "task_id", "cube_ids"): + assert getattr(stage_context, field) == getattr(context, field) assert cube_view.request.query == "hooked query" assert all(call["handler"] is handler for call in result_hook_calls) assert all(call["search_req"] is cube_view.request for call in result_hook_calls) @@ -143,23 +153,42 @@ def replace_memories(*, results, **_kwargs): assert response.data["text_mem"][0]["memories"] == [memory] -def test_search_post_processing_failure_triggers_only_failure_hook(monkeypatch): +@pytest.mark.parametrize("failed_stage", ["threshold", "dedup", "rerank"]) +def test_search_post_processing_failure_identifies_the_failed_operation(monkeypatch, failed_stage): error = RuntimeError("post processing failed") events = [] handler, _cube_view = _search_handler() - search_req = APISearchRequest(user_id="user", query="query") + search_req = APISearchRequest(user_id="user", query="query", dedup="sim") + observed_contexts = [] + register_hook( + H.SEARCH_BEFORE, lambda **kwargs: observed_contexts.append(kwargs["hook_context"]) + ) + for name in ( + H.SEARCH_MEMORY_RESULTS, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + ): + register_hook(name, lambda **kwargs: observed_contexts.append(kwargs["hook_context"])) monkeypatch.setattr( "memos.api.handlers.search_handler.get_current_trace_id", lambda: "trace-1", ) - def fail_post_processing(_results, _relativity): + def fail_post_processing(*_args, **_kwargs): raise error def record_failure(**kwargs): events.append((H.SEARCH_POST_PROCESS_FAILED, kwargs)) - handler._apply_relativity_threshold = fail_post_processing + if failed_stage == "threshold": + handler._apply_relativity_threshold = fail_post_processing + elif failed_stage == "dedup": + handler._dedup_text_memories = fail_post_processing + else: + monkeypatch.setattr( + "memos.api.handlers.search_handler.rerank_knowledge_mem", fail_post_processing + ) register_hook(H.SEARCH_POST_PROCESS_FAILED, record_failure) register_hook(H.SEARCH_CONTEXT_RENDER, lambda **_kwargs: events.append(("render", {}))) register_hook(H.SEARCH_AFTER, lambda **_kwargs: events.append(("after", {}))) @@ -176,3 +205,32 @@ def record_failure(**kwargs): context = failure["hook_context"] assert context.trace_id == "trace-1" assert context.user_id == "user" + assert context.source == "api.search" + assert context.attributes == {"mode": "fast"} + assert context.operation_id not in {prior.operation_id for prior in observed_contexts} + + +def test_search_handler_does_not_accept_hook_context(): + assert list(inspect.signature(SearchHandler.handle_search_memories).parameters) == [ + "self", + "search_req", + ] + + +def test_search_stage_operations_stay_isolated_between_searches_in_one_trace(monkeypatch): + handler, _cube_view = _search_handler() + monkeypatch.setattr( + "memos.api.handlers.search_handler.get_current_trace_id", lambda: "one-trace" + ) + contexts = [] + for name in ( + H.SEARCH_BEFORE, + H.SEARCH_RESULTS_AFTER_THRESHOLD, + H.SEARCH_RESULTS_AFTER_DEDUP, + H.SEARCH_RESULTS_AFTER_RERANK, + ): + register_hook(name, lambda **kwargs: contexts.append(kwargs["hook_context"])) + for query in ("first", "second"): + handler.handle_search_memories(APISearchRequest(user_id="user", query=query)) + assert len({context.operation_id for context in contexts}) == 8 + assert all(context.attributes == {"mode": "fast"} for context in contexts) diff --git a/tests/plugins/test_hook_context.py b/tests/plugins/test_hook_context.py index ad06e2f98..453c80fd2 100644 --- a/tests/plugins/test_hook_context.py +++ b/tests/plugins/test_hook_context.py @@ -186,3 +186,26 @@ def callback(*, hook_context, **_kwargs): ("operation", supplied_context), ("after", supplied_context), ] + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_hookable_supports_method_without_context_parameter(is_async): + context = build_hook_context(source="test") + observed = [] + + async def async_run(self, request): + return request + + def sync_run(self, request): + return request + + class Handler: + run = hookable("without_context", context_builder=lambda _self, _request: context)( + async_run if is_async else sync_run + ) + + for name in ("without_context.before", "without_context.after"): + register_hook(name, lambda **kwargs: observed.append(kwargs["hook_context"])) + result = Handler().run("request") + assert (asyncio.run(result) if is_async else result) == "request" + assert observed == [context, context] From 188cd47e5fc7c9aedea894d793d107cadc632d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=BE=E9=97=AE?= Date: Tue, 10 Nov 2026 13:29:46 +0800 Subject: [PATCH 17/22] perf(smartcomment): reduce search tracing overhead - Defer search event assembly and stage comparisons to the writer - Reduce redundant snapshot copies and serialization - Batch user index updates while checkpointing graphs per event - Preserve write retries and flush guarantees with regression tests --- .../memos_smartcomment/CONVENTIONS.md | 8 +- .../memos_smartcomment/adapter.py | 178 +++++++++++++----- .../memos_smartcomment/config.py | 9 +- .../memos_smartcomment/events.py | 14 +- .../memos_smartcomment/handlers/__init__.py | 21 +++ .../handlers/search_handler.py | 168 ++++++++++++----- .../memos_smartcomment/plugin.py | 5 +- .../memos_smartcomment/storage.py | 45 ++++- 8 files changed, 330 insertions(+), 118 deletions(-) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md index cf63e2442..dbb9d2d3c 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -6,7 +6,7 @@ ## 1. 公共模型与身份 -调用链为 `MemOS Hook → HookHandlers → TraceEvent → AsyncTraceAdapter → SmartCommentRecorder → JSON`。`HookHandlers` 组合 `AddHandler` 和 `SearchHandler`,两者也可以独立使用。 +调用链为 `MemOS Hook → HookHandlers → TraceEvent / DeferredTraceEvent → AsyncTraceAdapter → SmartCommentRecorder → JSON`。Search 结果阶段的 `DeferredTraceEvent` 在 writer 内构造为 `TraceEvent` 后直接写图,不再次入队。`HookHandlers` 组合 `AddHandler` 和 `SearchHandler`,两者也可以独立使用;未提供 `defer_search` 时保持同步事件交付。 | 对象 | 表示什么 | 关键字段 | | --- | --- | --- | @@ -389,7 +389,7 @@ session_metadata = { ## 6. 快照、导出与边界 -`_value()` 对 value 和 metadata、`_event()` 对事件 metadata,在 Hook 当前线程内执行快照;后台线程负责写图。dataclass 的 frozen 只禁止字段重新赋值,不深度冻结嵌套字典。 +`_value()` 对 value 和 metadata、`_event()` 对事件 metadata,在 Hook 当前线程内执行快照。Search 结果阶段只提取必要字段,逐字段截断和脱敏一次,复用已独立保存的列表作为节点值,不再先 deepcopy 再序列化整个条目。阶段关联和清理仍在请求线程完成;差异统计、分支及边的构造和写图在后台按入队顺序处理。事件时间和请求上下文在入队前固定,后台不读取业务原对象。长匹配键和结构化匿名正文用摘要保留截断前的匹配语义,不在队列中保留未脱敏的结构化正文。dataclass 的 frozen 只禁止字段重新赋值,不深度冻结嵌套字典。 | 规则 | 当前行为 | | --- | --- | @@ -407,7 +407,9 @@ session_metadata = { 输出文件按 `(user_id, trace_id)` 隔离,路径为 `用户--短哈希/日期/入口类型/时间__trace前缀--短哈希/graph.json`。可读前缀保留 Unicode 字母、最多 32 字符,哈希初始 12 位,发生碰撞时延长;完整身份保存在 `_user.json` 和每张图的 `meta.json` 中。入口类型分为 `search/add/memread/other`,由首次记录事件决定。日期和时间默认使用运行 MemOS 服务的机器或容器当地时区(配置值 `local`),按事件发生日期应用夏令时规则;通过 `MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE` 可指定 `Asia/Shanghai`、`UTC` 等 IANA 时区覆盖默认值。目录和索引使用当地时间,事件原始 UTC 时间戳保持不变。 -同一 trace 首次写入后固定目录,跨天异步事件、缓存淘汰、重启和时区变更不改变路径;重用 trace 的多次请求仍进入同一张图。每个用户的 `index.md` 展示按请求时间倒序的类型、摘要、完整 trace ID 和图链接;`_index.json` 提供 trace 到元数据的映射。摘要来自已快照的 query/message,最多 160 字符加省略号,不调用模型。首次访问用户时读取 `meta.json` 重建内存索引,后续成功写入重新生成磁盘索引;元数据先预留路径,图原子写入后再发布索引,避免失败写入产生新的可点击链接。图、元数据、索引分别原子替换,不是跨文件事务。 +同一 trace 首次写入后固定目录,跨天异步事件、缓存淘汰、重启和时区变更不改变路径;重用 trace 的多次请求仍进入同一张图。每个用户的 `index.md` 展示按请求时间倒序的类型、摘要、完整 trace ID 和图链接;`_index.json` 提供 trace 到元数据的映射。摘要来自已快照的 query/message,最多 160 字符加省略号,不调用模型。首次访问用户时读取 `meta.json` 重建内存索引。writer 每处理一个事件就提交图及元数据,再消费下一个事件,以缩短同进程后台构图的连续 CPU 工作,优先降低 API p99;首个事件决定目录,后续事件逐次更新摘要、操作列表和关联字段。图提交后标记用户索引待刷新,每秒集中生成一次两个用户索引。缓存上限在每个事件提交后按导出大小检查。元数据先预留路径,图与元数据提交后才发布索引;图、元数据、索引分别原子替换,不是跨文件事务。 + +`flush()` 成功表示队列、待写图和待发布索引均处理完成,仍需检查 `failed` 判断是否有事件构造失败。`processed` 在图和元数据提交后累加;可重试的持久化错误计入 `write_failures`,保留待写状态,每秒重试,期间暂停消费以限制内存积压。关闭超时只告警,worker 继续重试;持续存储故障时不能保证排空。队列积压和索引刷新会延迟可见性,进程崩溃可能丢失尚未提交的事件。启用 Search→Add 关联时,队列中此前处理的 Add 已提交,关联后立即提交当前 Search。 `output_path()` 只查询已经分配的路径,未知 trace 抛出 `FileNotFoundError`。Search lineage 通过用户索引遍历所有日期、入口类型下的 `graph.json`,不把 `meta.json` 或 `_index.json` 当作图。旧的平铺 JSON 格式不读取、不迁移,也不删除。缓存淘汰不删除磁盘快照。配置和运行方式见 [应用 README](../README.md)。 diff --git a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py index e5b8a0f6e..1468cc93e 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/adapter.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/adapter.py @@ -10,6 +10,7 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, Any, Protocol +from memos_smartcomment.events import DeferredTraceEvent from memos_smartcomment.lineage import connect_search_lineage from memos_smartcomment.recorder import SmartCommentRecorder from memos_smartcomment.storage import TraceStore @@ -29,6 +30,9 @@ logger = get_logger(__name__) +_INDEX_FLUSH_SECONDS = 1.0 +_RETRY_SECONDS = 1.0 +_TraceKey = tuple[str | None, str] class Recorder(Protocol): @@ -47,6 +51,8 @@ class AdapterStats: processed: int = 0 failed: int = 0 rejected: int = 0 + # Retried persistence failures; unlike failed, these do not discard an event. + write_failures: int = 0 @dataclass(slots=True) @@ -70,8 +76,11 @@ def __init__( self._recorder_factory = recorder_factory or ( lambda event: SmartCommentRecorder(settings, event) ) - self._queue: queue.Queue[TraceEvent] = queue.Queue(maxsize=settings.queue_size) - self._recorders: OrderedDict[tuple[str | None, str], _CachedRecorder] = OrderedDict() + self._queue: queue.Queue[TraceEvent | DeferredTraceEvent] = queue.Queue( + maxsize=settings.queue_size + ) + self._recorders: OrderedDict[_TraceKey, _CachedRecorder] = OrderedDict() + self._pending: dict[_TraceKey, list[TraceEvent]] = {} self._cached_graph_items = 0 self._thread: threading.Thread | None = None self._started = False @@ -79,6 +88,9 @@ def __init__( # Protect lifecycle state and stats; only the worker accesses the graph cache. self._state_lock = threading.Lock() self._stats = AdapterStats() + self._flush_requested = threading.Event() + self._idle = threading.Event() + self._idle.set() # Request-facing lifecycle: bounded, non-blocking submission and explicit drain/close. @property @@ -101,7 +113,7 @@ def start(self) -> None: if not self._started: self._start_worker() - def submit(self, event: TraceEvent) -> bool: + def submit(self, event: TraceEvent | DeferredTraceEvent) -> bool: with self._state_lock: if self._closed: self._stats.rejected += 1 @@ -113,26 +125,27 @@ def submit(self, event: TraceEvent) -> bool: self._stats.dropped += 1 else: self._stats.accepted += 1 + self._idle.clear() return True logger.warning("Dropping smartcomment event because the queue is full: %s", event.operation) return False - # A successful flush means tasks finished; stats.failed reports failed writes. + # Idle includes graph, metadata, and index publication, not only queue consumption. def flush(self, timeout: float = 5.0) -> bool: - deadline = time.monotonic() + timeout - with self._queue.all_tasks_done: - while self._queue.unfinished_tasks: - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._queue.all_tasks_done.wait(remaining) - return True + with self._state_lock: + if self._idle.is_set(): + return True + self._flush_requested.set() + if not self._started: + self._start_worker() + return self._idle.wait(timeout=max(0, timeout)) # Drain accepted events; a timeout warns without interrupting an active write. def close(self, timeout: float = 5.0) -> None: deadline = time.monotonic() + timeout with self._state_lock: self._closed = True + self._flush_requested.set() # Accepted events must drain even if start() was never called. if not self._started and not self._queue.empty(): self._start_worker() @@ -184,17 +197,17 @@ def sources() -> Iterator[dict[str, Any]]: logger.exception("Failed to connect Add lineage for Search trace %s", event.trace_id) # Replace atomically so readers cannot observe a partial snapshot. - def _export(self, event: TraceEvent, recorder: Recorder) -> int: + def _export(self, events: list[TraceEvent], recorder: Recorder) -> int: exported = recorder.export_graph() - self._store.write(event, exported) + self._store.write(events, exported) data = exported.get("data", {}) return sum(len(data.get(key, ())) for key in ("nodes", "edges", "operations", "sessions")) - # Worker-owned cache: keep only durable graphs and evict within configured limits. - # Evict the least recently used graph until all limits are satisfied. - def _trim_cache(self) -> None: + # Never evict uncommitted events, including when a checkpoint is being retried. + def _trim_cache(self) -> bool: now = time.monotonic() while self._recorders: + key = next(iter(self._recorders)) oldest = next(iter(self._recorders.values())) if ( len(self._recorders) <= self.settings.max_cached_traces @@ -202,51 +215,122 @@ def _trim_cache(self) -> None: and now - oldest.last_used < self.settings.cache_ttl_seconds ): break + if key in self._pending and not self._commit_trace(key): + return False _, removed = self._recorders.popitem(last=False) self._cached_graph_items -= removed.graph_items + return True - def _process(self, event: TraceEvent) -> None: + def _get_recorder(self, event: TraceEvent) -> _CachedRecorder: key = (event.user_id, event.trace_id) - cached = self._recorders.pop(key, None) + cached = self._recorders.get(key) if cached is None: recorder = self._recorder_factory(event) snapshot = self._read_snapshot(event) if snapshot is not None: recorder.restore_graph(snapshot) - else: + # A failed record may have partly mutated the graph. Replay only earlier + # successful observations, without losing their uncommitted changes. + for pending in self._pending.get(key, ()): + recorder.record(pending) + data = snapshot.get("data", {}) if snapshot is not None else {} + graph_items = sum( + len(data.get(field, ())) for field in ("nodes", "edges", "operations", "sessions") + ) + cached = _CachedRecorder(recorder, time.monotonic(), graph_items) + self._recorders[key] = cached + self._cached_graph_items += graph_items + return cached + + def _process(self, event: TraceEvent) -> None: + key = (event.user_id, event.trace_id) + cached = self._get_recorder(event) + try: + cached.recorder.record(event) + if self.settings.link_search_to_add and event.operation == "memos.search.rerank": + self._link_search_lineage(event, cached.recorder) + except Exception: + self._recorders.pop(key) self._cached_graph_items -= cached.graph_items - recorder = cached.recorder - # Cache only committed snapshots; failures reload the last durable graph. - recorder.record(event) - if self.settings.link_search_to_add and event.operation == "memos.search.rerank": - self._link_search_lineage(event, recorder) - graph_items = self._export(event, recorder) - self._recorders[key] = _CachedRecorder(recorder, time.monotonic(), graph_items) - self._cached_graph_items += graph_items - self._trim_cache() + raise + self._pending.setdefault(key, []).append(event) + cached.last_used = time.monotonic() + self._recorders.move_to_end(key) + + def _commit_trace(self, key: _TraceKey) -> bool: + events = self._pending[key] + try: + cached = self._get_recorder(events[0]) + graph_items = self._export(events, cached.recorder) + except Exception: + with self._state_lock: + self._stats.write_failures += 1 + logger.exception("Failed to checkpoint smartcomment trace %s; retaining for retry", key) + return False + self._cached_graph_items += graph_items - cached.graph_items + cached.graph_items = graph_items + del self._pending[key] + with self._state_lock: + self._stats.processed += len(events) + for _event in events: + self._queue.task_done() + return True + + def _flush_graphs(self) -> None: + # Stop consuming on I/O failure so the retry backlog cannot grow. + while self._pending: + for key in tuple(self._pending): + self._commit_trace(key) + if self._pending: + time.sleep(_RETRY_SECONDS) + + def _flush_indexes(self) -> None: + while failures := self._store.publish_indexes(): + with self._state_lock: + self._stats.write_failures += failures + time.sleep(_RETRY_SECONDS) + + def _mark_idle(self) -> bool: + with self._state_lock: + if self._queue.empty() and not self._pending and not self._store.pending_indexes: + self._flush_requested.clear() + self._idle.set() + return self._closed + return False def _run(self) -> None: + next_index = time.monotonic() + _INDEX_FLUSH_SECONDS try: while True: + timeout = max(0, min(0.05, next_index - time.monotonic())) try: - event = self._queue.get(timeout=0.05) + work = self._queue.get(timeout=timeout) except queue.Empty: - self._trim_cache() - with self._state_lock: - if self._closed and self._queue.empty(): - return - continue - try: - self._process(event) - except Exception: - with self._state_lock: - self._stats.failed += 1 - logger.exception("Failed to record smartcomment event: %s", event.operation) + pass else: - with self._state_lock: - self._stats.processed += 1 - finally: - self._queue.task_done() + try: + event = work.build() if isinstance(work, DeferredTraceEvent) else work + self._process(event) + except Exception: + with self._state_lock: + self._stats.failed += 1 + self._queue.task_done() + logger.exception("Failed to record smartcomment event: %s", work.operation) + else: + # Break up CPU-heavy recording to prioritize API tail latency. + self._flush_graphs() + + now = time.monotonic() + force = self._flush_requested.is_set() and self._queue.empty() + if now >= next_index or force: + self._flush_indexes() + next_index = time.monotonic() + _INDEX_FLUSH_SECONDS + if not self._trim_cache(): + self._flush_graphs() + self._trim_cache() + if self._mark_idle(): + return finally: - self._recorders.clear() - self._cached_graph_items = 0 + if not self._pending: + self._recorders.clear() + self._cached_graph_items = 0 diff --git a/apps/memtrace-memos-integration/memos_smartcomment/config.py b/apps/memtrace-memos-integration/memos_smartcomment/config.py index 71e0593d6..ca4738bb1 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/config.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/config.py @@ -30,7 +30,7 @@ class SmartCommentSettings: output_dir: Path = Path(".memos/memtrace") queue_size: int = 1024 - max_value_chars: float = 20_000 + max_value_chars: int = 20_000 # Enable SmartComment checks without changing immutable persisted-memory semantics. strict: bool = False project_id: str = "memos" @@ -48,12 +48,7 @@ def from_env(cls) -> SmartCommentSettings: return cls( output_dir=Path(os.getenv("MEMOS_SMARTCOMMENT_OUTPUT_DIR", ".memos/memtrace")), queue_size=_positive_int("MEMOS_SMARTCOMMENT_QUEUE_SIZE", 1024), - max_value_chars=float( - _positive_int( - "MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS", - 20_000, - ) - ), + max_value_chars=_positive_int("MEMOS_SMARTCOMMENT_MAX_VALUE_CHARS", 20_000), strict=_as_bool("MEMOS_SMARTCOMMENT_STRICT"), project_id=os.getenv("MEMOS_SMARTCOMMENT_PROJECT_ID", "memos"), max_cached_traces=_positive_int("MEMOS_SMARTCOMMENT_MAX_CACHED_TRACES", 64), diff --git a/apps/memtrace-memos-integration/memos_smartcomment/events.py b/apps/memtrace-memos-integration/memos_smartcomment/events.py index cc81ddd23..c810d712a 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/events.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/events.py @@ -2,10 +2,14 @@ from dataclasses import dataclass, field from datetime import UTC, datetime -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 +if TYPE_CHECKING: + from collections.abc import Callable + + # Handlers detach nested values; frozen only prevents field reassignment. @dataclass(frozen=True, slots=True) class TraceValue: @@ -56,3 +60,11 @@ class TraceEvent: created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat()) # Human-readable operation meaning; omitted comments use the recorder's default. comment: str | None = None + + +@dataclass(frozen=True, slots=True) +class DeferredTraceEvent: + """Build an event on the writer using only detached, already-redacted captures.""" + + operation: str + build: Callable[[], TraceEvent] diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py index 80d5a00d6..9ccfb9238 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/__init__.py @@ -1,10 +1,20 @@ # ruff: noqa: N999 """Add and Search Hook callbacks behind one plugin interface.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from memos_smartcomment.handlers.add_handler import AddHandler from memos_smartcomment.handlers.search_handler import SearchHandler +if TYPE_CHECKING: + from collections.abc import Callable + + from memos_smartcomment.events import DeferredTraceEvent, TraceEvent + + __all__ = ["AddHandler", "HookHandlers", "SearchHandler"] @@ -16,6 +26,17 @@ class HookHandlers(AddHandler, SearchHandler): keeping their correlation caches separate. """ + def __init__( + self, + submit: Callable[[TraceEvent], Any], + *, + max_value_chars: int = 20_000, + max_pending_adds: int = 1024, + defer_search: Callable[[DeferredTraceEvent], Any] | None = None, + ) -> None: + super().__init__(submit, max_value_chars=max_value_chars, max_pending_adds=max_pending_adds) + self._defer_search = defer_search + def clear(self) -> None: """Release both flows' correlation state at shutdown.""" AddHandler.clear(self) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index 46abe3430..4e5f072e6 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -6,13 +6,14 @@ from collections import Counter from collections.abc import Callable, Mapping -from copy import deepcopy from dataclasses import dataclass, replace +from functools import partial +from hashlib import sha256 from threading import Lock from typing import TYPE_CHECKING, Any from urllib.parse import quote -from memos_smartcomment.events import TraceEvent, TraceLink, TraceValue +from memos_smartcomment.events import DeferredTraceEvent, TraceEvent, TraceLink, TraceValue from memos_smartcomment.handlers.base import ( BaseHandler, _cube_ids, @@ -22,10 +23,16 @@ ) +# isort: split +from memos.log import get_logger + + if TYPE_CHECKING: from memos.plugins.hook_context import HookContext +logger = get_logger(__name__) + _SEARCH_LINK_CATEGORIES = { "threshold": "memory_threshold_passed", "dedup": "memory_deduplicated", @@ -40,6 +47,7 @@ } _BucketKey = tuple[str, str | None, int] +_MemoryKey = tuple[str, str | bytes] @dataclass(frozen=True) @@ -51,6 +59,7 @@ class _SearchBucket: bucket_index: int occurrence: int memories: list[dict[str, Any]] + identities: tuple[_MemoryKey, ...] @property def key(self) -> _BucketKey: @@ -76,8 +85,10 @@ def __init__( submit: Callable[[TraceEvent], Any], *, max_value_chars: int = 20_000, + defer_search: Callable[[DeferredTraceEvent], Any] | None = None, ) -> None: super().__init__(submit, max_value_chars=max_value_chars) + self._defer_search = defer_search self._search_stages: dict[str, dict[_BucketKey, _SearchList]] = {} self._search_stages_lock = Lock() @@ -269,7 +280,6 @@ def _on_search_results( buckets = self._search_result_buckets(results) trace_id = _current_trace_id(subject) - stage_label = _SEARCH_STAGE_LABELS.get(stage, stage) retained_comment = { "raw": ( "Raw memory candidates returned by retrieval, before API-level " @@ -300,7 +310,7 @@ def _on_search_results( # A disappearing bucket still has an empty result at the next stage. for key, prior in previous.items(): - buckets.setdefault(key, replace(prior.bucket, memories=[])) + buckets.setdefault(key, replace(prior.bucket, memories=[], identities=())) current = { key: _SearchList( @@ -312,13 +322,60 @@ def _on_search_results( for key, bucket in buckets.items() if bucket.memories or key in previous } + # Keep capture-side correlation independent of writer progress and dropped events. + with self._search_stages_lock: + if stage != "rerank": + self._search_stages[trace_id] = current + + query = ( + self._search_query_value(subject, request, identity_only=True) + if stage == "raw" + else None + ) + event = self._event( + subject, + operation=operation, + category=category, + comment=comment, + metadata=metadata, + ) + work = DeferredTraceEvent( + operation=operation, + build=partial( + self._build_search_event, + event, + stage=stage, + query=query, + previous=previous, + current=current, + ), + ) + if self._defer_search is None: + self._emit(work.build()) + else: + try: + self._defer_search(work) + except Exception: + logger.exception("Failed to enqueue smartcomment event: %s", operation) + + def _build_search_event( + self, + event: TraceEvent, + *, + stage: str, + query: TraceValue | None, + previous: dict[_BucketKey, _SearchList], + current: dict[_BucketKey, _SearchList], + ) -> TraceEvent: + """Assemble links and count removals without touching live request state.""" + stage_label = _SEARCH_STAGE_LABELS.get(stage, stage) outputs = tuple(result.node for result in current.values()) filtered_count = 0 if stage == "raw": - query = self._search_query_value(subject, request, identity_only=True) + assert query is not None inputs = (query,) branches, links = self._search_retrieval_branches( - subject, query=query, results=tuple(current.values()) + event, query=query, results=tuple(current.values()) ) outputs = branches + outputs else: @@ -337,34 +394,22 @@ def _on_search_results( ) ) filtered_count += self._removed_memory_count( - prior.bucket.memories, result.bucket.memories + prior.bucket.identities, result.bucket.identities ) links = tuple(stage_links) inputs = self._unique_values([link.source for link in links]) - with self._search_stages_lock: - self._search_stages[trace_id] = current - - event_metadata = { - **metadata, - "link_strategy": "bucket_lists", - "filtered_count": filtered_count, - } - self._emit( - self._event( - subject, - operation=operation, - category=category, - comment=comment, - inputs=inputs, - outputs=outputs, - links=links, - metadata=event_metadata, - ) + return replace( + event, + inputs=inputs, + outputs=outputs, + links=links, + metadata={ + **event.metadata, + "link_strategy": "bucket_lists", + "filtered_count": filtered_count, + }, ) - # Rerank and failure are terminal stages for this trace's cache. - if stage == "rerank": - self._clear_search_stages(subject) def _search_retrieval_branches( self, @@ -439,8 +484,7 @@ def _search_retrieval_branches( ) return tuple(branches), tuple(links) - @staticmethod - def _search_result_buckets(results: Any) -> dict[_BucketKey, _SearchBucket]: + def _search_result_buckets(self, results: Any) -> dict[_BucketKey, _SearchBucket]: if not isinstance(results, Mapping): return {} @@ -458,6 +502,7 @@ def _search_result_buckets(results: Any) -> dict[_BucketKey, _SearchBucket]: if not isinstance(memories, list | tuple): continue items = [] + identities = [] for memory in memories: metadata = _get(memory, "metadata", {}) # Zero is a valid score; only None falls through to the next field. @@ -466,11 +511,16 @@ def _search_result_buckets(results: Any) -> dict[_BucketKey, _SearchBucket]: score = _get(metadata, "score") if score is None: score = _get(memory, "score") + memory_id = _get(memory, "id") or _get(memory, "memory_id") + text = _memory_text(memory) + identities.append(self._memory_identity(memory_id, text)) items.append( { - "memory_id": _get(memory, "id") or _get(memory, "memory_id"), - "memory": _memory_text(memory), - "score": score, + # Known fields need no recursive dictionary walk. Scalars are + # cheap; unusual structured values still use normal redaction. + "memory_id": self._snapshot(memory_id), + "memory": self._snapshot(text), + "score": self._snapshot(score), } ) bucket = _SearchBucket( @@ -478,12 +528,37 @@ def _search_result_buckets(results: Any) -> dict[_BucketKey, _SearchBucket]: cube_id=cube_id, bucket_index=bucket_index, occurrence=occurrence, - # Count anonymous removals before display truncation. - memories=deepcopy(items), + memories=items, + identities=tuple(identities), ) buckets[bucket.key] = bucket return buckets + def _memory_identity(self, memory_id: Any, text: Any) -> _MemoryKey: + """Capture matching keys before truncation without retaining unbounded text.""" + if memory_id is not None: + kind, value = "id", str(memory_id) + elif isinstance(text, str): + kind, value = "text", text + else: + kind = "json" + try: + value = json.dumps(text, ensure_ascii=False, sort_keys=True, default=str) + except (TypeError, ValueError, RecursionError): + # Cycles and mixed mapping keys must not prevent safe result capture. + value = json.dumps(self._snapshot(text), ensure_ascii=False, sort_keys=True) + # Preserve JSON's default=str equality with an actual string observation. + if value.startswith('"'): + kind, value = "text", json.loads(value) + # Structured values may contain secrets; retain only a digest of their identity. + if ( + kind == "json" + or len(value) > self._max_value_chars + or (memory_id is not None and not isinstance(memory_id, str | int | float | bool)) + ): + return kind, sha256(value.encode("utf-8", errors="surrogatepass")).digest() + return kind, value + def _search_memory_list_value( self, subject: Any, @@ -506,8 +581,8 @@ def _search_memory_list_value( } return TraceValue( - # Bound each entry's fields without truncating the outer result list. - value=[self._snapshot(memory) for memory in bucket.memories], + # Captured entries are already detached, redacted and bounded. + value=bucket.memories, identity=f"search_memory_list:{_current_trace_id(subject)}:{stage}:{bucket_key}", category=f"search_memory_list_{stage}", class_name="search_memory_list", @@ -516,21 +591,14 @@ def _search_memory_list_value( ) @staticmethod - def _removed_memory_count(previous: list[dict[str, Any]], current: list[dict[str, Any]]) -> int: + def _removed_memory_count( + previous: tuple[_MemoryKey, ...], current: tuple[_MemoryKey, ...] + ) -> int: """Count removed occurrences without creating a discarded-memory list.""" - def identity(memory: dict[str, Any]) -> str: - memory_id = memory["memory_id"] - return json.dumps( - ["id", str(memory_id)] if memory_id is not None else ["text", memory["memory"]], - ensure_ascii=False, - sort_keys=True, - default=str, - ) - # Scores and expanded text can change while the same memory is retained. - before = Counter(identity(memory) for memory in previous) - after = Counter(identity(memory) for memory in current) + before = Counter(previous) + after = Counter(current) return sum((before - after).values()) # Search completion diff --git a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py index 472511dd8..e596929b1 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py @@ -39,6 +39,7 @@ def on_load(self) -> None: self.handlers = HookHandlers( self.adapter.submit, max_value_chars=settings.max_value_chars, + defer_search=self.adapter.submit, ) # start the adapter, which will begin processing trace events asynchronously. self.adapter.start() @@ -81,10 +82,12 @@ def on_shutdown(self) -> None: stats = getattr(self.adapter, "stats", None) if stats is not None: logger.info( - "smartcomment plugin stopped; accepted=%d processed=%d dropped=%d failed=%d", + "smartcomment plugin stopped; accepted=%d processed=%d dropped=%d " + "failed=%d write_failures=%d", stats.accepted, stats.processed, stats.dropped, stats.failed, + stats.write_failures, ) self.context = {"shared": {}, "configs": {}} diff --git a/apps/memtrace-memos-integration/memos_smartcomment/storage.py b/apps/memtrace-memos-integration/memos_smartcomment/storage.py index c35062987..e30af4648 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/storage.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/storage.py @@ -22,6 +22,8 @@ if TYPE_CHECKING: + from collections.abc import Sequence + from memos_smartcomment.config import SmartCommentSettings from memos_smartcomment.events import TraceEvent @@ -114,6 +116,7 @@ def __init__(self, settings: SmartCommentSettings) -> None: self.project_id = settings.project_id self._lock = threading.RLock() self._catalogs: OrderedDict[Path, dict[str, dict[str, Any]]] = OrderedDict() + self._dirty_users: dict[Path, str | None] = {} def _user_directory(self, user_id: str | None, *, create: bool = False) -> Path | None: digest = _digest(user_id) @@ -233,8 +236,9 @@ def _allocate(self, user: Path, event: TraceEvent) -> dict[str, Any]: _atomic_write(path / "meta.json", entry) return entry - def write(self, event: TraceEvent, graph: dict[str, Any]) -> None: - """Commit the graph atomically, then publish metadata and navigable indexes.""" + def write(self, events: Sequence[TraceEvent], graph: dict[str, Any]) -> None: + """Commit one trace snapshot and all of its pending metadata; defer indexes.""" + event = events[0] with self._lock: user = self._user_directory(event.user_id, create=True) entries = self._catalog(user, event.user_id) @@ -244,16 +248,39 @@ def write(self, event: TraceEvent, graph: dict[str, Any]) -> None: if entry["project_id"] != self.project_id: raise MemOSError("Trace directory belongs to a different project") path = self._path(user, entry["path"]) - entry["last_seen_at"] = max((entry["last_seen_at"], event.created_at), key=_timestamp) - entry["summary"] = entry["summary"] or _summary(event) - entry["session_id"] = entry["session_id"] or event.session_id - entry["task_id"] = entry["task_id"] or event.task_id - entry["cube_ids"] = sorted(set(entry["cube_ids"]) | set(event.cube_ids)) - entry["operations"] = sorted(set(entry["operations"]) | {event.operation}) + for observation in events: + entry["last_seen_at"] = max( + (entry["last_seen_at"], observation.created_at), key=_timestamp + ) + entry["summary"] = entry["summary"] or _summary(observation) + entry["session_id"] = entry["session_id"] or observation.session_id + entry["task_id"] = entry["task_id"] or observation.task_id + entry["cube_ids"] = sorted(set(entry["cube_ids"]) | set(observation.cube_ids)) + entry["operations"] = sorted(set(entry["operations"]) | {observation.operation}) _atomic_write(path, graph) _atomic_write(path.with_name("meta.json"), entry) entries[event.trace_id] = entry - self._publish(user, event.user_id, entries) + self._dirty_users[user] = event.user_id + + @property + def pending_indexes(self) -> bool: + """Whether committed graphs still need index publication.""" + with self._lock: + return bool(self._dirty_users) + + def publish_indexes(self) -> int: + """Refresh each dirty user's indexes once; retain failures for retry.""" + failed = 0 + with self._lock: + for user, user_id in tuple(self._dirty_users.items()): + try: + self._publish(user, user_id, self._catalog(user, user_id)) + except Exception: + failed += 1 + logger.exception("Failed to publish memtrace indexes: %s", user) + else: + del self._dirty_users[user] + return failed def _publish(self, user: Path, user_id: str | None, entries: dict[str, dict[str, Any]]) -> None: committed = { From 574b9e491e22df3be29acd2ccfb7119482deb8c8 Mon Sep 17 00:00:00 2001 From: jiangchen Date: Tue, 10 Nov 2026 20:32:29 +0800 Subject: [PATCH 18/22] fix(smartcomment): isolate search state and harden lifecycle Keep deferred Search assembly and persistence retries while isolating request state and graph identities. Recover empty reservation directories, validate numeric settings, align lineage defaults, and close the writer after application components. Add focused regression coverage for concurrent requests, snapshot preservation, storage recovery and application shutdown. Verified with 133 related tests, scoped make format and pre-commit checks. --- apps/memtrace-memos-integration/README.md | 57 +- .../memos_smartcomment/CONVENTIONS.md | 16 +- .../memos_smartcomment/config.py | 37 +- .../handlers/search_handler.py | 71 ++- .../memos_smartcomment/plugin.py | 23 +- .../memos_smartcomment/storage.py | 16 +- .../plugins/test_smartcomment_integration.py | 581 ++++++++++++++++++ 7 files changed, 742 insertions(+), 59 deletions(-) create mode 100644 tests/plugins/test_smartcomment_integration.py diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md index 991bb091f..6fa2eb6e2 100644 --- a/apps/memtrace-memos-integration/README.md +++ b/apps/memtrace-memos-integration/README.md @@ -136,6 +136,18 @@ Set these environment variables before loading the plugin. Restart MemOS after c Snapshots are written atomically. An idle or full in-memory cache can be evicted; a later event restores the saved graph and appends to it. Cache limits do not delete JSON files. Use a different output directory for each writer process, including when running multiple server workers. +Queue, string and graph-cache limits must be positive integers. Cache TTL accepts +positive finite seconds, including fractions. Invalid numeric environment values +fall back to the defaults above; invalid values passed directly to +`SmartCommentSettings` raise `ConfigurationError`. Both configuration entry points +leave Search-to-Add history linking disabled unless explicitly enabled. + +The plugin closes its writer after application components finish shutting down, +including when their cleanup raises an exception. Close waits up to five seconds +for accepted events and pending indexes. A timeout logs a warning and leaves the +writer running; persistent storage failures can prevent draining, and process exit +can still lose uncommitted events. + ### Connect Search results to their Add history ```bash @@ -334,7 +346,10 @@ separate list nodes or edges. `filtered_count` still counts removed occurrences buckets by memory ID (or original text for anonymous entries), including duplicates. Reordering, expanded text, and score changes for the same ID do not count as removals. -Branches and lists use `trace_id` for stage connections. List links +Branches and lists use a namespace allocated for each observed Search, so requests +sharing a user and trace cannot reuse each other's stage nodes. Query identities +include a digest of the captured value to survive the API's request copy; equal +captured queries within a trace may share a query node. List links use result type, cube ID, and the occurrence of that cube within the type; buckets without IDs use their position. Reordered unique cube IDs remain correctly linked. Repeated or missing cube IDs rely on stable occurrence order or position. A new @@ -343,9 +358,13 @@ bucket without a previous-stage list is retained without a guessed source edge. The plugin copies `HookContext.operation_id` unchanged into event, operation, and edge metadata. Optional Add-history links retain the rerank Hook's operation ID. -Search observation ends at `search.results.after_rerank`. Each trace caches only -its latest recorded lists; skipped stages leave that cache intact. Rerank and -post-processing failure clear that trace's cache; shutdown clears all caches. The current rerank event describes +Search observation ends at `search.results.after_rerank`. Stage state is keyed by +user, trace and the working request object, which is retained until cleanup. Each +request caches its latest recorded lists; skipped stages leave that state intact. +Rerank, post-processing failure and capture failure release that request's state; +shutdown clears all state. There is no capacity or time-based eviction of pending +Search state, so missing terminal Hooks can retain it until shutdown. +The current rerank event describes the API's descending `metadata.relativity` sort; the tracing handler preserves observed order and does not sort results itself. @@ -432,28 +451,24 @@ operation's data contract; unknown lineage keeps its nodes without a guessed edg ## Verify the integration -Run the Search-to-Add lineage regression tests from this directory: - -```bash -PYTHONPATH=. python -m pytest tests/test_search_lineage.py -q -``` - -They exercise actual Add/Search hooks, final-hit selection, shared ancestors, -cross-trace refinement, user/cube isolation, opt-in behavior, and durable restore. - -Run the local search-branch regression tests (ignored by Git) from this directory -with the integration and its dependencies installed: +With the integration and its dependencies installed, run the tracked regression +tests from the MemOS repository root. Explicit source paths select this checkout: ```bash -PYTHONPATH=. python -m pytest tests/handlers/test_search_handler.py -q +PYTHONPATH=apps/memtrace-memos-integration:src python -m pytest \ + tests/plugins/test_smartcomment_integration.py \ + tests/plugins/test_hook_context.py tests/api/test_search_pipeline_hooks.py -q ``` -These tests cover populated and empty branches, list size independent of graph size, -filtered contents including duplicates, cube identity, operation isolation, and -exported list continuity through post-processing in strict and non-strict modes. Additional plugin regression tests are maintained locally. +These tests cover request/user isolation, concurrent Search API handlers, detached +snapshots, removed counts, Add-history linking after restart, configuration, +reservation recovery and application shutdown with pending writes. They use local +temporary storage without running databases or model services. For an installed integration, follow the Quick start package and entry-point check, enable the plugin, and run a memory operation. Inspect the resulting JSON graph's `graph_id`, `data.nodes`, and `data.edges`, and check the service logs for writer failures or dropped -events. Shutdown logs report accepted, processed, dropped, and failed event counts; -queue acceptance alone does not confirm persistence. +events. Shutdown logs report accepted, processed, dropped, failed and write-failure +counts. `processed` advances after graph and metadata commit; `flush()` also waits +for indexes. Queue acceptance alone does not confirm persistence, and a successful +flush does not undo earlier dropped or failed events. diff --git a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md index dbb9d2d3c..34c3158ab 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md +++ b/apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md @@ -90,7 +90,7 @@ cube 列表取第一个非空字段:`writable_cube_ids → readable_cube_ids ## 2. `TraceValue` 节点总表 -下面的 `{cube}` 在 Add 链中表示 `cube_id or 'unknown'`;`{trace}` 为解析后的 trace,也是 Search 各阶段的关联依据。`{status}` 在调度链中为 `after` 或 `failed`。 +下面的 `{cube}` 在 Add 链中表示 `cube_id or 'unknown'`;`{trace}` 为解析后的 trace。Search 的 `{search}` 为 `{trace}:{uuid4.hex}`,在首次观察结果时分配,由同一请求的后续阶段复用。`{status}` 在调度链中为 `after` 或 `failed`。 ### 2.1 AddHandler:输入、提取、持久化与后台增强 @@ -125,10 +125,12 @@ Recorder 对导出的 `persisted_memory.metadata` 额外添加 `snapshot_status= | 业务含义 | `category` | `class_name` | `identity` | `value` | | --- | --- | --- | --- | --- | -| 搜索查询 | `search_query` | `str` | `search_query:{trace}` | 请求 query;无字段时默认空字符串 | -| 召回类型分支 | `search_memory_type` | `search_memory_type` | `search_memory_type:{trace}:{result_type}` | results 顶层类型字符串,例如 `text_mem`、`tool_mem` | -| 召回 cube 分支 | `search_cube` | `search_cube` | `search_cube:{trace}:{result_type}:{bucket_index}` | cube ID;缺失时为 `bucket-{bucket_index}` | -| 阶段结果列表 | `search_memory_list_{stage}` | `search_memory_list` | `search_memory_list:{trace}:{stage}:{bucket_key}` | 当前桶的记忆列表,每项包含 `memory_id`、`memory`、`score` | +| 搜索查询 | `search_query` | `str` | `search_query:{trace}:{query_digest}` | 请求 query;无字段时默认空字符串 | +| 召回类型分支 | `search_memory_type` | `search_memory_type` | `search_memory_type:{search}:{result_type}` | results 顶层类型字符串,例如 `text_mem`、`tool_mem` | +| 召回 cube 分支 | `search_cube` | `search_cube` | `search_cube:{search}:{result_type}:{bucket_index}` | cube ID;缺失时为 `bucket-{bucket_index}` | +| 阶段结果列表 | `search_memory_list_{stage}` | `search_memory_list` | `search_memory_list:{search}:{stage}:{bucket_key}` | 当前桶的记忆列表,每项包含 `memory_id`、`memory`、`score` | + +`query_digest` 是捕获后 query 值的 JSON 编码的 SHA256 摘要,使 `search.before` 的原请求与结果 Hook 的工作副本引用同一查询。同一用户/trace 内相同的查询快照可以共享查询节点,结果阶段的命名空间仍按请求独立。内存阶段状态按 `(user_id, trace_id, id(request))` 隔离,并持有请求对象,防止对象 ID 在状态存续期间被复用;后台事件只持有已捕获的值和命名空间,不持有请求。rerank、后处理异常和捕获异常清理对应状态,shutdown 清理全部状态。本实现不按容量或 TTL 淘汰待完成 Search,缺失终止 Hook 的状态可能保留至关闭。 | 搜索身份/位置字段 | 规则 | | --- | --- | @@ -326,9 +328,9 @@ API 后处理当前始终调用 `rerank_knowledge_mem`,因此收到 rerank Hoo 四个结果阶段另加 `link_strategy="bucket_lists"` 和 `filtered_count`。raw 阶段 filtered_count 为 0;其余阶段为相对最近记录阶段、所有桶移除的条目数之和。rerank 的 algorithm/order_by/direction 是插件当前固定写入的描述,不是对实际后端算法的动态识别;插件也不自己重新过滤、去重或排序。`top_k` 是当前 Hook 请求中的值,可能已经被业务层扩充。 -缓存以 trace_id 为键,只保存该 trace 最近实际记录阶段的列表快照,按 bucket_key 索引。已记录阶段替换前一份快照;关闭的阶段不触碰缓存,raw 开始时丢弃该 trace 的旧快照;rerank 和 post_process.failed 清理该 trace 的缓存,shutdown 清理全部。 +缓存按第 2.2 节的用户、trace 和请求对象隔离,只保存该请求最近实际记录阶段的列表快照,按 bucket_key 索引。已记录阶段替换前一份快照;关闭的阶段不触碰缓存;raw 开始时重建该请求的状态和结果命名空间,rerank 和 post_process.failed 清理该请求的缓存,shutdown 清理全部。 -各阶段事件使用对应 HookContext.operation_id,同一事件中的所有桶和连线共用该 ID。节点身份和缓存使用 trace_id;插件只消费 Hook 已提供的上下文。 +各阶段事件使用对应 HookContext.operation_id,同一事件中的所有桶和连线共用该 ID。结果节点使用该请求的命名空间,事件和图仍保留原 trace_id;插件不修改业务请求或 Hook 上下文。 观测终点为 `search.results.after_rerank`;插件没有订阅 `search.context.render` 或 `search.after`。 diff --git a/apps/memtrace-memos-integration/memos_smartcomment/config.py b/apps/memtrace-memos-integration/memos_smartcomment/config.py index ca4738bb1..64f563c78 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/config.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/config.py @@ -1,10 +1,13 @@ from __future__ import annotations +import math import os from dataclasses import dataclass from pathlib import Path +from memos.exceptions import ConfigurationError + def _positive_int(name: str, default: int) -> int: raw = os.getenv(name) @@ -24,6 +27,17 @@ def _as_bool(name: str, default: bool = False) -> bool: return raw.strip().lower() in {"1", "true", "yes", "on"} +def _positive_float(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + value = float(raw) + except ValueError: + return default + return value if math.isfinite(value) and value > 0 else default + + @dataclass(frozen=True, slots=True) class SmartCommentSettings: """Runtime settings owned by the external tracing plugin.""" @@ -39,10 +53,29 @@ class SmartCommentSettings: max_cached_graph_items: int = 50_000 cache_ttl_seconds: float = 300 # Import only the persisted ancestors of final Search results. - link_search_to_add: bool = True + link_search_to_add: bool = False # "local" follows the service machine; an IANA timezone overrides it. output_timezone: str = "local" + def __post_init__(self) -> None: + # Direct callers must not accidentally disable queue or cache bounds. + for name in ( + "queue_size", + "max_value_chars", + "max_cached_traces", + "max_cached_graph_items", + ): + value = getattr(self, name) + if type(value) is not int or value <= 0: + raise ConfigurationError(f"{name} must be a positive integer") + ttl = self.cache_ttl_seconds + if ( + type(ttl) not in (int, float) + or ttl <= 0 + or (isinstance(ttl, float) and not math.isfinite(ttl)) + ): + raise ConfigurationError("cache_ttl_seconds must be positive and finite") + @classmethod def from_env(cls) -> SmartCommentSettings: return cls( @@ -55,7 +88,7 @@ def from_env(cls) -> SmartCommentSettings: max_cached_graph_items=_positive_int( "MEMOS_SMARTCOMMENT_MAX_CACHED_GRAPH_ITEMS", 50_000 ), - cache_ttl_seconds=_positive_int("MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS", 300), + cache_ttl_seconds=_positive_float("MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS", 300), link_search_to_add=_as_bool("MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD"), output_timezone=os.getenv("MEMOS_SMARTCOMMENT_OUTPUT_TIMEZONE", "local"), ) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py index 4e5f072e6..6156156b8 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/handlers/search_handler.py @@ -12,6 +12,7 @@ from threading import Lock from typing import TYPE_CHECKING, Any from urllib.parse import quote +from uuid import uuid4 from memos_smartcomment.events import DeferredTraceEvent, TraceEvent, TraceLink, TraceValue from memos_smartcomment.handlers.base import ( @@ -77,6 +78,14 @@ class _SearchList: node: TraceValue +@dataclass(frozen=True) +class _SearchState: + # Retain the request so its object ID cannot be reused while stages are pending. + request: Any + namespace: str + lists: dict[_BucketKey, _SearchList] + + class SearchHandler(BaseHandler): """Capture one result-list node per type/cube bucket at each enabled Search stage.""" @@ -89,7 +98,7 @@ def __init__( ) -> None: super().__init__(submit, max_value_chars=max_value_chars) self._defer_search = defer_search - self._search_stages: dict[str, dict[_BucketKey, _SearchList]] = {} + self._search_stages: dict[tuple[str | None, str, int], _SearchState] = {} self._search_stages_lock = Lock() def clear(self) -> None: @@ -144,7 +153,7 @@ def _search_query_value( identity_only: bool, ) -> TraceValue: trace_id = _current_trace_id(subject) - return self._value( + query = self._value( value=_get(request, "query", ""), identity=f"search_query:{trace_id}", category="search_query", @@ -153,6 +162,10 @@ def _search_query_value( comment="The raw query the user uploaded for memory retrieval.", metadata={"memos_stage": "search.before"}, ) + # Before receives the original request; result Hooks receive its working copy. + # Match the captured query across that copy without borrowing another query's value. + digest = sha256(json.dumps(query.value, ensure_ascii=False).encode()).hexdigest() + return replace(query, identity=f"{query.identity}:{digest}") # Retrieved candidates # Retrieval returns candidates grouped by memory type and storage cube. @@ -277,8 +290,13 @@ def _on_search_results( if request is None: return subject = self._search_subject(hook_context, request) + request_key = self._search_cache_key(subject, request) + with self._search_stages_lock: + # Take ownership before capture so failures cannot strand this request's state. + state = self._search_stages.pop(request_key, None) + if state is None or stage == "raw": + state = _SearchState(request, f"{_current_trace_id(subject)}:{uuid4().hex}", {}) buckets = self._search_result_buckets(results) - trace_id = _current_trace_id(subject) retained_comment = { "raw": ( @@ -302,11 +320,8 @@ def _on_search_results( stage, f"Memory candidates retained after the {stage} stage.", ) - with self._search_stages_lock: - # Skipped or missing stages leave the last observed lists as predecessors. - previous = self._search_stages.pop(trace_id, {}) - if stage == "raw": - previous = {} + # Skipped or missing stages leave the last observed lists as predecessors. + previous = state.lists # A disappearing bucket still has an empty result at the next stage. for key, prior in previous.items(): @@ -316,17 +331,12 @@ def _on_search_results( key: _SearchList( bucket, self._search_memory_list_value( - subject, stage=stage, bucket=bucket, retained_comment=retained_comment + state.namespace, stage=stage, bucket=bucket, retained_comment=retained_comment ), ) for key, bucket in buckets.items() if bucket.memories or key in previous } - # Keep capture-side correlation independent of writer progress and dropped events. - with self._search_stages_lock: - if stage != "rerank": - self._search_stages[trace_id] = current - query = ( self._search_query_value(subject, request, identity_only=True) if stage == "raw" @@ -344,6 +354,7 @@ def _on_search_results( build=partial( self._build_search_event, event, + namespace=state.namespace, stage=stage, query=query, previous=previous, @@ -357,11 +368,18 @@ def _on_search_results( self._defer_search(work) except Exception: logger.exception("Failed to enqueue smartcomment event: %s", operation) + return + + # Only completed captures become predecessors; writer progress is independent. + with self._search_stages_lock: + if stage != "rerank": + self._search_stages[request_key] = _SearchState(request, state.namespace, current) def _build_search_event( self, event: TraceEvent, *, + namespace: str, stage: str, query: TraceValue | None, previous: dict[_BucketKey, _SearchList], @@ -375,7 +393,7 @@ def _build_search_event( assert query is not None inputs = (query,) branches, links = self._search_retrieval_branches( - event, query=query, results=tuple(current.values()) + namespace, query=query, results=tuple(current.values()) ) outputs = branches + outputs else: @@ -413,13 +431,12 @@ def _build_search_event( def _search_retrieval_branches( self, - subject: Any, + namespace: str, *, query: TraceValue, results: tuple[_SearchList, ...], ) -> tuple[tuple[TraceValue, ...], tuple[TraceLink, ...]]: """Expand raw retrieval into query -> result type -> cube -> memory list.""" - trace_id = _current_trace_id(subject) types: dict[str, TraceValue] = {} branches: list[TraceValue] = [] links: list[TraceLink] = [] @@ -429,7 +446,7 @@ def _search_retrieval_branches( if result_type not in types: type_node = self._value( value=result_type, - identity=f"search_memory_type:{trace_id}:{result_type}", + identity=f"search_memory_type:{namespace}:{result_type}", category="search_memory_type", class_name="search_memory_type", identity_only=False, @@ -451,7 +468,7 @@ def _search_retrieval_branches( cube_label = cube_id if cube_id is not None else f"bucket-{bucket.bucket_index}" cube_node = self._value( value=cube_label, - identity=f"search_cube:{trace_id}:{result_type}:{bucket.bucket_index}", + identity=f"search_cube:{namespace}:{result_type}:{bucket.bucket_index}", category="search_cube", class_name="search_cube", identity_only=False, @@ -561,7 +578,7 @@ def _memory_identity(self, memory_id: Any, text: Any) -> _MemoryKey: def _search_memory_list_value( self, - subject: Any, + namespace: str, *, stage: str, bucket: _SearchBucket, @@ -583,7 +600,7 @@ def _search_memory_list_value( return TraceValue( # Captured entries are already detached, redacted and bounded. value=bucket.memories, - identity=f"search_memory_list:{_current_trace_id(subject)}:{stage}:{bucket_key}", + identity=f"search_memory_list:{namespace}:{stage}:{bucket_key}", category=f"search_memory_list_{stage}", class_name="search_memory_list", comment=retained_comment, @@ -604,7 +621,7 @@ def _removed_memory_count( # Search completion # Reranking completes the search result flow. A post-processing exception # links the query to a failure status. Both outcomes release the cached - # result lists for that trace. + # result lists for that request. def on_search_post_process_failed( self, *, @@ -616,6 +633,7 @@ def on_search_post_process_failed( ) -> None: stage = "search.post_process.failed" subject = self._search_subject(hook_context, search_req) + self._clear_search_stages(subject, search_req) query = self._search_query_value(subject, search_req, identity_only=True) status = self._operation_status_value(subject, error, stage=stage) self._emit( @@ -642,8 +660,11 @@ def on_search_post_process_failed( }, ) ) - self._clear_search_stages(subject) - def _clear_search_stages(self, subject: Any) -> None: + @staticmethod + def _search_cache_key(subject: Any, request: Any) -> tuple[str | None, str, int]: + return _get(subject, "user_id"), _current_trace_id(subject), id(request) + + def _clear_search_stages(self, subject: Any, request: Any) -> None: with self._search_stages_lock: - self._search_stages.pop(_current_trace_id(subject), None) + self._search_stages.pop(self._search_cache_key(subject, request), None) diff --git a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py index e596929b1..2e5d638c9 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/plugin.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/plugin.py @@ -1,7 +1,8 @@ from __future__ import annotations -from collections.abc import Callable -from typing import Any +from collections.abc import AsyncIterator, Callable, Mapping +from contextlib import asynccontextmanager +from typing import TYPE_CHECKING, Any from memos_smartcomment.adapter import AsyncTraceAdapter from memos_smartcomment.config import SmartCommentSettings @@ -14,6 +15,10 @@ from memos.plugins.hook_defs import H +if TYPE_CHECKING: + from fastapi import FastAPI + + logger = get_logger(__name__) AdapterFactory = Callable[[SmartCommentSettings], AsyncTraceAdapter] @@ -74,6 +79,20 @@ def on_load(self) -> None: def init_components(self, context: dict[str, Any]) -> None: self.context = context + def init_app(self) -> None: + original_lifespan = self._app.router.lifespan_context + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[Mapping[str, Any] | None]: + try: + async with original_lifespan(app) as state: + yield state + finally: + # Components may still emit events while shutting down. + self.on_shutdown() + + self._app.router.lifespan_context = lifespan + def on_shutdown(self) -> None: if self.handlers is not None: self.handlers.clear() diff --git a/apps/memtrace-memos-integration/memos_smartcomment/storage.py b/apps/memtrace-memos-integration/memos_smartcomment/storage.py index e30af4648..349e880a1 100644 --- a/apps/memtrace-memos-integration/memos_smartcomment/storage.py +++ b/apps/memtrace-memos-integration/memos_smartcomment/storage.py @@ -10,6 +10,7 @@ import threading from collections import OrderedDict +from contextlib import suppress from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING, Any @@ -65,6 +66,17 @@ def _atomic_write(path: Path, value: dict[str, Any] | str) -> None: temporary.unlink(missing_ok=True) +def _reserve(path: Path, value: dict[str, Any]) -> None: + """Reserve a new identity without consuming a directory name on write failure.""" + try: + _atomic_write(path, value) + except Exception: + # Remove only an empty reservation directory; never discard existing files. + with suppress(OSError): + path.parent.rmdir() + raise + + def _entry_type(event: TraceEvent) -> str: if event.operation.startswith("memos.search."): return "search" @@ -136,7 +148,7 @@ def _user_directory(self, user_id: str | None, *, create: bool = False) -> Path # An earlier colliding user may have been deleted; check longer candidates too. if create: if available is not None: - _atomic_write(available / "_user.json", {"user_id": user_id}) + _reserve(available / "_user.json", {"user_id": user_id}) return available raise MemOSError("Cannot allocate a distinct memtrace user directory") return None @@ -233,7 +245,7 @@ def _allocate(self, user: Path, event: TraceEvent) -> dict[str, Any]: "path": (path / "graph.json").relative_to(user).as_posix(), } # Reserve the identity before writing the graph so a failed write retries in place. - _atomic_write(path / "meta.json", entry) + _reserve(path / "meta.json", entry) return entry def write(self, events: Sequence[TraceEvent], graph: dict[str, Any]) -> None: diff --git a/tests/plugins/test_smartcomment_integration.py b/tests/plugins/test_smartcomment_integration.py new file mode 100644 index 000000000..52e404b86 --- /dev/null +++ b/tests/plugins/test_smartcomment_integration.py @@ -0,0 +1,581 @@ +"""Plugin isolation, persistence recovery and shutdown without external services.""" + +import errno +import json +import threading + +from contextlib import asynccontextmanager, nullcontext +from copy import deepcopy +from dataclasses import asdict + +import pytest + + +pytest.importorskip("memos_smartcomment") +pytest.importorskip("smartcomment") + +from memos_smartcomment import adapter as adapter_module +from memos_smartcomment import storage +from memos_smartcomment.adapter import AsyncTraceAdapter +from memos_smartcomment.config import SmartCommentSettings +from memos_smartcomment.events import DeferredTraceEvent, TraceEvent, TraceValue +from memos_smartcomment.handlers import HookHandlers +from memos_smartcomment.plugin import SmartCommentPlugin + +from memos.exceptions import ConfigurationError + + +def observation(trace="trace-a", user="user-a"): + return TraceEvent( + operation="memos.search.request", + category="search_request", + trace_id=trace, + user_id=user, + outputs=(TraceValue(value="example", identity="query"),), + ) + + +def search_fixture(query="query", user="user-a"): + request = {"trace_id": "shared", "user_id": user, "query": query, "relativity": 0.5} + results = { + "text_mem": [ + {"cube_id": "cube-a", "memories": [{"id": query, "memory": query, "score": 0.0}]} + ] + } + return request, results + + +@pytest.mark.parametrize("deferred", [False, True]) +@pytest.mark.parametrize("terminal", ["rerank", "failure"]) +def test_search_stages_remain_with_their_user(deferred, terminal): + events, pending = [], [] + handler = HookHandlers(events.append, defer_search=pending.append if deferred else None) + first, first_results = search_fixture("first", "user-a") + second, second_results = search_fixture("second", "user-b") + for request, results in ((first, first_results), (second, second_results)): + handler.on_search_memory_results(search_req=request, results=results) + if terminal == "failure": + handler.on_search_post_process_failed( + handler=None, search_req=first, error=ValueError("example") + ) + else: + handler.on_search_results_after_rerank(search_req=first, results=first_results) + handler.on_search_results_after_rerank(search_req=second, results=second_results) + events.extend(work.build() for work in pending) + + reranks = [event for event in events if event.operation == "memos.search.rerank"] + assert len(reranks) == (1 if terminal == "failure" else 2) + for event in reranks: + marker = "first" if event.user_id == "user-a" else "second" + assert len(event.links) == 1 + assert ( + event.links[0].source.value + == event.links[0].target.value + == [{"memory_id": marker, "memory": marker, "score": 0.0}] + ) + assert not handler._search_stages + + +@pytest.mark.parametrize("same_query", [False, True]) +def test_overlapping_requests_keep_distinct_graph_chains(tmp_path, same_query): + adapter = AsyncTraceAdapter(SmartCommentSettings(output_dir=tmp_path, strict=True)) + handler = HookHandlers(adapter.submit, defer_search=adapter.submit) + requests, results = zip( + *(search_fixture(marker) for marker in ("first", "second")), strict=True + ) + if same_query: + requests[1]["query"] = requests[0]["query"] + original = deepcopy((requests, results)) + try: + for request in requests: + handler.on_search_before(request=request) + # MemOS copies requests before retrieval and rebuilds context at each stage. + working = deepcopy(requests) + for request, result in zip(working, results, strict=True): + handler.on_search_memory_results(search_req=request, results=result) + for request, result in zip(working, results, strict=True): + handler.on_search_results_after_rerank(search_req=request, results=result) + assert not handler._search_stages + assert adapter.flush(timeout=5) + assert adapter.stats.processed == 6 + assert adapter.stats.failed == adapter.stats.write_failures == 0 + data = json.loads(adapter.output_path("shared", "user-a").read_text())["data"] + nodes = {node["full_node_id"]: node for node in data["nodes"]} + reranks = [edge for edge in data["edges"] if edge["category"] == "memory_reranked"] + assert len(reranks) == 2 + for edge in reranks: + source, target = ( + nodes[edge[key]] for key in ("source_full_node_id", "target_full_node_id") + ) + assert json.loads(source["value"]) == json.loads(target["value"]) + for marker in ("first", "second"): + raw = next( + node + for node in nodes.values() + if node["category"] == "search_memory_list_raw" + and json.loads(node["value"])[0]["memory"] == marker + ) + current = raw["full_node_id"] + for category in ("memory_retrieval", "memory_cube_branch", "memory_type_branch"): + parents = [ + edge["source_full_node_id"] + for edge in data["edges"] + if edge["category"] == category and edge["target_full_node_id"] == current + ] + assert len(parents) == 1 + current = parents[0] + assert json.loads(nodes[current]["value"]) == ("first" if same_query else marker) + assert (requests, results) == original + finally: + adapter.close() + + +@pytest.mark.parametrize("stage", ["raw", "threshold", "rerank", "failure"]) +@pytest.mark.parametrize("boundary", ["_snapshot", "_event"]) +def test_failed_capture_releases_only_its_request_state(monkeypatch, stage, boundary): + handler = HookHandlers(lambda event: None, defer_search=lambda work: None) + request, results = search_fixture() + other_request, other_results = search_fixture("other") + other_request["trace_id"] = "other-trace" + handler.on_search_memory_results(search_req=request, results=results) + handler.on_search_memory_results(search_req=other_request, results=other_results) + + def fail(*args, **kwargs): + raise ValueError("injected capture failure") + + with monkeypatch.context() as patch: + patch.setattr(handler, boundary, fail) + with pytest.raises(ValueError, match="injected capture failure"): + if stage == "failure": + handler.on_search_post_process_failed( + handler=None, search_req=request, error=ValueError("example") + ) + else: + callback = { + "raw": handler.on_search_memory_results, + "threshold": handler.on_search_results_after_threshold, + "rerank": handler.on_search_results_after_rerank, + }[stage] + callback(search_req=request, results=results) + assert len(handler._search_stages) == 1 + handler.on_search_results_after_rerank(search_req=other_request, results=other_results) + assert not handler._search_stages + + +def test_deferred_events_keep_snapshots_and_removed_counts(): + events, pending = [], [] + handler = HookHandlers(events.append, defer_search=pending.append, max_value_chars=8) + request, results = search_fixture() + memories = results["text_mem"][0]["memories"] + memories[:] = [ + {"id": "kept", "memory": {"text": "before", "api_key": "fixture-secret"}, "score": 0}, + {"memory": "abcdefgh-first"}, + {"memory": "abcdefgh-second"}, + ] + handler.on_search_before(request=request) + handler.on_search_memory_results(search_req=request, results=results) + memories.pop(1) + handler.on_search_results_after_threshold(search_req=request, results=results) + handler.on_search_results_after_rerank(search_req=request, results=results) + memories[0]["memory"]["text"] = "after" + request["trace_id"] = "changed" + assert all(isinstance(work, DeferredTraceEvent) for work in pending) + events.extend(work.build() for work in pending) + assert [event.metadata.get("filtered_count") for event in events] == [None, 0, 1, 0] + assert {event.trace_id for event in events} == {"shared"} + captured = json.dumps([asdict(event) for event in events]) + assert "fixture-secret" not in captured and "" in captured + assert '"text": "before"' in captured and '"text": "after"' not in captured + assert not handler._search_stages + + +@pytest.mark.parametrize("same_user", [False, True]) +@pytest.mark.parametrize("dedup, events_per_request", [("no", 4), ("mmr", 5)]) +def test_plugin_preserves_concurrent_api_search_results( + tmp_path, monkeypatch, clean_hooks, same_user, dedup, events_per_request +): + from concurrent.futures import ThreadPoolExecutor + from types import SimpleNamespace + + from memos.api.handlers.base_handler import HandlerDependencies + from memos.api.handlers.search_handler import SearchHandler + from memos.api.product_models import APISearchRequest + from memos.plugins.hook_defs import H + from memos.plugins.hooks import register_hook + + monkeypatch.setattr("memos.api.handlers.search_handler.get_current_trace_id", lambda: "shared") + handler = SearchHandler( + HandlerDependencies( + naive_mem_cube=object(), + mem_scheduler=object(), + searcher=object(), + deepsearch_agent=object(), + ) + ) + requests = [ + APISearchRequest(user_id="user-a", query="first", relativity=0.5, dedup=dedup, top_k=5), + APISearchRequest( + user_id="user-a" if same_user else "user-b", + query="second", + relativity=0.5, + dedup=dedup, + top_k=5, + ), + ] + original = [request.model_dump() for request in requests] + + def retrieve(request): + _, results = search_fixture(request.query) + results["text_mem"][0]["memories"][0]["metadata"] = { + "relativity": 1.0, + "memory_type": "LongTermMemory", + } + return results + + monkeypatch.setattr( + handler, "_build_cube_view", lambda request: SimpleNamespace(search_memories=retrieve) + ) + expected = [handler.handle_search_memories(request).model_dump() for request in requests] + monkeypatch.setenv("MEMOS_SMARTCOMMENT_OUTPUT_DIR", str(tmp_path)) + monkeypatch.setenv("MEMOS_SMARTCOMMENT_STRICT", "true") + plugin = SmartCommentPlugin() + barrier = threading.Barrier(2) + + def wait_for_both_raw_stages(**kwargs): + barrier.wait(timeout=5) + + try: + plugin.on_load() + register_hook(H.SEARCH_MEMORY_RESULTS, wait_for_both_raw_stages) + with ThreadPoolExecutor(max_workers=2) as pool: + actual = list(pool.map(handler.handle_search_memories, requests)) + assert [response.model_dump() for response in actual] == expected + assert [request.model_dump() for request in requests] == original + assert not plugin.handlers._search_stages + assert plugin.adapter.flush(timeout=5) + assert plugin.adapter.stats.processed == 2 * events_per_request + assert plugin.adapter.stats.failed == plugin.adapter.stats.write_failures == 0 + for user in {request.user_id for request in requests}: + data = json.loads(plugin.adapter.output_path("shared", user).read_text())["data"] + nodes = {node["full_node_id"]: node for node in data["nodes"]} + links = [ + edge + for edge in data["edges"] + if edge["category"] + in {"memory_threshold_passed", "memory_deduplicated", "memory_reranked"} + ] + assert len(links) == (events_per_request - 2) * (2 if same_user else 1) + for edge in links: + assert ( + nodes[edge["source_full_node_id"]]["value"] + == nodes[edge["target_full_node_id"]]["value"] + ) + if not same_user: + foreign = "second" if user == "user-a" else "first" + assert all(foreign not in node["value"] for node in nodes.values()) + finally: + plugin.on_shutdown() + + +@pytest.mark.parametrize("link_history", [False, True]) +def test_add_snapshots_and_search_lineage_survive_restart(tmp_path, link_history): + from memos.plugins.hook_context import HookContext + + settings = SmartCommentSettings(output_dir=tmp_path, link_search_to_add=link_history) + adapter = AsyncTraceAdapter(settings) + handler = HookHandlers(adapter.submit, defer_search=adapter.submit) + context = HookContext(trace_id="add", user_id="user-a", cube_ids=("cube-a",)) + messages = [{"role": "user", "content": "Remember the blue notebook"}] + request = { + "trace_id": "add", + "user_id": "user-a", + "writable_cube_ids": ["cube-a"], + "messages": messages, + } + memory = {"id": "memory-a", "memory": "The notebook is blue"} + try: + handler.on_add_before(request=request) + handler.on_mem_reader_extract_after( + hook_context=context, + scene_data=[messages], + type="chat", + info=None, + mode="fine", + user_name="cube-a", + result=[memory], + ) + handler.on_text_memory_add_after( + hook_context=context, + text_memory=None, + memories=[memory], + kwargs={}, + result=["memory-a"], + ) + handler.on_add_after(request=request) + messages[0]["content"] = "changed after capture" + memory["memory"] = "changed after capture" + assert adapter.flush(timeout=5) + assert adapter.stats.processed == 3 and adapter.stats.failed == 0 + add_path = adapter.output_path("add", "user-a") + add_snapshot = add_path.read_bytes() + finally: + adapter.close() + + reopened = AsyncTraceAdapter(settings) + handler = HookHandlers(reopened.submit, defer_search=reopened.submit) + search, results = search_fixture() + results["text_mem"][0]["memories"] = [ + {"id": "memory-a", "memory": "The notebook is blue", "score": 0.0} + ] + try: + handler.on_search_before(request=search) + handler.on_search_memory_results(search_req=search, results=results) + handler.on_search_results_after_rerank(search_req=search, results=results) + assert reopened.flush(timeout=5) + assert reopened.stats.processed == 3 + assert reopened.stats.failed == reopened.stats.write_failures == 0 + data = json.loads(reopened.output_path("shared", "user-a").read_text())["data"] + lineage = [edge for edge in data["edges"] if edge["category"] == "memory_search_lineage"] + assert len(lineage) == int(link_history) + if link_history: + assert lineage[0]["metadata"]["rank"] == 1 + assert lineage[0]["metadata"]["score"] == 0.0 + assert {edge["category"] for edge in data["edges"]} >= { + "memory_extraction", + "memory_persistence", + } + assert any( + json.loads(node["value"]) == "The notebook is blue" for node in data["nodes"] + ) + assert any( + json.loads(node["value"]) + == [{"role": "user", "content": "Remember the blue notebook"}] + for node in data["nodes"] + ) + assert add_path.read_bytes() == add_snapshot + finally: + reopened.close() + + +@pytest.mark.parametrize("filename", ["_user.json", "meta.json"]) +@pytest.mark.parametrize("reopen", [False, True]) +def test_failed_reservations_recover_without_exhausting_names( + tmp_path, monkeypatch, filename, reopen +): + settings = SmartCommentSettings(output_dir=tmp_path) + store = storage.TraceStore(settings) + event = observation() + original = storage.os.replace + + def disk_full(source, destination): + if destination.name == filename: + raise OSError(errno.ENOSPC, "injected reservation failure") + return original(source, destination) + + with monkeypatch.context() as patch: + patch.setattr(storage.os, "replace", disk_full) + for _ in range(4): + with pytest.raises(OSError, match="injected reservation failure"): + store.write([event], {"version": 1}) + if reopen: + store = storage.TraceStore(settings) + store.write([event], {"version": 2}) + assert store.publish_indexes() == 0 + path = store.output_path(event.trace_id, event.user_id) + assert json.loads(path.read_text()) == {"version": 2} + assert list(json.loads((path.parents[3] / "_index.json").read_text())["traces"]) == [ + event.trace_id + ] + assert not list(tmp_path.rglob("*.tmp")) + + +@pytest.mark.parametrize("filename", ["_user.json", "meta.json"]) +def test_reservation_failure_preserves_other_files(tmp_path, monkeypatch, filename): + store = storage.TraceStore(SmartCommentSettings(output_dir=tmp_path)) + original = storage.os.replace + preserved = [] + + def fail_with_existing_file(source, destination): + if destination.name == filename: + marker = destination.parent / "existing.json" + marker.write_text('{"preserved": true}') + preserved.append(marker) + raise OSError(errno.ENOSPC, "injected reservation failure") + return original(source, destination) + + with monkeypatch.context() as patch: + patch.setattr(storage.os, "replace", fail_with_existing_file) + with pytest.raises(OSError, match="injected reservation failure"): + store.write([observation()], {"version": 1}) + store.write([observation()], {"version": 2}) + assert len(preserved) == 1 + assert json.loads(preserved[0].read_text()) == {"preserved": True} + + +@pytest.mark.parametrize( + "field", ["queue_size", "max_value_chars", "max_cached_traces", "max_cached_graph_items"] +) +@pytest.mark.parametrize("value", [0, -1, 1.5, True, "4", None]) +def test_settings_reject_invalid_integer_limits(field, value): + with pytest.raises(ConfigurationError, match=field): + SmartCommentSettings(**{field: value}) + + +@pytest.mark.parametrize("value", [0, -1, float("nan"), float("inf"), True, "4", None]) +def test_settings_reject_invalid_cache_timeout(value): + with pytest.raises(ConfigurationError, match="cache_ttl_seconds"): + SmartCommentSettings(cache_ttl_seconds=value) + + +def test_settings_defaults_agree_between_entrypoints(monkeypatch): + monkeypatch.delenv("MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD", raising=False) + assert SmartCommentSettings().link_search_to_add is False + assert SmartCommentSettings.from_env().link_search_to_add is False + + +@pytest.mark.parametrize( + "raw, expected", [("0.5", 0.5), ("nan", 300), ("inf", 300), ("0", 300), ("bad", 300)] +) +def test_environment_accepts_finite_positive_fractional_timeout(monkeypatch, raw, expected): + monkeypatch.setenv("MEMOS_SMARTCOMMENT_CACHE_TTL_SECONDS", raw) + assert SmartCommentSettings.from_env().cache_ttl_seconds == expected + + +@pytest.mark.parametrize("failure", [None, "startup", "cleanup"]) +def test_application_lifespan_preserves_state_and_closes_writer( + tmp_path, monkeypatch, clean_hooks, failure +): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + monkeypatch.setenv("MEMOS_SMARTCOMMENT_OUTPUT_DIR", str(tmp_path)) + plugin = SmartCommentPlugin() + plugin.on_load() + adapter = plugin.adapter + request, results = search_fixture() + plugin.handlers.on_search_memory_results(search_req=request, results=results) + + @asynccontextmanager + async def original_lifespan(app): + if failure == "startup": + raise ValueError("injected startup failure") + yield {"marker": "original-state"} + # Component shutdown may still emit accepted observations. + assert adapter.submit(observation("cleanup")) + if failure == "cleanup": + raise ValueError("injected cleanup failure") + + app = FastAPI(lifespan=original_lifespan) + plugin._bind_app(app) + plugin.init_app() + expectation = ( + pytest.raises(ValueError, match=f"injected {failure} failure") if failure else nullcontext() + ) + try: + with expectation, TestClient(app) as client: + assert client.app_state["marker"] == "original-state" + assert not adapter._thread.is_alive() + assert not plugin.handlers._search_stages + assert adapter.stats.processed == (1 if failure == "startup" else 2) + assert adapter.stats.failed == adapter.stats.write_failures == 0 + assert not adapter._store.pending_indexes + assert not adapter.submit(observation("late")) + for path in tmp_path.glob("*/_index.json"): + assert json.loads(path.read_text())["traces"] + finally: + adapter.close() + + +@pytest.mark.parametrize("filename", ["_user.json", "meta.json"]) +def test_application_shutdown_drains_backlog_after_reservation_retry( + tmp_path, monkeypatch, clean_hooks, filename +): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + monkeypatch.setenv("MEMOS_SMARTCOMMENT_OUTPUT_DIR", str(tmp_path)) + monkeypatch.setattr(adapter_module, "_RETRY_SECONDS", 0.01) + plugin = SmartCommentPlugin() + plugin.on_load() + adapter = plugin.adapter + original = storage.os.replace + blocked, attempts = threading.Event(), threading.Event() + blocked.set() + failures = 0 + + def disk_full(source, destination): + nonlocal failures + if destination.name == filename and blocked.is_set(): + failures += 1 + if failures >= 4: + attempts.set() + raise OSError(errno.ENOSPC, "injected reservation failure") + return original(source, destination) + + @asynccontextmanager + async def original_lifespan(app): + yield + blocked.clear() + + monkeypatch.setattr(storage.os, "replace", disk_full) + app = FastAPI(lifespan=original_lifespan) + plugin._bind_app(app) + plugin.init_app() + try: + with TestClient(app): + for _ in range(4): + assert adapter.submit(observation()) + assert attempts.wait(2) + assert adapter.stats.processed == 0 + assert not adapter._thread.is_alive() + assert adapter.stats.processed == 4 + assert adapter.stats.write_failures >= 4 + assert adapter.stats.failed == adapter.stats.dropped == 0 + assert not adapter._pending and not adapter._store.pending_indexes + path = adapter.output_path("trace-a", "user-a") + assert len(json.loads(path.read_text())["data"]["operations"]) == 4 + assert "trace-a" in json.loads((path.parents[3] / "_index.json").read_text())["traces"] + finally: + blocked.clear() + adapter.close() + + +@pytest.mark.parametrize("filename", ["graph.json", "_index.json", "index.md"]) +def test_close_timeout_keeps_retry_work_until_storage_recovers(tmp_path, monkeypatch, filename): + monkeypatch.setattr(adapter_module, "_RETRY_SECONDS", 0.01) + adapter = AsyncTraceAdapter(SmartCommentSettings(output_dir=tmp_path)) + original = storage.os.replace + blocked, attempted = threading.Event(), threading.Event() + blocked.set() + + def disk_full(source, destination): + if destination.name == filename and blocked.is_set(): + attempted.set() + raise OSError(errno.ENOSPC, "injected storage failure") + return original(source, destination) + + monkeypatch.setattr(storage.os, "replace", disk_full) + try: + assert adapter.submit(observation()) + assert not adapter.flush(timeout=0.05) + assert attempted.wait(2) + adapter.close(timeout=0.01) + assert adapter._thread.is_alive() + assert adapter.stats.write_failures >= 1 + assert adapter.stats.failed == adapter.stats.dropped == 0 + assert not adapter.submit(observation("late")) + blocked.clear() + assert adapter.flush(timeout=5) + adapter.close() + assert not adapter._thread.is_alive() + assert adapter.stats.processed == 1 + assert not adapter._pending and not adapter._store.pending_indexes + path = adapter.output_path("trace-a", "user-a") + assert len(json.loads(path.read_text())["data"]["operations"]) == 1 + assert list(json.loads((path.parents[3] / "_index.json").read_text())["traces"]) == [ + "trace-a" + ] + assert (path.parents[3] / "index.md").is_file() + finally: + blocked.clear() + adapter.close() From 7b6140a607561a5b83dbc2d976c4010367eeb7c5 Mon Sep 17 00:00:00 2001 From: qzs Date: Tue, 17 Nov 2026 16:36:41 +0800 Subject: [PATCH 19/22] chore(smartcomment): add regression tests and refresh user docs - Add tests for Add/Search handlers, lineage, recording, and serialization - Cover queue limits and trace recovery after restart - Update ignore rules to track integration tests - Consolidate bilingual user guides and remove superseded documentation --- apps/memtrace-memos-integration/.gitignore | 2 - apps/memtrace-memos-integration/FEATURES.md | 242 -------- .../README.changes.md | 97 --- apps/memtrace-memos-integration/README.md | 553 +++++------------- apps/memtrace-memos-integration/README_zh.md | 223 +++++++ .../memos_smartcomment/CONVENTIONS.md | 439 -------------- .../tests/handlers/test_add_handler.py | 128 ++++ .../tests/handlers/test_search_handler.py | 91 +++ .../tests/test_adapter.py | 80 +++ .../tests/test_lineage.py | 91 +++ .../tests/test_recorder.py | 88 +++ .../tests/test_serialization.py | 27 + 12 files changed, 879 insertions(+), 1182 deletions(-) delete mode 100644 apps/memtrace-memos-integration/FEATURES.md delete mode 100644 apps/memtrace-memos-integration/README.changes.md create mode 100644 apps/memtrace-memos-integration/README_zh.md delete mode 100644 apps/memtrace-memos-integration/memos_smartcomment/CONVENTIONS.md create mode 100644 apps/memtrace-memos-integration/tests/handlers/test_add_handler.py create mode 100644 apps/memtrace-memos-integration/tests/handlers/test_search_handler.py create mode 100644 apps/memtrace-memos-integration/tests/test_adapter.py create mode 100644 apps/memtrace-memos-integration/tests/test_lineage.py create mode 100644 apps/memtrace-memos-integration/tests/test_recorder.py create mode 100644 apps/memtrace-memos-integration/tests/test_serialization.py diff --git a/apps/memtrace-memos-integration/.gitignore b/apps/memtrace-memos-integration/.gitignore index 2739346ef..fd869f6c4 100644 --- a/apps/memtrace-memos-integration/.gitignore +++ b/apps/memtrace-memos-integration/.gitignore @@ -16,8 +16,6 @@ __pycache__/ *.py[cod] # Tests and tool output -tests -test_*.py *_test.py conftest.py .pytest_cache/ diff --git a/apps/memtrace-memos-integration/FEATURES.md b/apps/memtrace-memos-integration/FEATURES.md deleted file mode 100644 index 450ee9d4d..000000000 --- a/apps/memtrace-memos-integration/FEATURES.md +++ /dev/null @@ -1,242 +0,0 @@ -# MemOS × MemTrace 集成:产品功能说明 - -## 1. 概览 - -| 产品 | 一句话定位 | 归属 | -| --- | --- | --- | -| **本插件**(`memos_smartcomment`) | 面向开发者的调试工具。把每次记忆写入和检索都记录成一张执行图,记录本身不影响业务结果。 | MemOS 可选插件,默认关闭,按需开启 | -| **MemTrace** | 读取执行图,沿着数据依赖往回查,找出导致答案出错的那一步操作,并给出诊断报告。 | 浙大 ZJUNLP 开源项目,[独立仓库](https://github.com/zjunlp/MemTrace),不随本插件提供 | -| **smartcomment** | 两者之间的「通用语言」,定义执行图的数据格式,负责导出 JSON 和可视化。 | 公开 PyPI 包,本插件的依赖 | - -简单说,本插件负责记录,MemTrace 负责分析,smartcomment 是两者共用的数据格式。 - -### 1.1 定位:开发者内部迭代的调试工具 - -本插件服务于开发者的开发与调试。开发者在本地或测试环境中通过环境变量按需开启,用来排查 bad case、分析评测失败、验证改动效果。调试结束后关闭即可。使用 MemOS 的用户对它完全无感,生产部署也无需安装或开启。 - -| 关注点 | 说明 | -| --- | --- | -| 面向用户 | 开发者和算法 / 评测同学,用于内部迭代时定位问题。 | -| 使用场景 | 复现和排查 bad case、跑基准评测并做错误归因、验证提取 / 增强 / 检索策略的改动是否生效。 | -| 如何开启 | 启动服务前设置 `MEMOS_ENABLED_PLUGINS=smartcomment`。需要检索溯源时再加 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true`。不设置时插件不会加载,去掉变量后重启即关闭。 | -| 对用户的影响 | 无感。插件只旁路订阅 Hook,不改变任何请求、响应和业务结果,API 行为与未开启时完全一致。 | -| 生产部署 | 无需安装,也无需开启。线上服务保持默认配置即可,不会产生任何执行图文件或额外开销。 | - -## 2. 要解决的问题 - -MemOS 回答错了,或者「应该记住的事没记住」,原因往往不在最后一步,而是藏在中间某个环节: - -- **写入时**:MemReader 没有从对话里提取出这条事实; -- **后台增强时**:MemRead 调度器把记忆合并、改写或删除了; -- **检索时**:记忆其实还在,但被相关度阈值、去重或重排过滤掉了。 - -MemOS 的日志只能看到零散的事件,看不出「这条记忆从哪条消息来、经过了哪些步骤、在哪一步消失了」。本插件和 MemTrace 合在一起,目的是在开发调试阶段帮助开发者把这条链路看清楚。 - -## 3. 整体关系 - -```text - ┌──────────────────── 本插件负责 ────────────────────┐ ┌──────── MemTrace 负责 ────────┐ - │ │ │ │ - ┌───────────┐ Hook │ ┌──────────────┐ ┌──────────────┐ ┌──────────┐ │ │ ┌───────────┐ ┌───────────┐ │ - │ MemOS │ ───▶ │ │ 观测与快照 │ ─▶│ 建图与关联 │ ─▶│ 落盘与索引 │ │ ─▶│ │ 回溯探查 │ ─▶│ 归因与报告 │ │ - │ Add/Search│ │ └──────────────┘ └──────────────┘ └──────────┘ │ │ └───────────┘ └───────────┘ │ - └───────────┘ │ │ │ │ - └──────────────────────────┬─────────────────────────┘ └───────────────▲───────────────┘ - │ graph.json(smartcomment 执行图格式) │ - └────────────────────────────────────────────────┘ -``` - -- **数据流方向是单向的**:MemOS → 本插件 → 执行图文件 → MemTrace。MemTrace 不会回写 MemOS,本插件也不会调用 MemTrace。 -- **连接点只有一个**:本插件输出的 `graph.json`。它采用 smartcomment 格式,和 MemTrace 已支持的 Mem0、EverMemOS、RAG 等系统的执行图格式相同,所以 MemTrace 可以直接加载,不需要额外转换。 -- **两者可以单独使用**:不装 MemTrace 时,本插件的产物也能直接查看、可视化或用脚本分析。 - -## 4. 本插件的功能 - -开发者开启插件后,它运行在 MemOS 服务进程内,通过 MemOS 已有的 Hook 机制工作,**不修改 MemOS 的任何业务代码**。以下功能都只在插件开启时生效。 - -### 4.1 记录写入链路 - -记录一次 `/product/add` 中,记忆从原始对话到写入存储的全过程: - -```text -原始消息批次 ──提取──▶ 提取出的候选记忆 ──写入──▶ 已写入存储的记忆(带 memory_id) -``` - -功能:监测这批消息被提取成了哪几条记忆,每条记忆写入后的 ID 是什么,提取或写入有没有失败 - -### 4.2 记录后台增强 - -MemOS 的 MemRead 调度器会在请求返回**之后**继续异步处理记忆:精炼增强、写入增强结果、删除旧记忆。本插件把这些迟到的操作**追加到原来那张 Add 图上**,一张图就能看完一次写入的前因后果: - -```text -已写入的记忆 ──增强──▶ 增强后的记忆 ──写入──▶ 新的已写入记忆 - └──删除──▶ 调度结果 -``` - -功能:监测 MemRead 调度器将记忆后来被改成了什么 - -### 4.3 记录检索管线 - -记录一次 `/product/search` 的每个阶段,图中**只保留各阶段处理后仍存在的记忆列表**: - -```text -查询 → 记忆类型 → cube → 原始召回 → 阈值过滤 → 去重 → 重排(最终结果) -``` - -功能:通过比较相邻阶段的结果列表,监测记忆有没有被召回、在哪个阶段消失,以及最终返回结果的顺序和分数。被剔除的记忆不再单独生成列表节点,阶段 metadata 保留过滤数量统计。结果列表的 category 按 raw、threshold、dedup、rerank 区分,DOT 图以不同颜色展示各阶段。 - -### 4.4 检索结果溯源 - -开启 `MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true` 后,本插件会把 Search 的最终结果**连回它们在历史 Add 图中的来源**,并把来源链路复制进这张 Search 图: - -```text -原始消息批次 → 提取 → 写入 → 增强 → 写入 ◀──溯源── 最终检索结果 -``` - -连接边由最终检索结果列表指向记忆单元,comment 展示该结果在列表内从 1 开始的 rank 和实际 score。这样只看一张 Search 图,就能知道「这条结果最初来自哪批消息、中间经历了什么」。 - -这是本插件最关键的产品能力:它把原本分散在多次请求里的写入图和检索图串成一条完整的证据链,MemTrace 拿到这张图后,可以从检索结果一路追溯到原始对话。 - -### 4.5 请求目录,开箱即读 - -输出文件按「用户 / 日期 / 请求类型」组织,每个用户自动生成一份 `index.md`,按时间倒序列出每次请求的类型、查询或消息摘要、trace ID 和图文件链接。不需要任何工具,打开 Markdown 就能找到想看的那次请求。 - -| 时间 | 类型 | 请求摘要 | Trace ID | 图 | -| --- | --- | --- | --- | --- | -| 2026-10-15T14:36:10+08:00 | search | 我什么时候去游泳? | 462f7ef9... | graph | -| 2026-10-15T14:35:22+08:00 | add | 2 messages: 我上周开始学游泳了… | 0ba7563d... | graph | - -通过请求头 `x-trace-id` 可以指定 trace ID,方便把执行图和业务日志对应起来。 - -### 4.6 可视化与脚本分析 - -本插件的产物是标准 smartcomment 执行图,因此可以直接使用 smartcomment 自带的能力: - -- `smartcomment-visualize` 一键生成交互式 HTML、PNG 图片或 DOT 源文件; -- 用 `ExecNetwork` 写几行 Python,查询祖先 / 后代节点、按类别筛选,例如「这条消息最后变成了哪些记忆」「哪些记忆在去重阶段被剔除」。 - -> 严格来说,可视化和 `ExecNetwork` 是 **smartcomment 的能力**,本插件负责产出能被它们直接读取的文件。 - -### 4.7 调试时不干扰服务 - -本插件不面向生产部署,但开发者常常需要在接近真实的环境中复现问题。下面这些设计保证:即使在调试时开启插件,MemOS 的正常服务也不受影响。 - -| 特性 | 说明 | -| --- | --- | -| **默认关闭** | 安装后不会生效,必须显式设置 `MEMOS_ENABLED_PLUGINS=smartcomment` 才会启动插件,去掉后重启即可关闭。生产部署保持默认即可,无需任何额外配置。 | -| **非侵入** | 只订阅 Hook,不改变任何请求、响应和业务结果。插件出错不会影响 MemOS 请求,调用方对插件是否开启无感。 | -| **不阻塞请求** | Hook 中只做数据快照,建图、溯源和写文件都在后台线程完成。队列满时丢弃事件并告警,不会为了记录而拖慢业务。 | -| **可靠落盘** | 原子写文件,服务重启或缓存淘汰后,迟到的事件会从磁盘恢复,继续追加到原来那张图上。 | - -## 5. MemTrace 中与本插件相关的功能 - -MemTrace 是一个完整的记忆系统错误分析框架,下面与本插件相关的能力由 MemTrace 提供,需要单独安装 [MemTrace 仓库](https://github.com/zjunlp/MemTrace)并配置 OpenAI 兼容的模型。本插件不包含也不修改 MemTrace 的任何功能。 - -### 5.1 共同基础:operation-variable 执行图 - -MemTrace 把记忆系统的运行过程表示为一张「操作—变量」执行图:**变量**是运行中产生的具体数据,**操作**是产生或使用这些数据的步骤,**边**表示数据依赖。答案出错时,沿着依赖往回追,就能找到错误是在哪一步引入的。 - -本插件就是按这套模型来记录 MemOS 的,两边的概念一一对应: - -| MemTrace 概念 | 在本插件产物中的对应 | -| --- | --- | -| **变量** | 原始消息批次(`input_message_batch`)、提取出的记忆(`extracted_memory`)、已写入的记忆(`persisted_memory`)、增强后的记忆(`enhanced_memory`)、检索查询(`search_query`)、各阶段保留的检索结果列表(`search_memory_list_{stage}`) | -| **操作** | MemOS 中的提取、写入、后台增强、归档 / 删除、召回、阈值过滤、去重、重排,以及检索结果溯源,统一以 `memos.` 开头命名 | -| **依赖** | Add 内的「消息 → 提取 → 写入 → 增强」、Search 内的各阶段结果链,以及「最终检索结果 → 记忆单元」的溯源边;只记录确定的关系,不按时间先后猜测 | -| **说明文字** | 每个节点、边和操作都带一句英文 `comment`,MemTrace 探查时会展示给 Agent,帮助它理解每一步在做什么 | - -因此,本插件输出的 `graph.json` 可以用 `ExecNetwork.import_graph()` 直接加载,这也是 MemTrace 内部读取执行图的同一个入口,不需要格式转换。 - -### 5.2 交互式图探查:可直接使用 - -这是 MemTrace 中与本插件结合最紧密的能力,由两个组件组成: - -- **`GraphTraceNotebook`**:加载一张执行图,从指定的起点节点出发,向 Agent 提供「查看节点、展开上游 / 下游、查看操作」等图工具; -- **`GraphTraceAgent`**:一个 LLM Agent,根据你的问题调用这些工具逐步查看图,最后基于图中的证据给出结论。 - -用于本插件产物时: - -| 环节 | 做法 | -| --- | --- | -| **选哪张图** | 推荐用**开启溯源后的 Search 图**。它同时包含检索的各个阶段和最终结果的 Add 来源,Agent 在一张图里就能从检索结果一路追到原始对话。不开溯源时 Search 图只有检索部分,Add 图只有写入部分,Agent 无法跨图回溯。 | -| **从哪里开始** | 怀疑「检索出了问题」时,以最终结果列表(`search_memory_list_rerank`)为起点;怀疑「某句话没记住」时,以对应的消息批次节点为起点。 | -| **怎么提问** | 告诉 Agent 预期答案,让它逐步检查相关消息有没有被提取、写入、增强,最终有没有被检索到,在哪个操作丢失了。 | -| **能得到什么** | 一个基于图中证据的结论,例如「该事实在 MemReader 提取时未被抽出」或「该记忆被召回,但在阈值过滤阶段被剔除」。 | - -### 5.3 批量错误归因与诊断报告:需要补充问答数据 - -如果要分析的不是单个问题,而是一整轮评测中的大量失败用例,可以使用 MemTrace 的两个批量工具: - -- **批量错误归因**(`run_error_attribution.py`):对每个失败用例自动定位导致错误的关键操作,并判断错误类型; -- **诊断报告**(`generate_report.py`):汇总所有归因结果,用 LLM 总结目标系统的常见失败模式,输出系统级报告。 - -这两个工具不能直接读取本插件的产物,原因是它们需要一张**完整的问答图**,即除了写入和检索,还要有问题、模型回答、标准答案和标注。本插件只记录 MemOS **服务端内**的写入和检索;「根据检索结果生成回答」通常发生在客户端或评测脚本中,因此不会出现在图里。 - -| 批量归因需要的内容 | 本插件是否提供 | -| --- | --- | -| 记忆写入、后台增强的执行过程 | ✅ Add 图 | -| 检索各阶段的执行过程与结果 | ✅ Search 图 | -| 检索结果到原始消息的依赖链 | ✅ 开启溯源后的 Search 图 | -| 问题、模型回答、标准答案、错误标注 | ❌ 需要在评测侧补充 | - -也就是说,本插件提供了批量归因中最难获取的那部分数据,即 MemOS 内部的执行过程。剩下的问答数据需要和评测结果一起,整理成 MemTrace 批量工具要求的格式(MemTraceBench 格式)。 - -## 6. 典型使用场景 - -### 场景一:单个 bad case 排查(只用本插件) - -> 用户说「我告诉过你我周二周四游泳」,MemOS 却答不上来。 - -1. 打开该用户的 `index.md`,找到那次 Search 和之前的 Add; -2. 看 Add 图:这句话有没有被提取成记忆?后台增强时有没有被删掉或改写? -3. 看 Search 图:这条记忆有没有被召回?是被阈值、去重还是重排剔除的? -4. 开启溯源(即 Add 和 Search 图相连)后,只看 Search 图就能把整条链路看全。 - -**价值**:几分钟内判断问题出在提取、增强还是检索,不用翻日志,也不用加调试代码。 - -### 场景二:AI 分析(本插件 + MemTrace Agent) - -> 主要场景:链路很长、记忆很多,人工看图太累,使用 Agent 进行错误归因。 - -1. 本插件产出 Search 图(建议开启溯源); -2. 用 MemTrace 的 `GraphTraceNotebook` 加载这张图,以最终检索结果或可疑的消息批次为起点; -3. 告诉 `GraphTraceAgent` 预期答案,让它逐步检查「相关消息有没有被提取、写入、最终检索到,是哪个操作丢掉的」。 - -**价值**:由 Agent 基于图中证据给出结论,适合复杂 case 或需要可解释结论的场合。 - -### 场景三:系统级评测与归因(本插件 + MemTrace 批量工具,需要额外整理数据) - -> 在 LoCoMo 等基准上评测 MemOS,想知道失败主要集中在哪些环节。 - -1. 用独立的输出目录跑评测,本插件记录全部 Add / Search; -2. 在评测脚本中完成问答和打分,把失败用例的问题、答案、标注和对应执行图组织成 MemTraceBench 格式; -3. 运行 MemTrace 的批量归因,再用诊断报告汇总失败模式。 - -**价值**:从「某个 case 错了」上升到「MemOS 的哪个模块最常出错」,为优化提供方向。 - -## 7. 快速开始(开发 / 调试环境) - -以下步骤只用于本地或测试环境的调试,生产部署请跳过。 - -```bash -# 1. 在运行 MemOS 的同一环境中安装 -python -m pip install "smartcomment>=0.1.2" -python -m pip install -e apps/memtrace-memos-integration - -# 2. 按需开启插件进行调试(必须在启动服务前设置;不设置则插件不加载) -export MEMOS_ENABLED_PLUGINS=smartcomment -export MEMOS_SMARTCOMMENT_OUTPUT_DIR="$PWD/.memos/memtrace" -export MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true # 推荐:开启检索溯源 -python -m uvicorn memos.api.server_api:app --host 127.0.0.1 --port 8000 --workers 1 - -# 3. 正常调用 /product/add 和 /product/search 后,查看请求目录 -find .memos/memtrace -name index.md - -# 4. 调试结束:取消环境变量后重启服务,即恢复为未开启插件的状态 -unset MEMOS_ENABLED_PLUGINS -``` - -## 8. 相关链接 - -- [MemTrace](https://github.com/zjunlp/MemTrace) · [论文](https://arxiv.org/abs/2605.28732) · [MemTraceBench](https://huggingface.co/datasets/zjunlp/MemTraceBench) -- [smartcomment](https://github.com/zjunlp/smartcomment) · [PyPI](https://pypi.org/project/smartcomment/) diff --git a/apps/memtrace-memos-integration/README.changes.md b/apps/memtrace-memos-integration/README.changes.md deleted file mode 100644 index 9c556ab7d..000000000 --- a/apps/memtrace-memos-integration/README.changes.md +++ /dev/null @@ -1,97 +0,0 @@ -# SmartComment 集成优化说明 - -基线:`feat/smartcomment-integration` 分支的 `e80b7ced`。第 1–6 节记录已提交的首轮优化 `305e3ea7`;第 7 节记录随后按 TODO 清单进行的整理。本文不重复介绍基线已有的 Hook 和插件功能。 - -## 1. 修改范围 - -- 在现有架构内精简插件实现,合入 Hook 测试优化,修正文档与实现不一致的描述。 -- 保持公开接口、配置及默认值、Hook 位置与顺序、异常处理边界不变。 -- 保留图的业务身份、来源关系、顺序、分类和输出说明,以及脱敏、快照隔离、锁、弱引用、队列、缓存、原子落盘和旧图恢复机制。 -- 不修改 MemTrace,不修改依赖约束或锁文件。MemOS 核心仅纠正一行 Reader 分组注释,可执行逻辑未改。 - -## 2. 插件代码修改 - -以下路径相对本目录的 `memos_smartcomment/`。可执行修改集中在 4 个文件、9 个私有函数。 - -| 文件 / 函数 | 修改内容 | 等价依据 | -| --- | --- | --- | -| `handlers/add_handler.py` · `_fine_transfer_values` | 将按索引切片取输入改为 `zip(..., strict=True)` 直接配对。 | 仅在 `per_input` 且数量相等时执行;顺序和空组保留。写入结果仍按唯一 ID 匹配。 | -| `handlers/add_handler.py` · `_on_scheduler_memory_operation` | 直接区分归档和删除;在添加状态节点前计算持久化来源状态,删除事后两次筛选。 | 分类和说明字符串不变;完整、部分、未知来源及空结果的判断相同。 | -| `handlers/base.py` · `_hook_subject` | 删除 `trace_id` 的重复提取。 | 保留显式 `None` 的语义;内部字典键顺序变化,但现有调用方仅按字段名读取,不影响业务输出。 | -| `handlers/base.py` · `_unique_values` | 用字典推导替代逐项赋值循环。 | 同身份保留最后一份值,位置仍按首次出现顺序排列。 | -| `handlers/search_handler.py` · `_search_memory_value` | 直接读取已规范化字段,删除重复默认值和字符串转换。 | 唯一构造入口 `_search_result_items` 已填齐字段并完成类型转换;外部输入检查保留。 | -| `handlers/search_handler.py` · `_search_memory_match_key` | 直接读取规范化字段。 | ID / 原始正文的选择、cube 范围、JSON 编码和 SHA-256 算法均未变。 | -| `handlers/search_handler.py` · `_match_search_nodes` | 接收已构造节点,复用 `match_key`;删除重复计算和缺失键兜底,调整私有参数及类型标注。 | 候选缓存仅写入已计算匹配键的节点;重复候选 FIFO 配对及过滤规则不变。 | -| `handlers/search_handler.py` · `_on_search_results` | 删除本地列表的多余 tuple 转换;传递当前节点;直接拼接输出 tuple。 | 内容和顺序不变;过滤节点不进入候选缓存,业务值的快照复制仍保留。 | -| `recorder.py` · `_normalize_memory_units` | 将嵌套字典推导改为直接更新边的来源和目标 ID。 | 使用相同别名映射,保留其他边属性、边顺序和快照恢复规则。 | - -其余插件 Python 文件仅整理注释、文档字符串或导入分组标记,没有修改执行逻辑。`adapter.py`、`handlers/base.py`、`plugin.py` 中的三处 `isort: split` 用于兼容根目录与插件的 Ruff 配置,原有绝对导入及执行顺序不变。 - -## 3. Hook 测试修改 - -以下路径相对仓库根目录。以 `93e40823` 核对上游测试归属,仅优化本次集成新增的测试。 - -| 文件 | 修改内容 | -| --- | --- | -| `tests/conftest.py` | 新增显式启用的 `clean_hooks` fixture,替代重复清理代码;不使用全局 autouse。 | -| `tests/plugins/test_hook_context.py` | 注册表列齐 14 个 Hook;合并上下文字段及复制检查;保留新旧签名兼容、单次调用、同步和异步上下文复用;删除 UUID 格式、类型提示等低价值检查。 | -| `tests/api/test_search_pipeline_hooks.py` | 撤出追加的 Hook 测试,保留上游六个测试及辅助代码;相对上游仅保留 Hook 接口所需的两处 `hook_context` 参数断言调整。 | -| `tests/api/test_search_hook_context.py` | 独立验证七个成功 Hook 的顺序与共享上下文、请求/响应替换、五个结果管道对最终响应的影响及失败传播。 | -| `tests/mem_reader/test_hook_pipeline.py` | 用一个成功用例检查参数、上下文和结果替换;用一个失败用例检查原异常及后续成功 Hook 不执行。 | -| `tests/multi_mem_cube/test_single_cube_hooks.py` | 使用固定有效 ID;验证写入参数、返回 ID 替换及后续调度消费;精简失败路径重复断言。 | -| `tests/mem_scheduler/test_operation_observer.py` | 保留 Observer 成功管道与失败传播;执行 MemRead 处理逻辑,覆盖 fine-transfer、增强写入、归档、delete、soft-delete,并验证替换结果被后续步骤使用。 | -| `tests/mem_scheduler/test_mem_read_grouping.py` | 删除“显式声明优先”的过期预期,按实际实现验证 `per_input`、`batch`、`unknown`;保留输入顺序、失败空组及异常传播检查。 | - -其余上游原有测试未修改、跳过或放宽。七个共享 Hook 测试文件已同步到独立 Hook 工作树,Hook PR 不依赖插件即可验证。 - -## 4. 文档纠偏 - -更新插件 `README.md` 和 `memos_smartcomment/CONVENTIONS.md`: - -- Reader 分组由绑定的方法实现自动识别,不读取显式分组声明。 -- 成功 Hook 表示被观察的方法返回,不独立证明数据库写入成功;失败 Hook 只能观察传播至该边界的异常。 -- 当前 MemRead refresh 调用没有独立观测点,不单独生成刷新事件。 -- 移除未随仓库提供的插件测试命令和过时失败数量,保留安装、启用及 JSON 产物检查说明。 - -`src/memos/mem_scheduler/task_schedule_modules/handlers/mem_read_handler.py` 仅删除注释中“除非显式声明”的过期表述,自动分组和 Reader 顺序保持逻辑未改。 - -## 5. 验证结果与边界 - -使用 Python 3.12.14、Poetry 2.1.3、现有锁文件及 SmartComment 0.1.2。原版与优化版在同一依赖环境的独立进程中运行相同插件回归,并核对模块加载路径,避免混用工作树代码。 - -| 验证范围 | 结果 | -| --- | --- | -| 独立 Hook 工作树 | 41 passed | -| 集成工作树 Hook + Reader 顺序/分组 | 49 passed | -| 本地插件回归:原版 / 优化版 | 31 passed / 31 passed | -| 相关离线 MemOS 模块 | 266 passed,1 skipped(上游已有) | -| 根目录与插件配置的定点 Ruff、格式检查 | 通过 | -| 两工作树 `git diff --check` | 通过 | - -上述测试集合存在重叠,不应累加通过数量。插件回归包含真实 SmartComment 图构造和 JSON 写入/读回;数据库、LLM 等外部边界使用替身。插件本地回归覆盖添加关联、来源配对、搜索阶段匹配、旧图恢复、脱敏、快照隔离及生命周期。 - -等价结论针对现有公开使用路径和业务输出;源码位置字段 `filename`、`lineno`、`trigger_point` 可随路径和行号变化,不承诺 JSON 字节完全一致。 - -完整 `make test` 在 Ollama 外部调用处达到 180 秒诊断上限后终止,未完成全量测试,也未进行真实部署环境的端到端验收。此限制不应写成“全量通过”。 - -## 6. 提交范围 - -提交内容为上述插件清理、Hook 测试优化、文档及本说明。插件目录下本地维护的 `tests/`、`tmp/` 和日志继续被 Git 忽略,不纳入提交。通用 Hook 与独立插件仍按两个上游 PR 的定位准备,不在本轮重新划分基线已有提交。 - -## 7. Trace 说明与代码布局整理(基于 `305e3ea7`) - -| TODO | 本轮处理 | -| --- | --- | -| 1. 字段命名与说明 | 保留 category、class_name、operation 和 identity 取值;新增 `TraceEvent.comment`,让操作/session 展示业务说明;补齐显式连线说明,澄清写入返回、搜索过滤、桶内排名、归档与删除的观测边界。 | -| 2. 历史兼容 | 删除 `recorder._normalize_memory_units()` 的旧图格式迁移。当前格式恢复、ID 占位补全、首个完整快照及缓存淘汰后的续写保留。 | -| 3. 冗余字段 | 删除未被读取或导出的 `TraceValue.name` 及构造参数;导出图的 `name` 仍由 identity 生成。 | -| 4. 可读性 | 为规范化 Search 候选增加一个内部 `_SearchItem` TypedDict。保留多个真实入口共用的记忆展开函数,不改来源匹配算法。 | -| 5. 代码布局 | Add 的调度处理紧接调度回调;Search 的共用流程紧接阶段回调,缓存清理归入关联状态;Adapter 集中生命周期方法。加简短功能分组注释,不新建模块。 | - -修改集中在 `events.py`、`handlers/base.py`、`handlers/add_handler.py`、`handlers/search_handler.py`、`recorder.py`、`adapter.py`,同步更新 README 和 CONVENTIONS。MemOS 核心、仓库原有测试、Hook 测试、MemTrace、依赖及锁文件均未改动。 - -本轮不是完整输出逐字等价:图中的 comment 有意更新,`TraceValue` 构造不再接受 name,不再自动迁移历史图;来源关系、节点身份、分类、业务值和现有调用点的处理规则保持不变。持久化节点已有完整快照时不会只为更新说明而覆盖历史 comment。历史 split-version/无 snapshot_status 图应放在独立目录,不继续追加。 - -验证:修改前插件回归 **31 passed**;修改后 **33 passed**,包括真实 SmartComment 图导出/读回、当前快照恢复及占位补全、说明传递、缓存恢复和失败恢复。相关 MemOS 离线回归 **266 passed, 1 skipped**(原有 skip),数据库/LLM 边界仍使用替身。插件测试仍只在本地维护。未重新部署完整 MemOS;此前全量测试的 Ollama 环境限制不变。 - -Hook/Reader 定点回归 **49 passed**(与上述离线集合重叠);改动文件的 pre-commit、两套 Ruff 配置检查/格式检查及 `git diff --check` 通过。Adapter 所有方法的语法树与基线一致,仅顺序和注释变化。 diff --git a/apps/memtrace-memos-integration/README.md b/apps/memtrace-memos-integration/README.md index 6fa2eb6e2..82aed1977 100644 --- a/apps/memtrace-memos-integration/README.md +++ b/apps/memtrace-memos-integration/README.md @@ -1,474 +1,223 @@ -# MemOS smartcomment integration +# SmartComment for MemOS -Trace how memories move through MemOS add, Search API, and asynchronous MemRead operations. This external plugin listens to existing MemOS Hooks and writes a smartcomment execution graph as JSON for each trace. Installing the plugin does not enable it automatically. +English | [简体中文](README_zh.md) -**Dependency:** use the [public `smartcomment` package on PyPI](https://pypi.org/project/smartcomment/). The plugin requires `smartcomment>=0.1.2`; you do not need a smartcomment source checkout. The integration in this directory is installed separately. +SmartComment helps you understand **how MemOS turns messages into memories and retrieves them later**. This optional integration records memory operations as execution graphs, so you can investigate questions such as “Why wasn't this fact remembered?” or “Where did this search result come from?” + +Use it when debugging a failed case, evaluating memory quality, or checking changes to extraction and retrieval strategies. The plugin is **disabled by default**, including after installation. Enable it when needed in your development or test environment; normal MemOS usage does not require it. + +## What you can see + +| Capability | Questions it helps answer | +| --- | --- | +| Memory creation | Which messages produced a memory? What was extracted, and which memory IDs were returned by the write operation? | +| Background processing | How did MemRead enhance memories, write the enhanced results, or delete older memories? | +| Search stages | Was a memory retrieved? Did it disappear during filtering or deduplication? What were the final order and scores? | +| Search-to-Add history | Which original messages and processing steps led to a final search result? This feature is optional. | +| Request browsing | Which graph belongs to a request? Each user gets an `index.md` with timestamps, request summaries, trace IDs, and graph links. | + +The graphs follow these flows: + +```text +Add: messages → extracted memories → written memories → background enhancement +Search: query → memory type → memory cube → retrieval → filtering → deduplication → reranking +``` + +A memory cube is a MemOS memory collection. Search graphs show the result lists at the stages that actually ran; disabled filtering or deduplication stages are skipped. Background MemRead events can arrive after the API response and extend the original graph when they carry the same trace context. + +The integration observes operations without changing their business results. Recording still consumes resources, and a graph reflects the operations observed by the plugin, rather than an independent check of the database. ## Quick start -### 1. Install into the MemOS environment +### 1. Prepare your MemOS environment -Use Python 3.12 or newer. Activate the **same environment that runs MemOS**, then run these commands from the MemOS repository root: +Use **Python 3.12 or newer** and the **same Python environment that runs MemOS**. First follow the [MemOS deployment instructions](../../README.md) to configure the service, models, and storage. The commands below assume you are in the MemOS repository root. ```bash -# Skip this line if MemOS is already installed in the active environment. +# Skip this line if this MemOS checkout is already installed in the active environment. python -m pip install -e . + python -m pip install "smartcomment>=0.1.2" python -m pip install -e apps/memtrace-memos-integration ``` -The integration package also declares both MemoryOS and smartcomment as dependencies, but installing the public smartcomment package explicitly makes its source clear. If your MemOS environment was created with `uv` and has no `pip` module, use its interpreter explicitly instead: +`smartcomment` is a public PyPI package; no separate source checkout is needed. The integration in this directory must also be installed. -```bash -uv pip install --python .venv/bin/python -e . -uv pip install --python .venv/bin/python "smartcomment>=0.1.2" -uv pip install --python .venv/bin/python -e apps/memtrace-memos-integration -``` +If your environment uses `uv` and has no `pip` module, use `uv pip install --python .venv/bin/python` in place of `python -m pip install`, adjusting the interpreter path to your environment. -Skip the first line if MemOS is already installed there. +### 2. Enable the plugin and start the service -Check that both packages and the MemOS plugin entry point are visible to this interpreter: +Set these variables **before starting MemOS**: ```bash -python -c 'from importlib.metadata import version, entry_points; print("smartcomment", version("smartcomment")); print("integration", version("memos-smartcomment-integration")); print("plugin", [ep.name for ep in entry_points(group="memos.plugins") if ep.name == "smartcomment"])' +export MEMOS_ENABLED_PLUGINS=smartcomment +export MEMOS_SMARTCOMMENT_OUTPUT_DIR="$PWD/.memos/memtrace" + +# Optional: include the origins of final search results. Disabled by default. +export MEMOS_SMARTCOMMENT_LINK_SEARCH_TO_ADD=true + +python -m uvicorn memos.api.server_api:app --host 127.0.0.1 --port 8000 --workers 1 ``` -The last line should show `plugin ['smartcomment']`. If it does not, install the integration again with the interpreter used to start MemOS. +Look for `smartcomment plugin loaded` in the startup log. If you use other opt-in plugins, keep their names in `MEMOS_ENABLED_PLUGINS`, separated by commas. Restart the service whenever you change plugin settings. + +The example uses one worker. Each writer process needs its own output directory; multiple workers must not write to the same directory. -### 2. Enable the plugin and start MemOS +### 3. Add a memory and search for it -Configure the MemOS service as described in the [self-hosting guide](../../README.md). Set the plugin variables **before** starting the service: +Use the [local API documentation](http://127.0.0.1:8000/docs) or your existing client to call `/product/add` and `/product/search`. With an existing user and memory cube, you can try the following commands in another terminal. Replace `YOUR_USER_ID` and `YOUR_CUBE_ID` with your own values; see the [MemOS README](../../README.md) if you need to create a cube first. ```bash -export MEMOS_ENABLED_PLUGINS=smartcomment -export MEMOS_SMARTCOMMENT_OUTPUT_DIR=.memos/memtrace -python -m uvicorn memos.api.server_api:app --host 127.0.0.1 --port 8000 +curl -X POST http://127.0.0.1:8000/product/add \ + -H 'Content-Type: application/json' \ + -d '{ + "user_id": "YOUR_USER_ID", + "writable_cube_ids": ["YOUR_CUBE_ID"], + "messages": [{"role": "user", "content": "I go swimming on Tuesdays and Thursdays."}], + "async_mode": "sync" + }' + +curl -X POST http://127.0.0.1:8000/product/search \ + -H 'Content-Type: application/json' \ + -d '{ + "user_id": "YOUR_USER_ID", + "readable_cube_ids": ["YOUR_CUBE_ID"], + "query": "When do I go swimming?" + }' ``` -`MEMOS_ENABLED_PLUGINS` accepts comma-separated names if you use other opt-in plugins. The output path is relative to the service's working directory; use an absolute path when that directory may change. Startup should log `smartcomment plugin loaded`. +Check that the Add request succeeds before searching. Graph files are written asynchronously; for Search-to-Add history, wait for the Add graph to appear before running Search. Background enhancement may continue afterward. -### 3. Run a memory operation and inspect the graph +To match a graph with your application logs, you can supply an `x-trace-id` request header. Use a unique trace ID for each request if you want separate graphs: requests with the same user and trace ID share a graph. Add and Search do not need the same trace ID for history linking. -Call a working MemOS `/product/add` or `/product/search` endpoint (available in the service's `/docs` page). Add records the message → extraction → persistence flow; Search records retrieval, filtering, deduplication, and reranking. MemRead scheduler events may arrive later and extend the same trace. +### 4. Find the recorded requests -After the operation, list the saved graphs: +With the output directory above, run this from the repository root: ```bash -find .memos/memtrace -type f -name 'index.md' -print +find .memos/memtrace -type f -name index.md ``` -Open your user's `index.md` to browse requests, newest first. Each row includes the -request time with UTC offset, entry type, a bounded query/message preview, the full -trace ID, and a relative link to its `graph.json`. +Open your user's `index.md`. It lists requests newest first, with the request time and UTC offset, type, query or message summary, full trace ID, and a link to `graph.json`. + +The main files are organized as follows; names below are illustrative and supporting index files are omitted: ```text .memos/memtrace/ -└── alice--/ - ├── _user.json +└── user--/ ├── index.md - ├── _index.json - └── 2026-10-15/ - ├── search/ - │ └── 14-32-10.125__--/ + └── 2026-11-17/ + ├── add/ + │ └──