diff --git a/catalog.json b/catalog.json index c38afd8..4a19fef 100644 --- a/catalog.json +++ b/catalog.json @@ -110,6 +110,13 @@ "link": "https://mastra.ai", "logo": "https://www.google.com/s2/favicons?sz=32&domain=mastra.ai" }, + "memoryhub": { + "key": "memoryhub", + "description": "MemoryHub: Kubernetes-native agent memory with hybrid vector + keyword search, cross-encoder reranking, and reciprocal-rank fusion. PostgreSQL + pgvector backend.", + "kind": "cloud", + "link": "https://github.com/redhat-ai-americas/memory-hub", + "logo": null + }, "mem0": { "link": "https://mem0.ai", "logo": "https://www.google.com/s2/favicons?sz=32&domain=mem0.ai", diff --git a/outputs/personamem/memoryhub/rag/32k.json.gz b/outputs/personamem/memoryhub/rag/32k.json.gz new file mode 100644 index 0000000..92f62c0 Binary files /dev/null and b/outputs/personamem/memoryhub/rag/32k.json.gz differ diff --git a/pyproject.toml b/pyproject.toml index c7183a4..9d80c62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "hindsight-all>=0.4", "supermemory>=0.1", "httpx>=0.27", + "memoryhub>=0.15", "qdrant-client>=1.13", "fastapi[standard]>=0.135.1", "uvicorn>=0.41.0", diff --git a/results-manifest.json b/results-manifest.json index 23cfd95..20cb83f 100644 --- a/results-manifest.json +++ b/results-manifest.json @@ -756,5 +756,21 @@ "wall_s": 10665.0, "tokens_in": 67479034, "tokens_out": 1049086 + }, + { + "path": "outputs/personamem/memoryhub/rag/32k.json", + "dataset": "personamem", + "run_name": "memoryhub", + "memory": "memoryhub", + "mode": "rag", + "split": "32k", + "total_queries": 589, + "correct": 493, + "accuracy": 0.8370118845500849, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 1344.3, + "avg_context_tokens": 159989.6, + "category": null } ] \ No newline at end of file diff --git a/src/memory_bench/memory/__init__.py b/src/memory_bench/memory/__init__.py index 2b7e5e0..ae98d08 100644 --- a/src/memory_bench/memory/__init__.py +++ b/src/memory_bench/memory/__init__.py @@ -6,6 +6,7 @@ from .mastra_om import MastraOMMemoryProvider from .mem0 import Mem0MemoryProvider from .mem0_cloud import Mem0CloudMemoryProvider +from .memoryhub import MemoryHubProvider from .hybrid_search import HybridSearchMemoryProvider from .ogham import OghamMemoryProvider from .supermemory import SupermemoryMemoryProvider @@ -25,6 +26,7 @@ "mastra-om": MastraOMMemoryProvider, "mem0": Mem0MemoryProvider, "mem0-cloud": Mem0CloudMemoryProvider, + "memoryhub": MemoryHubProvider, "ogham": OghamMemoryProvider, "qdrant": HybridSearchMemoryProvider, "supermemory": SupermemoryMemoryProvider, diff --git a/src/memory_bench/memory/memoryhub.py b/src/memory_bench/memory/memoryhub.py new file mode 100644 index 0000000..064014a --- /dev/null +++ b/src/memory_bench/memory/memoryhub.py @@ -0,0 +1,146 @@ +"""MemoryHub provider for AMB. + +Uses the memoryhub SDK to talk to a MemoryHub MCP server over +streamable-HTTP. MemoryHub stores verbatim conversations and retrieves +via hybrid vector + keyword search with reciprocal-rank fusion and +cross-encoder reranking. + +Required env vars +----------------- +MEMORYHUB_URL MCP server endpoint (e.g. https://…/mcp/) +MEMORYHUB_API_KEY API key for session auth + +Optional env vars +----------------- +MEMORYHUB_PROJECT_ID Project for memory isolation (default: amb-benchmark) +MEMORYHUB_K Retrieval depth (default: 70) +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from pathlib import Path +from typing import Any + +from memoryhub import MemoryHubClient + +from ..models import Document +from .base import MemoryProvider + +logger = logging.getLogger(__name__) + + +class MemoryHubProvider(MemoryProvider): + name = "memoryhub" + description = ( + "MemoryHub: hybrid vector + keyword search with " + "cross-encoder reranking and reciprocal-rank fusion." + ) + kind = "cloud" + link = "https://github.com/redhat-ai-americas/memory-hub" + concurrency = 1 + + def __init__(self): + self._url: str | None = None + self._api_key: str | None = None + self._project_id: str | None = None + self._k: int = 70 + + def prepare( + self, + store_dir: Path, + unit_ids: set[str] | None = None, + reset: bool = True, + ) -> None: + self._url = os.environ.get("MEMORYHUB_URL") + self._api_key = os.environ.get("MEMORYHUB_API_KEY") + self._project_id = os.environ.get("MEMORYHUB_PROJECT_ID", "amb-benchmark") + self._k = int(os.environ.get("MEMORYHUB_K", "70")) + if not self._url or not self._api_key: + raise RuntimeError( + "MEMORYHUB_URL and MEMORYHUB_API_KEY are required. " + "Point MEMORYHUB_URL at the MCP server's streamable-HTTP endpoint." + ) + + # ------------------------------------------------------------------ + # Ingest + # ------------------------------------------------------------------ + + def ingest(self, documents: list[Document]) -> None: + asyncio.run(self._ingest(documents)) + + async def _ingest(self, documents: list[Document]) -> None: + async with MemoryHubClient(url=self._url, api_key=self._api_key) as client: + try: + await client.create_project( + self._project_id, + description="AMB benchmark memory isolation", + ) + logger.info("Created project %s", self._project_id) + except Exception: + logger.debug("Project %s already exists", self._project_id) + + for i, doc in enumerate(documents): + owner = f"amb-{doc.user_id}" if doc.user_id else "amb-default" + result = await client.write( + content=doc.content, + scope="project", + project_id=self._project_id, + owner_id=owner, + content_type="experiential", + force=True, + ) + + if not (result.memory): + logger.warning( + "Write returned no memory for doc %s", doc.id, + ) + + if (i + 1) % 50 == 0: + logger.info("Ingested %d/%d documents", i + 1, len(documents)) + + logger.info("Ingestion complete: %d documents", len(documents)) + + # ------------------------------------------------------------------ + # Retrieve + # ------------------------------------------------------------------ + + def retrieve( + self, + query: str, + k: int | None = None, + user_id: str | None = None, + query_timestamp: str | None = None, + ) -> tuple[list[Document], dict | None]: + return asyncio.run(self._retrieve(query, k, user_id)) + + async def _retrieve( + self, + query: str, + k: int | None, + user_id: str | None, + ) -> tuple[list[Document], dict | None]: + effective_k = self._k if (k is None or k == 10) else k + owner = f"amb-{user_id}" if user_id else "amb-default" + + async with MemoryHubClient(url=self._url, api_key=self._api_key) as client: + results = await client.search( + query=query, + max_results=effective_k, + owner_id=owner, + project_id=self._project_id, + weight_threshold=0.0, + mode="full_only", + max_response_tokens=0, + ) + + documents = [ + Document(id=m.id, content=m.content, user_id=user_id) + for m in results.results + ] + return documents, None + + def cleanup(self) -> None: + pass