diff --git a/src/memos/dream/README.md b/src/memos/dream/README.md index 3fe7dab09..d23aa3999 100644 --- a/src/memos/dream/README.md +++ b/src/memos/dream/README.md @@ -127,3 +127,93 @@ Dream is designed as a community-building surface. Good places to contribute: Projects can ship their own plugin with the same logical name (`dream`). When multiple providers expose `dream`, the plugin manager keeps the implementation with the highest priority. + + +## Opt-in lifecycle maintenance (PostgreSQL) + +Dream retention is explicit and disabled by default. The initial atomic backend is +PostgreSQL with pgvector. Existing providers remain usable with lifecycle disabled; +enabling it on a backend without the capability raises an error. No schema, public +HTTP route, request model or new core dependency is required. + +Supply lifecycle options in the component context before initializing the Dream +plugin (the ordinary server bootstrap owns this context): + +```python +context["configs"]["dream_lifecycle"] = { + "enabled": True, + "policy": { + "half_life_seconds": 604800, + "never_hit_grace_seconds": 604800, + "event_max_age_seconds": 86400, + "receipt_capacity": 128, + }, + "cube_ids": ["authorized-cube"], + "checkpoint_directory": "/var/lib/memos/dream-cursors", + "interval_seconds": 0, # manual; positive enables a cooperative worker + "limits": {"page_size": 100, "max_nodes": 10000, "max_seconds": 60}, +} +plugin.init_components(context) +report = plugin.lifecycle.run_maintenance("authorized-cube", dry_run=True) +print(report.model_dump_json(indent=2)) +``` + +Only explicitly registered cubes are visited, including quiet cubes with no new +Dream signals. Additional scopes require `plugin.lifecycle.register_cube(cube_id)`. +The caller authorizes those scopes; the runner does not enumerate tenants or add +permissions. Maintenance reads bounded metadata pages, revalidates under row locks +and never hard-deletes memories or changes stored content/embeddings/confidence. +It derives effective strength from fixed event-time anchors, not repeated decay. +Retrieval counts reinforce retention but do not constitute positive usefulness. + +The decorated API search path records only final returned IDs after result pipes. +Native successful feedback updates notify exact affected source IDs: superseded +sources are invalidated; in-place corrections contribute a negative outcome without +invalidating corrected current content. General answer sentiment does not invalidate +all retrieved memories. Notifications use detached snapshots and fail open for the +original operation. `plugin.lifecycle.observer.delivery_snapshot()` exposes failed +delivery counters. These notifications are not a durable outbox or a transaction +with the search/feedback operation, and the SDK search path is not automatically +covered. Use explicit scoped events for SDK or application feedback: + +```python +from datetime import datetime, timezone +from memos.dream.lifecycle import EventKind, LifecycleEvent + +service = plugin.lifecycle.service +event = LifecycleEvent( + event_id="stable-producer-event-42", memory_id="returned-memory-id", + cube_id="authorized-cube", at=datetime.now(timezone.utc), + kind=EventKind.POSITIVE, +) +service.observe([event], now=datetime.now(timezone.utc)) +``` + +Retain the same event ID, timestamp and payload across retries. Identical replay is +a no-op within the accepted-event-age horizon; conflicting IDs, expired events and +live receipt-capacity exhaustion fail rather than silently losing deduplication. +Capacity is deliberately bounded. This is not unlimited exactly-once processing. +Policy fingerprints prevent silently reinterpreting existing evidence after a +configuration change. Malformed metadata is quarantined, not treated as low quality. + +`service.inspect(ids, cube_id=..., now=...)` explains decisions without initializing +state or modifying receipts. `service.set_pinned` exempts passive archive decisions; +explicit invalidation still wins. `service.restore(memory_id, cube_id=..., reason=..., +now=...)` restores lifecycle-owned archives, requiring `override_invalidation=True` +for explicitly invalidated evidence. It does not reconstruct lost content. + +A checkpoint records backend/scope/policy identity and last committed page decisions +with evidence revisions. Atomic replacement follows database commit; a crash in +between can replay a page safely. A failed page rolls back and retains its previous +cursor, reporting the bounded retry IDs. Reports cap detailed decisions and count +omissions, retaining aggregate reasons. Dry runs write neither lifecycle state nor +checkpoint/lock files. Time/cancellation limits are cooperative between pages, not +hard deadlines for arbitrary backend calls. Plugin shutdown unregisters owned hooks +and acknowledges outstanding work; a timed-out backend operation is not claimed +to have stopped. + +For native tests, provide `MEMOS_TEST_PG_DSN` for a disposable PostgreSQL+pgvector +database and run `pytest tests/dream -q`. Each test owns/drops its own schema. With +no DSN, backend tests skip explicitly. Install the existing tree/scheduler extras +and the backend's documented psycopg2-binary requirement; no model calls are needed. +Synthetic retention/throughput tests validate mechanics, not real-agent QA gains. diff --git a/src/memos/dream/contextualization.py b/src/memos/dream/contextualization.py index 911caf2ac..f137811d7 100644 --- a/src/memos/dream/contextualization.py +++ b/src/memos/dream/contextualization.py @@ -417,6 +417,11 @@ def _persist_group(self, *, graph_db, group: DreamContextGroup, context_node, cu summary_strategy=summary_strategy, ) + lifecycle = self.context.get("shared", {}).get("dream_lifecycle") + if lifecycle is not None: + metadata = lifecycle.metadata_for_new( + context_id, cube_id, metadata, now=datetime.now(timezone.utc) + ) graph_db.add_node(context_id, summary, metadata, user_name=cube_id) return _build_context_event( context_id=context_id, diff --git a/src/memos/dream/lifecycle/__init__.py b/src/memos/dream/lifecycle/__init__.py new file mode 100644 index 000000000..693ad0f6b --- /dev/null +++ b/src/memos/dream/lifecycle/__init__.py @@ -0,0 +1,32 @@ +"""Opt-in lifecycle evidence and reversible Dream retention policy.""" + +from memos.dream.lifecycle.contracts import ( + LIFECYCLE_METADATA_KEY, + EventKind, + LifecycleConfig, + LifecycleError, + LifecycleEvent, + LifecycleState, + MaintenanceDecision, +) +from memos.dream.lifecycle.policy import ( + apply_event, + decide_maintenance, + initialize_state, + restore_state, +) + + +__all__ = [ + "LIFECYCLE_METADATA_KEY", + "EventKind", + "LifecycleConfig", + "LifecycleError", + "LifecycleEvent", + "LifecycleState", + "MaintenanceDecision", + "apply_event", + "decide_maintenance", + "initialize_state", + "restore_state", +] diff --git a/src/memos/dream/lifecycle/contracts.py b/src/memos/dream/lifecycle/contracts.py new file mode 100644 index 000000000..14901babb --- /dev/null +++ b/src/memos/dream/lifecycle/contracts.py @@ -0,0 +1,217 @@ +"""Validated metadata and event contracts for opt-in Dream maintenance.""" + +from __future__ import annotations + +import hashlib +import json + +from copy import deepcopy +from datetime import datetime, timezone +from enum import Enum +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from memos.exceptions import MemoryError as MemOSMemoryError +from memos.graph_dbs.base import MANAGED_LIFECYCLE_KEY, MANAGED_LIFECYCLE_VERSION + + +LIFECYCLE_METADATA_KEY = MANAGED_LIFECYCLE_KEY + + +class LifecycleError(MemOSMemoryError): + """An observation or transition cannot safely change lifecycle evidence.""" + + +class LifecycleMetadataError(LifecycleError): + """Stored evidence is invalid or incompatible and must remain quarantined.""" + + +class LifecycleCapacityError(LifecycleError): + """Live deduplication receipts cannot be evicted to admit another event.""" + + +def detach_lifecycle_identity(metadata: dict[str, Any], *, source_memory_id: str) -> dict[str, Any]: + """Prepare an explicit new identity without inheriting observations or receipts. + + Source/provenance fields remain intact. The small Dream-origin marker allows + the lifecycle service to initialize fresh evidence even for a system-sourced + diary that was originally recognized only by its reserved envelope. This + function changes a new copy, never the source node's managed state. + """ + fresh = deepcopy(metadata) + if LIFECYCLE_METADATA_KEY not in fresh: + return fresh + fresh.pop(LIFECYCLE_METADATA_KEY) + internal = fresh.get("internal_info") or {} + if not isinstance(internal, dict): + raise LifecycleError("Cannot preserve malformed derived-memory provenance") + dream = internal.get("dream") or {} + if not isinstance(dream, dict): + raise LifecycleError("Cannot preserve malformed Dream-origin provenance") + dream.setdefault("kind", "derived_memory") + dream["lifecycle_origin_memory_id"] = source_memory_id + internal["dream"] = dream + fresh["internal_info"] = internal + return fresh + + +def utc_time(value: datetime | str, *, legacy: bool = False) -> datetime: + """Normalize a time; only legacy stored times may omit a timezone.""" + if isinstance(value, str): + try: + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise LifecycleError("Invalid lifecycle timestamp") from exc + if not isinstance(value, datetime): + raise LifecycleError("A lifecycle timestamp must be a datetime") + if value.tzinfo is None or value.utcoffset() is None: + if not legacy: + raise LifecycleError("New lifecycle timestamps require a timezone") + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +class EventKind(str, Enum): + RETRIEVED = "retrieved" + POSITIVE = "positive" + NEGATIVE = "negative" + INVALIDATED = "invalidated" + + +class LifecycleConfig(BaseModel): + """Retention parameters; their fingerprint prevents silent reinterpretation.""" + + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + half_life_seconds: float = Field(default=604800, gt=0, strict=True) + never_hit_grace_seconds: float = Field(default=604800, ge=0, strict=True) + archive_strength_threshold: float = Field(default=0.1, ge=0, le=1, strict=True) + retrieval_reinforcement: float = Field(default=0.1, ge=0, strict=True) + strength_cap: float = Field(default=2.0, gt=0, strict=True) + low_usefulness_threshold: float = Field(default=0.2, ge=0, le=1, strict=True) + min_outcome_count: int = Field(default=3, ge=1, strict=True) + event_max_age_seconds: float = Field(default=86400, gt=0, strict=True) + future_tolerance_seconds: float = Field(default=60, ge=0, strict=True) + receipt_capacity: int = Field(default=128, ge=1, le=100000, strict=True) + + def fingerprint(self) -> str: + payload = json.dumps(self.model_dump(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +class LifecycleEvent(BaseModel): + """An immutable, scoped producer event; IDs must not be recycled.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + event_id: str = Field(min_length=1, max_length=512, strict=True) + memory_id: str = Field(min_length=1, max_length=1024, strict=True) + cube_id: str = Field(min_length=1, max_length=1024, strict=True) + at: datetime + kind: EventKind + reason: str = Field(default="", max_length=1024, strict=True) + + @field_validator("at", mode="before") + @classmethod + def aware_event_time(cls, value: datetime | str) -> datetime: + try: + return utc_time(value) + except LifecycleError as exc: + raise ValueError(str(exc)) from exc + + def digest(self) -> str: + payload = json.dumps(self.model_dump(mode="json"), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +class EventReceipt(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identity: str = Field(pattern=r"^[0-9a-f]{64}$") + payload: str = Field(pattern=r"^[0-9a-f]{64}$") + expires_at: datetime + + @field_validator("expires_at", mode="before") + @classmethod + def aware_expiry(cls, value: datetime | str) -> datetime: + return utc_time(value) + + +class LifecycleState(BaseModel): + """The reserved graph envelope; ordinary writers cannot replace it.""" + + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + version: int = Field( + default=MANAGED_LIFECYCLE_VERSION, + ge=MANAGED_LIFECYCLE_VERSION, + le=MANAGED_LIFECYCLE_VERSION, + strict=True, + ) + memory_id: str = Field(min_length=1, strict=True) + cube_id: str = Field(min_length=1, strict=True) + policy_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$") + source_dream_id: str = "" + created_at: datetime + base_strength: float = Field(default=1.0, ge=0, strict=True) + pinned: bool = Field(default=False, strict=True) + last_hit_at: datetime | None = None + hit_count: int = Field(default=0, ge=0, strict=True) + positive_mass: float = Field(default=0.0, ge=0, strict=True) + negative_mass: float = Field(default=0.0, ge=0, strict=True) + outcome_count: int = Field(default=0, ge=0, strict=True) + mass_anchor_at: datetime | None = None + invalidated_by_feedback: bool = Field(default=False, strict=True) + status: str = Field(default="active", pattern=r"^(active|archived)$") + protect_archived_status: bool = Field(default=False, strict=True) + revision: int = Field(default=0, ge=0, strict=True) + archived_at: datetime | None = None + archive_reason: str = "" + restored_at: datetime | None = None + restore_reason: str = "" + receipts: tuple[EventReceipt, ...] = () + + @field_validator( + "created_at", "last_hit_at", "mass_anchor_at", "archived_at", "restored_at", mode="before" + ) + @classmethod + def stored_time(cls, value: Any) -> datetime | None: + return None if value is None else utc_time(value, legacy=True) + + +class MaintenanceDecision(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + action: str = Field(pattern=r"^(keep|archive|skip)$") + reason: str + strength: float = Field(ge=0) + usefulness: float = Field(ge=0, le=1) + + +def checked_state(state: LifecycleState, config: LifecycleConfig) -> LifecycleState: + """Revalidate even model_copy inputs before a storage reducer trusts them.""" + try: + state = LifecycleState.model_validate(state.model_dump(mode="python")) + except ValueError as exc: + raise LifecycleError("Invalid stored lifecycle envelope") from exc + if state.protect_archived_status != (state.status == "archived"): + raise LifecycleError("Lifecycle archive ownership/status mismatch") + if state.status == "archived" and ( + state.archived_at is None or not state.archive_reason.strip() + ): + raise LifecycleError("Lifecycle archive lacks transition provenance") + if state.invalidated_by_feedback and not state.protect_archived_status: + raise LifecycleError("Invalidated lifecycle evidence must remain archived") + if (state.hit_count == 0) != (state.last_hit_at is None): + raise LifecycleError("Lifecycle retrieval count/time mismatch") + if state.outcome_count == 0 and (state.positive_mass != 0 or state.negative_mass != 0): + raise LifecycleError("Lifecycle outcome mass lacks observed outcomes") + if state.policy_fingerprint != config.fingerprint(): + raise LifecycleError("Lifecycle policy mismatch; explicit migration is required") + if len(state.receipts) > config.receipt_capacity: + raise LifecycleError("Stored receipt capacity exceeds lifecycle policy") + identities = [receipt.identity for receipt in state.receipts] + if len(set(identities)) != len(identities): + raise LifecycleError("Duplicate stored lifecycle receipt identity") + return state diff --git a/src/memos/dream/lifecycle/observation.py b/src/memos/dream/lifecycle/observation.py new file mode 100644 index 000000000..417054a27 --- /dev/null +++ b/src/memos/dream/lifecycle/observation.py @@ -0,0 +1,159 @@ +"""Observe final returned memories and explicitly committed feedback operations.""" + +from __future__ import annotations + +import threading + +from collections import defaultdict +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + +from memos.dream.lifecycle.contracts import EventKind, LifecycleError, LifecycleEvent + + +if TYPE_CHECKING: + from memos.dream.lifecycle.service import DreamLifecycleService, ObservationReport + + +@dataclass(frozen=True) +class ObservationDelivery: + """Cumulative bounded delivery counters for this process; no user content retained.""" + + accepted: int = 0 + duplicates: int = 0 + skipped: int = 0 + missing: int = 0 + failed: int = 0 + last_event_id: str | None = None + last_error_type: str | None = None + + +class DreamLifecycleObserver: + """Bridge native notifications to explicit stable lifecycle event identities. + + Native notifications are best effort and represent individual server executions, + not HTTP-request idempotency. SDK callers can invoke observe_returned/observe + with a retained event ID and original timestamp for bounded replay safety. + Only committed observations affect retention; delivery is not atomic with the + search response or the preceding feedback write. + """ + + def __init__(self, service: DreamLifecycleService) -> None: + self.service = service + self._delivery = ObservationDelivery() + self._lock = threading.Lock() + + def delivery_snapshot(self) -> ObservationDelivery: + """Read counters and the last delivery error without exposing memory content.""" + with self._lock: + return self._delivery + + def _record(self, event_id: str, report: ObservationReport) -> None: + with self._lock: + old = self._delivery + self._delivery = ObservationDelivery( + accepted=old.accepted + report.accepted, + duplicates=old.duplicates + report.duplicates, + skipped=old.skipped + report.skipped, + missing=old.missing + report.missing, + failed=old.failed, + last_event_id=event_id, + ) + + def _commit(self, events: list[LifecycleEvent], *, event_id: str, at: datetime) -> None: + try: + report = self.service.observe(events, now=datetime.now(timezone.utc)) + except Exception as exc: + with self._lock: + self._delivery = replace( + self._delivery, + failed=self._delivery.failed + 1, + last_event_id=event_id, + last_error_type=type(exc).__name__, + ) + raise + self._record(event_id, report) + + def observe_returned( + self, *, request: Any, results: dict[str, Any], event_id: str, at: datetime + ) -> None: + """Count each final memory ID once per cube; intermediate/source IDs do not count.""" + if not event_id or not isinstance(results, dict): + raise LifecycleError("Observation requires an event ID and final result dictionary") + allowed = set( + getattr(request, "readable_cube_ids", None) or [getattr(request, "user_id", "")] + ) + grouped = defaultdict(set) + buckets = results.get("text_mem", []) + if not isinstance(buckets, list): + raise LifecycleError("Final text memory results must be a list") + for bucket in buckets: + if not isinstance(bucket, dict): + continue + cube_id = bucket.get("cube_id") or bucket.get("mem_cube_id") + if cube_id not in allowed: + continue + memories = bucket.get("memories", []) + if not isinstance(memories, list): + raise LifecycleError("Final memory bucket must contain a list") + for memory in memories: + if isinstance(memory, dict): + memory_id = memory.get("id") + if isinstance(memory_id, str) and memory_id: + grouped[cube_id].add(memory_id) + if sum(map(len, grouped.values())) > 1000: + raise LifecycleError("Final observation exceeds the 1000-memory delivery limit") + for cube_id in sorted(grouped): + events = [ + LifecycleEvent( + memory_id=memory_id, + cube_id=cube_id, + event_id=event_id, + at=at, + kind=EventKind.RETRIEVED, + ) + for memory_id in sorted(grouped[cube_id]) + ] + self._commit(events, event_id=event_id, at=at) + + def on_search_completed( + self, *, request: Any, result: Any, event_id: str, at: datetime + ) -> None: + """Consume the detached snapshot produced after all search.after pipes.""" + if getattr(result, "code", None) != 200: + return + self.observe_returned( + request=request, results=getattr(result, "data", None), event_id=event_id, at=at + ) + + def on_feedback_committed( + self, + *, + cube_id: str, + memory_ids: list[str], + action: str, + event_id: str, + at: datetime, + ) -> None: + """Attribute evidence only to the exact IDs in a completed native update. + + A superseded source is invalidated. An in-place correction contributes a + negative outcome without invalidating its newly corrected current content. + Generic answer sentiment and all retrieved IDs are deliberately ignored. + """ + kinds = {"superseded": EventKind.INVALIDATED, "corrected": EventKind.NEGATIVE} + if action not in kinds: + raise LifecycleError("Unknown committed feedback action") + events = [ + LifecycleEvent( + memory_id=item, + cube_id=cube_id, + event_id=event_id, + at=at, + kind=kinds[action], + reason="Committed feedback " + action, + ) + for item in dict.fromkeys(memory_ids) + ] + self._commit(events, event_id=event_id, at=at) diff --git a/src/memos/dream/lifecycle/policy.py b/src/memos/dream/lifecycle/policy.py new file mode 100644 index 000000000..4125f8bc0 --- /dev/null +++ b/src/memos/dream/lifecycle/policy.py @@ -0,0 +1,205 @@ +"""Pure event-time reducers; no storage, model calls or mutable global clock.""" + +from __future__ import annotations + +import hashlib +import math + +from datetime import datetime, timedelta + +from memos.dream.lifecycle.contracts import ( + EventKind, + EventReceipt, + LifecycleCapacityError, + LifecycleConfig, + LifecycleError, + LifecycleEvent, + LifecycleState, + MaintenanceDecision, + checked_state, + utc_time, +) + + +def initialize_state( + *, + memory_id: str, + cube_id: str, + created_at: datetime | str, + config: LifecycleConfig, + base_strength: float = 1.0, + pinned: bool = False, + source_dream_id: str = "", +) -> LifecycleState: + """Initialize evidence without inferring usefulness from Dream confidence.""" + return LifecycleState( + memory_id=memory_id, + cube_id=cube_id, + created_at=utc_time(created_at, legacy=True), + policy_fingerprint=config.fingerprint(), + base_strength=base_strength, + pinned=pinned, + source_dream_id=source_dream_id, + ) + + +def _decay(seconds: float, config: LifecycleConfig) -> float: + return 2.0 ** (-max(0.0, seconds) / config.half_life_seconds) + + +def _masses(state: LifecycleState, config: LifecycleConfig, now: datetime) -> tuple[float, float]: + anchor = state.mass_anchor_at or state.created_at + factor = _decay((now - anchor).total_seconds(), config) + return state.positive_mass * factor, state.negative_mass * factor + + +def _usefulness(positive: float, negative: float) -> float: + # Scale before adding to avoid overflow for valid large finite input masses. + scale = max(1.0, positive, negative) + prior = 1.0 / scale + return (positive / scale + prior) / (positive / scale + negative / scale + 2 * prior) + + +def apply_event( + state: LifecycleState, + event: LifecycleEvent, + config: LifecycleConfig, + *, + now: datetime, +) -> LifecycleState: + """Reduce one committed event with bounded, event-time deduplication.""" + state = checked_state(state, config) + now = utc_time(now) + if (state.memory_id, state.cube_id) != (event.memory_id, event.cube_id): + raise LifecycleError("Lifecycle event scope mismatch") + try: + event = LifecycleEvent.model_validate(event.model_dump(mode="python")) + except ValueError as exc: + raise LifecycleError("Invalid lifecycle event") from exc + age = (now - event.at).total_seconds() + if age > config.event_max_age_seconds: + raise LifecycleError("Lifecycle event is too old for the deduplication horizon") + if age < -config.future_tolerance_seconds: + raise LifecycleError("Lifecycle event is too far in the future") + if event.at < state.created_at - timedelta(seconds=config.future_tolerance_seconds): + raise LifecycleError("Lifecycle event predates memory creation") + + identity = hashlib.sha256(event.event_id.encode()).hexdigest() + digest = event.digest() + live = tuple(receipt for receipt in state.receipts if receipt.expires_at >= now) + for receipt in live: + if receipt.identity == identity: + if receipt.payload != digest: + raise LifecycleError("Lifecycle event identity conflict") + return state + if len(live) >= config.receipt_capacity: + raise LifecycleCapacityError("Lifecycle event receipt capacity exhausted") + receipt = EventReceipt( + identity=identity, + payload=digest, + expires_at=event.at + timedelta(seconds=config.event_max_age_seconds), + ) + event_time = min(event.at, now) + anchor = max(now, state.mass_anchor_at or state.created_at) + positive, negative = _masses(state, config, anchor) + changes = { + "revision": state.revision + 1, + "receipts": (*live, receipt), + "mass_anchor_at": anchor, + "positive_mass": positive, + "negative_mass": negative, + } + if event.kind == EventKind.RETRIEVED: + changes["hit_count"] = state.hit_count + 1 + changes["last_hit_at"] = max(event_time, state.last_hit_at or event_time) + elif event.kind in (EventKind.POSITIVE, EventKind.NEGATIVE): + weight = _decay((anchor - event_time).total_seconds(), config) + name = "positive_mass" if event.kind == EventKind.POSITIVE else "negative_mass" + changes[name] += weight + changes["outcome_count"] = state.outcome_count + 1 + elif event.kind == EventKind.INVALIDATED: + changes.update( + invalidated_by_feedback=True, + status="archived", + protect_archived_status=True, + archived_at=event_time, + archive_reason="feedback_invalidated", + ) + try: + return LifecycleState.model_validate({**state.model_dump(), **changes}) + except ValueError as exc: + raise LifecycleError("Lifecycle evidence overflow or invalid reduction") from exc + + +def decide_maintenance( + state: LifecycleState, + config: LifecycleConfig, + *, + now: datetime, + graph_status: str = "activated", +) -> MaintenanceDecision: + """Evaluate original evidence; repeated sweeps do not compound decay.""" + state = checked_state(state, config) + now = utc_time(now) + anchor = max( + value + for value in (state.created_at, state.last_hit_at, state.restored_at) + if value is not None + ) + age = max(0.0, (now - anchor).total_seconds()) + reinforced = state.base_strength * ( + 1 + config.retrieval_reinforcement * math.log1p(state.hit_count) + ) + strength = min(config.strength_cap, reinforced) * _decay(age, config) + usefulness = _usefulness(*_masses(state, config, now)) + + def decision(action, reason): + return MaintenanceDecision( + action=action, reason=reason, strength=strength, usefulness=usefulness + ) + + if state.invalidated_by_feedback: + return decision("archive", "feedback_invalidated") + if graph_status != "activated": + return decision( + "skip", "externally_archived" if graph_status == "archived" else "external_status" + ) + if state.pinned: + return decision("keep", "pinned") + if state.last_hit_at is None and age < config.never_hit_grace_seconds: + return decision("keep", "never_hit_grace") + if ( + state.outcome_count >= config.min_outcome_count + and usefulness < config.low_usefulness_threshold + ): + return decision("archive", "low_usefulness") + if strength < config.archive_strength_threshold: + return decision("archive", "stale") + return decision("keep", "retained") + + +def restore_state( + state: LifecycleState, + config: LifecycleConfig, + *, + now: datetime, + reason: str, + override_invalidation: bool = False, +) -> LifecycleState: + """Explicitly recover an archived item; never infer recovery from a hit.""" + state = checked_state(state, config) + if not reason or not reason.strip(): + raise LifecycleError("Lifecycle restoration requires a reason") + if state.invalidated_by_feedback and not override_invalidation: + raise LifecycleError("Restoring invalidated evidence requires an explicit override") + return LifecycleState.model_validate( + { + **state.model_dump(), + "status": "active", + "protect_archived_status": False, + "invalidated_by_feedback": False, + "restored_at": utc_time(now), + "restore_reason": reason, + "revision": state.revision + 1, + } + ) diff --git a/src/memos/dream/lifecycle/runner.py b/src/memos/dream/lifecycle/runner.py new file mode 100644 index 000000000..2a17188db --- /dev/null +++ b/src/memos/dream/lifecycle/runner.py @@ -0,0 +1,395 @@ +"""Bounded, resumable maintenance with scoped metadata-only keyset scans.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import threading +import time + +from contextlib import contextmanager, suppress +from datetime import datetime # noqa: TC003 - Pydantic resolves report timestamps at runtime. +from pathlib import Path +from typing import TYPE_CHECKING, Literal +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field + +from memos.dream.lifecycle.contracts import ( + LifecycleCapacityError, + LifecycleError, + LifecycleMetadataError, + utc_time, +) + + +if TYPE_CHECKING: + from collections.abc import Callable + + from memos.dream.lifecycle.service import DreamLifecycleService + + +class MaintenanceLimits(BaseModel): + """Cooperative limits checked between bounded atomic pages, not SQL timeouts.""" + + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + page_size: int = Field(default=100, ge=1, le=1000, strict=True) + max_report_decisions: int = Field(default=1000, ge=0, le=1000, strict=True) + max_nodes: int = Field(default=10000, ge=1, le=10000000, strict=True) + max_seconds: float = Field(default=60, gt=0, strict=True) + + +class MaintenanceRecord(BaseModel): + """One retention decision tied to the evidence revision actually read under lock.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + memory_id: str + action: str + reason: str + changed: bool + eligible: bool + evidence_revision: int | None + + +class MaintenanceCursor(BaseModel): + """A cursor is bound to the backend, cube and immutable retention policy.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + version: Literal[1] = 1 + binding: str = Field(pattern=r"^[0-9a-f]{64}$") + cube_id: str = Field(min_length=1, strict=True) + policy_fingerprint: str = Field(pattern=r"^[0-9a-f]{64}$") + sweep_id: str = Field(min_length=1, strict=True) + after: str | None = Field(default=None, min_length=1, strict=True) + complete: bool = Field(default=False, strict=True) + last_page: tuple[MaintenanceRecord, ...] = Field(default=(), max_length=1000) + + +class MaintenanceReport(BaseModel): + """Bounded operational summary; content, labels and receipt payloads are omitted.""" + + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + version: Literal[1] = 1 + cube_id: str + sweep_id: str + at: datetime + dry_run: bool + policy_fingerprint: str + backend: str + backend_capability: Literal["atomic_lifecycle_metadata"] = "atomic_lifecycle_metadata" + eligible: int = 0 + invalid: int = 0 + capacity_exceeded: int = 0 + restored: int = 0 + decisions: tuple[MaintenanceRecord, ...] = Field(default=(), max_length=1000) + omitted_decisions: int = 0 + reason_counts: dict[str, int] = Field(default_factory=dict) + retry_ids: tuple[str, ...] = Field(default=(), max_length=1000) + scanned: int = 0 + archived: int = 0 + would_archive: int = 0 + retained: int = 0 + skipped: int = 0 + missing: int = 0 + failed: int = 0 + complete: bool = False + next_after: str | None = None + stop_reason: str = "complete" + error_type: str | None = None + elapsed_seconds: float = 0 + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise LifecycleError("Duplicate checkpoint field") + result[key] = value + return result + + +class DreamMaintenanceRunner: + """Scan one cube with a single owner and publish cursors after committed pages. + + A page reduction is atomic. Failures retain the previous page cursor; a crash + after commit but before checkpoint publication can replay a page, and lifecycle + transitions remain idempotent. This is not exactly-once external delivery. + Completed cursors begin a new sweep on the next call so quiet cubes still age. + """ + + def __init__( + self, + service: DreamLifecycleService, + *, + checkpoint: str | Path | None = None, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.service = service + self.checkpoint = Path(checkpoint) if checkpoint is not None else None + self.monotonic = monotonic + self._owner = threading.Lock() + + def _binding(self, cube_id: str) -> str: + graph = self.service.graph + config = getattr(graph, "config", None) + location = { + field: getattr(config, field, None) + for field in ( + "host", + "port", + "db_name", + "schema_name", + "user", + "user_name", + ) + } + payload = { + "backend": f"{type(graph).__module__}.{type(graph).__qualname__}", + "location": location, + "cube_id": cube_id, + "policy": self.service.config.fingerprint(), + } + return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest() + + def _load_cursor(self, cube_id: str) -> MaintenanceCursor: + binding = self._binding(cube_id) + if self.checkpoint is not None and self.checkpoint.exists(): + try: + with self.checkpoint.open("rb") as stream: + raw = stream.read(2097153) + if len(raw) > 2097152: + raise LifecycleError("Maintenance checkpoint exceeds size limit") + data = json.loads( + raw, + object_pairs_hook=_unique_object, + parse_constant=lambda _value: (_ for _ in ()).throw( + ValueError("Nonfinite JSON") + ), + ) + cursor = MaintenanceCursor.model_validate(data) + except (ValueError, OSError) as exc: + raise LifecycleError("Invalid maintenance checkpoint") from exc + if ( + cursor.binding != binding + or cursor.cube_id != cube_id + or cursor.policy_fingerprint != self.service.config.fingerprint() + ): + raise LifecycleError("Maintenance checkpoint scope/binding mismatch") + if not cursor.complete: + return cursor + return MaintenanceCursor( + binding=binding, + cube_id=cube_id, + policy_fingerprint=self.service.config.fingerprint(), + sweep_id=uuid4().hex, + ) + + def _save_cursor(self, cursor: MaintenanceCursor) -> None: + if self.checkpoint is None: + return + path = self.checkpoint + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + descriptor, temporary = tempfile.mkstemp( + prefix=path.name + ".", suffix=".tmp", dir=path.parent + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(cursor.model_dump_json(indent=2)) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + temporary = None + if os.name != "nt": + descriptor = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + if temporary is not None: + with suppress(OSError): + os.unlink(temporary) + + @contextmanager + def _ownership(self, *, dry_run: bool): + if not self._owner.acquire(blocking=False): + raise LifecycleError("This maintenance runner already has an active owner") + lock = None + try: + if self.checkpoint is not None and not dry_run: + self.checkpoint.parent.mkdir(parents=True, exist_ok=True) + lock = self.checkpoint.with_name(self.checkpoint.name + ".lock").open("a+b") + try: + if os.name == "nt": + import msvcrt + + if lock.tell() == 0: + lock.write(b"0") + lock.flush() + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + raise LifecycleError( + "Maintenance checkpoint already has an active owner" + ) from exc + yield + finally: + # Closing releases the OS lock after errors or process termination; + # an empty lock file is not itself evidence of a live owner. + if lock is not None: + lock.close() + self._owner.release() + + def run( + self, + *, + cube_id: str, + now: datetime, + limits: MaintenanceLimits | None = None, + dry_run: bool = False, + cancelled: Callable[[], bool] | None = None, + ) -> MaintenanceReport: + """Run bounded pages; callers decide scheduling and explicitly register quiet cubes. + + New IDs inserted behind a cursor are eligible on a later full sweep. + Wall time is cooperative between database pages; arbitrary backend calls + cannot be forcibly cancelled or advertised as hard deadline enforcement. + """ + if not isinstance(cube_id, str) or not cube_id.strip(): + raise LifecycleError("Maintenance requires an explicit cube scope") + now = utc_time(now) + limits = MaintenanceLimits.model_validate((limits or MaintenanceLimits()).model_dump()) + with self._ownership(dry_run=dry_run): + cursor = self._load_cursor(cube_id) + started = self.monotonic() + values = { + "cube_id": cube_id, + "sweep_id": cursor.sweep_id, + "at": now, + "dry_run": dry_run, + "scanned": 0, + "archived": 0, + "would_archive": 0, + "retained": 0, + "skipped": 0, + "missing": 0, + "failed": 0, + "next_after": cursor.after, + "policy_fingerprint": self.service.config.fingerprint(), + "backend": type(self.service.graph).__name__, + "eligible": 0, + "invalid": 0, + "capacity_exceeded": 0, + "omitted_decisions": 0, + "reason_counts": {}, + "retry_ids": (), + } + decisions = [] + stop_reason = "complete" + complete = False + error_type = None + while True: + if cancelled is not None and cancelled(): + stop_reason = "cancelled" + break + if self.monotonic() - started >= limits.max_seconds: + stop_reason = "time_budget" + break + remaining = limits.max_nodes - values["scanned"] + if remaining <= 0: + stop_reason = "node_budget" + break + page = self.service.graph.list_lifecycle_nodes( + user_name=cube_id, after=cursor.after, limit=min(limits.page_size, remaining) + ) + if not page: + proposed = cursor.model_copy(update={"complete": True}) + complete = True + else: + ids = [row["id"] for row in page] + if ( + len(ids) != len(set(ids)) + or any(item <= (cursor.after or "") for item in ids) + or ids != sorted(ids) + ): + raise LifecycleError("Backend violated lifecycle keyset ordering") + try: + outcomes = self.service.maintain( + ids, cube_id=cube_id, now=now, dry_run=dry_run + ) + except Exception as exc: + values["failed"] += 1 + values["retry_ids"] = tuple(ids) + values["invalid"] += int(isinstance(exc, LifecycleMetadataError)) + values["capacity_exceeded"] += int(isinstance(exc, LifecycleCapacityError)) + error_type = type(exc).__name__ + stop_reason = "page_failed" + break + values["scanned"] += len(ids) + page_records = [] + for outcome in outcomes: + record = MaintenanceRecord( + memory_id=outcome.memory_id, + action=outcome.action, + reason=outcome.reason, + changed=outcome.changed, + eligible=outcome.eligible, + evidence_revision=outcome.evidence_revision, + ) + page_records.append(record) + if len(decisions) < limits.max_report_decisions: + decisions.append(record) + else: + values["omitted_decisions"] += 1 + values["eligible"] += int(outcome.eligible) + reasons = values["reason_counts"] + reasons[outcome.reason] = reasons.get(outcome.reason, 0) + 1 + if outcome.action == "archive": + if dry_run: + values["would_archive"] += 1 + elif outcome.changed: + values["archived"] += 1 + else: + values["skipped"] += 1 + elif outcome.action == "keep": + values["retained"] += 1 + elif outcome.action == "missing": + values["missing"] += 1 + else: + values["skipped"] += 1 + proposed = cursor.model_copy( + update={"after": ids[-1], "last_page": tuple(page_records)} + ) + if not dry_run: + try: + self._save_cursor(proposed) + except OSError as exc: + complete = False + stop_reason = "checkpoint_failed" + values["retry_ids"] = tuple(row["id"] for row in page) + error_type = type(exc).__name__ + break + cursor = proposed + values["next_after"] = cursor.after + if complete: + break + elapsed = max(0.0, self.monotonic() - started) + return MaintenanceReport( + **values, + complete=complete, + stop_reason=stop_reason, + error_type=error_type, + elapsed_seconds=elapsed, + decisions=tuple(decisions), + ) diff --git a/src/memos/dream/lifecycle/runtime.py b/src/memos/dream/lifecycle/runtime.py new file mode 100644 index 000000000..fe8decaff --- /dev/null +++ b/src/memos/dream/lifecycle/runtime.py @@ -0,0 +1,185 @@ +"""Opt-in plugin wiring and explicitly scoped quiet-cube maintenance scheduling.""" + +from __future__ import annotations + +import hashlib +import threading +import time + +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from memos.dream.lifecycle.contracts import LifecycleConfig, LifecycleError +from memos.dream.lifecycle.observation import DreamLifecycleObserver +from memos.dream.lifecycle.runner import DreamMaintenanceRunner, MaintenanceLimits +from memos.dream.lifecycle.service import DreamLifecycleService +from memos.log import get_logger + + +if TYPE_CHECKING: + from memos.dream.lifecycle.runner import MaintenanceReport + from memos.graph_dbs.base import BaseGraphDB + + +logger = get_logger(__name__) + + +class LifecycleRuntimeConfig(BaseModel): + """Explicit lifecycle options supplied through plugin component configuration. + + Zero interval keeps maintenance manual. A worker visits only registered cubes; + it never enumerates tenants from storage or depends on new Dream signals. + """ + + model_config = ConfigDict(frozen=True, extra="forbid", allow_inf_nan=False) + + enabled: bool = Field(default=False, strict=True) + policy: LifecycleConfig = Field(default_factory=LifecycleConfig) + limits: MaintenanceLimits = Field(default_factory=MaintenanceLimits) + cube_ids: tuple[str, ...] = () + max_registered_cubes: int = Field(default=1000, ge=1, le=10000, strict=True) + interval_seconds: float = Field(default=0, ge=0, strict=True) + checkpoint_directory: str | None = None + + @field_validator("cube_ids") + @classmethod + def scoped_ids(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if any(not isinstance(item, str) or not item.strip() for item in value): + raise ValueError("Maintenance cube IDs must be nonempty strings") + return tuple(dict.fromkeys(value)) + + +class DreamLifecycleRuntime: + """One service shared by persistence, observations and bounded maintenance. + + Default plugin configuration does not construct this object. The optional + worker has cooperative stop semantics between pages/cubes. Backend calls can + outlive a shutdown timeout; callers receive False instead of a false claim + that an in-flight operation was stopped or rolled back. + """ + + def __init__(self, graph: BaseGraphDB, config: LifecycleRuntimeConfig) -> None: + self.config = LifecycleRuntimeConfig.model_validate(config.model_dump()) + if not self.config.enabled: + raise LifecycleError("Construct lifecycle runtime only when explicitly enabled") + self.service = DreamLifecycleService(graph, self.config.policy) + self.observer = DreamLifecycleObserver(self.service) + self._stop = threading.Event() + self._lock = threading.RLock() + self._thread: threading.Thread | None = None + self._runners: dict[str, DreamMaintenanceRunner] = {} + self._reports: dict[str, MaintenanceReport] = {} + self._closed = False + self._active = 0 + self._idle = threading.Event() + self._idle.set() + for cube_id in self.config.cube_ids: + self.register_cube(cube_id) + + def register_cube(self, cube_id: str) -> None: + """Register an explicit authorized scope, including one with no recent writes.""" + if not isinstance(cube_id, str) or not cube_id.strip(): + raise LifecycleError("Maintenance requires a nonempty cube ID") + with self._lock: + if self._closed: + raise LifecycleError("Lifecycle runtime is closed") + if cube_id in self._runners: + return + if len(self._runners) >= self.config.max_registered_cubes: + raise LifecycleError("Registered maintenance cube capacity exhausted") + checkpoint = None + if self.config.checkpoint_directory is not None: + filename = hashlib.sha256(cube_id.encode()).hexdigest() + ".json" + checkpoint = Path(self.config.checkpoint_directory) / filename + self._runners[cube_id] = DreamMaintenanceRunner(self.service, checkpoint=checkpoint) + + @property + def registered_cubes(self) -> tuple[str, ...]: + """Expose an immutable snapshot of explicitly configured maintenance scopes.""" + with self._lock: + return tuple(self._runners) + + @property + def worker_alive(self) -> bool: + with self._lock: + return self._thread is not None and self._thread.is_alive() + + def run_maintenance( + self, + cube_id: str, + *, + now: datetime | None = None, + dry_run: bool = False, + limits: MaintenanceLimits | None = None, + ) -> MaintenanceReport: + """Run a registered cube manually; an active worker cannot own it concurrently.""" + with self._lock: + if self._closed: + raise LifecycleError("Lifecycle runtime is closed") + runner = self._runners.get(cube_id) + if runner is None: + raise LifecycleError("Register a maintenance scope before running it") + self._active += 1 + self._idle.clear() + try: + report = runner.run( + cube_id=cube_id, + now=now or datetime.now(timezone.utc), + limits=limits or self.config.limits, + dry_run=dry_run, + cancelled=self._stop.is_set, + ) + with self._lock: + self._reports[cube_id] = report + return report + finally: + with self._lock: + self._active -= 1 + if self._active == 0: + self._idle.set() + + def latest_reports(self) -> dict[str, MaintenanceReport]: + """Return detached report models with bounded cardinality and no memory content.""" + with self._lock: + return {cube: report.model_copy(deep=True) for cube, report in self._reports.items()} + + def start(self) -> None: + """Start at most one worker; a zero interval is deliberately manual-only.""" + with self._lock: + if self._closed: + raise LifecycleError("Cannot start a closed lifecycle runtime") + if self.config.interval_seconds == 0 or self.worker_alive: + return + self._thread = threading.Thread( + target=self._work, name="memos-dream-maintenance", daemon=True + ) + self._thread.start() + + def _work(self) -> None: + while not self._stop.wait(self.config.interval_seconds): + for cube_id in self.registered_cubes: + if self._stop.is_set(): + return + try: + self.run_maintenance(cube_id) + except Exception: + # Retain the cursor and continue other explicitly registered + # cubes. Operational failure is not a retention decision. + logger.exception("Dream lifecycle periodic maintenance failed") + + def shutdown(self, *, timeout: float = 5) -> bool: + """Signal cooperative cancellation and acknowledge whether the worker stopped.""" + if not isinstance(timeout, int | float) or not 0 <= timeout <= 300: + raise LifecycleError("Shutdown timeout must be between zero and 300 seconds") + with self._lock: + self._closed = True + self._stop.set() + worker = self._thread + deadline = time.monotonic() + timeout + if worker is not None and worker is not threading.current_thread(): + worker.join(timeout=timeout) + self._idle.wait(timeout=max(0, deadline - time.monotonic())) + return (worker is None or not worker.is_alive()) and self._idle.is_set() diff --git a/src/memos/dream/lifecycle/service.py b/src/memos/dream/lifecycle/service.py new file mode 100644 index 000000000..9f8e51f21 --- /dev/null +++ b/src/memos/dream/lifecycle/service.py @@ -0,0 +1,454 @@ +"""Transactional observation and reversible lifecycle operations for Dream memories.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from memos.dream.lifecycle.contracts import ( + LIFECYCLE_METADATA_KEY, + LifecycleConfig, + LifecycleError, + LifecycleEvent, + LifecycleMetadataError, + LifecycleState, + checked_state, + utc_time, +) +from memos.dream.lifecycle.policy import ( + apply_event, + decide_maintenance, + initialize_state, + restore_state, +) +from memos.graph_dbs.base import BaseGraphDB, NodeMetadataUpdate + + +if TYPE_CHECKING: + from collections.abc import Callable + from datetime import datetime + + +@dataclass(frozen=True) +class ObservationReport: + """Counts describe committed observations, never attempted or queued delivery.""" + + accepted: int = 0 + duplicates: int = 0 + skipped: int = 0 + missing: int = 0 + + +@dataclass(frozen=True) +class MemoryLifecycleOutcome: + """Small maintenance result, excluding content and bounded deduplication receipts.""" + + memory_id: str + action: str + reason: str + changed: bool = False + strength: float | None = None + usefulness: float | None = None + eligible: bool = False + evidence_revision: int | None = None + + +@dataclass(frozen=True) +class MemoryLifecycleInspection: + """Read-only retention explanation; excludes content and deduplication hashes.""" + + memory_id: str + cube_id: str + found: bool = True + eligible: bool = False + managed: bool = False + quarantined: bool = False + graph_status: str | None = None + next_action: str = "skip" + reason: str = "not_dream" + policy_fingerprint: str | None = None + revision: int | None = None + created_at: datetime | None = None + last_hit_at: datetime | None = None + hit_count: int | None = None + outcome_count: int | None = None + pinned: bool | None = None + invalidated_by_feedback: bool | None = None + strength: float | None = None + usefulness: float | None = None + receipt_count: int | None = None + receipt_capacity: int | None = None + + +class DreamLifecycleService: + """Own lifecycle mutations through one backend's atomic metadata capability. + + Policy parameters are immutable for an existing envelope. Changing them + requires a separately reviewed migration, rather than silently reinterpreting + evidence on the next read. Each public batch is scoped and atomic; distinct + calls/cubes and external notification delivery are not a distributed transaction. + """ + + def __init__(self, graph: BaseGraphDB, config: LifecycleConfig) -> None: + if not graph.supports_atomic_lifecycle: + raise LifecycleError("Dream lifecycle requires an atomic metadata backend") + self.graph = graph + self.config = LifecycleConfig.model_validate(config.model_dump()) + + @staticmethod + def is_dream(metadata: dict[str, Any]) -> bool: + """Recognize native Dream writes/Contexts without treating all system data as Dream.""" + internal = metadata.get("internal_info") + return ( + LIFECYCLE_METADATA_KEY in metadata + or metadata.get("source") == "dream" + or (isinstance(internal, dict) and isinstance(internal.get("dream"), dict)) + ) + + def metadata_for_new( + self, memory_id: str, cube_id: str, metadata: dict[str, Any], *, now: datetime + ) -> dict[str, Any]: + """Attach fresh evidence before a new Dream node is committed. + + Generated confidence remains provenance, not a probability of correctness + or the initial retention strength. Caller-supplied reserved state is replaced + for this new identifier; existing identifiers use atomic initialization. + """ + now = utc_time(now) + state = initialize_state( + memory_id=memory_id, + cube_id=cube_id, + created_at=metadata.get("created_at", now), + config=self.config, + source_dream_id=str(metadata.get("source_dream_id", "")), + ) + if (state.created_at - now).total_seconds() > self.config.future_tolerance_seconds: + raise LifecycleError("Memory creation is in the future") + return { + **metadata, + "created_at": state.created_at.isoformat(), + LIFECYCLE_METADATA_KEY: state.model_dump(mode="json"), + } + + def _state( + self, memory_id: str, cube_id: str, metadata: dict[str, Any], now: datetime + ) -> LifecycleState | None: + if not self.is_dream(metadata): + return None + try: + if LIFECYCLE_METADATA_KEY in metadata: + state = checked_state( + LifecycleState.model_validate(metadata[LIFECYCLE_METADATA_KEY]), self.config + ) + else: + if "created_at" not in metadata: + raise LifecycleError("Dream memory lacks a creation timestamp") + state = initialize_state( + memory_id=memory_id, + cube_id=cube_id, + created_at=metadata["created_at"], + config=self.config, + source_dream_id=str(metadata.get("source_dream_id", "")), + ) + if (state.memory_id, state.cube_id) != (memory_id, cube_id): + raise LifecycleError("Stored lifecycle scope mismatch") + if (state.created_at - now).total_seconds() > self.config.future_tolerance_seconds: + raise LifecycleError("Stored creation timestamp is in the future") + for timestamp in ( + state.last_hit_at, + state.mass_anchor_at, + state.archived_at, + state.restored_at, + ): + if timestamp is not None: + if (timestamp - now).total_seconds() > self.config.future_tolerance_seconds: + raise LifecycleError("Stored lifecycle evidence is in the future") + if ( + state.created_at - timestamp + ).total_seconds() > self.config.future_tolerance_seconds: + raise LifecycleError("Stored lifecycle evidence predates creation") + if state.protect_archived_status and metadata.get("status") != "archived": + raise LifecycleError("Protected lifecycle and graph status disagree") + return state + except (LifecycleError, ValueError, TypeError, AttributeError) as exc: + raise LifecycleMetadataError( + "Malformed Dream lifecycle metadata; no mutation applied" + ) from exc + + @staticmethod + def _fields(state: LifecycleState) -> dict[str, Any]: + fields = {LIFECYCLE_METADATA_KEY: state.model_dump(mode="json")} + if state.protect_archived_status: + fields.update( + status="archived", + archived_at=state.archived_at.isoformat() if state.archived_at else None, + archive_reason=state.archive_reason, + ) + return fields + + def observe(self, events: list[LifecycleEvent], *, now: datetime) -> ObservationReport: + """Commit an explicit same-cube batch; duplicate identity never adds evidence. + + Stable event IDs and original producer timestamps must survive retries. + Old events and live-receipt capacity exhaustion fail conservatively. An + invalid row rolls back the entire batch, so a caller can safely retry. + """ + now = utc_time(now) + if len(events) > 1000: + raise LifecycleError("Observation batches are bounded at 1000 events") + if not events: + return ObservationReport() + events = [LifecycleEvent.model_validate(event.model_dump()) for event in events] + cube_id = events[0].cube_id + if any(event.cube_id != cube_id for event in events): + raise LifecycleError("Observation batches must have one cube scope") + grouped = defaultdict(list) + for event in events: + grouped[event.memory_id].append(event) + + def reduce(memory_id, metadata): + state = self._state(memory_id, cube_id, metadata, now) + if state is None: + return NodeMetadataUpdate(None, (0, 0, len(grouped[memory_id]))) + accepted = duplicates = 0 + original = state + for event in grouped[memory_id]: + updated = apply_event(state, event, self.config, now=now) + if updated == state: + duplicates += 1 + else: + accepted += 1 + state = updated + return NodeMetadataUpdate( + self._fields(state) if state != original else None, (accepted, duplicates, 0) + ) + + rows = self.graph.mutate_nodes_metadata(list(grouped), reduce, user_name=cube_id) + accepted = duplicates = skipped = missing = 0 + for row in rows: + if not row.found: + missing += len(grouped[row.memory_id]) + else: + added, repeated, ignored = row.result + accepted += added + duplicates += repeated + skipped += ignored + return ObservationReport(accepted, duplicates, skipped, missing) + + def inspect( + self, ids: list[str], *, cube_id: str, now: datetime + ) -> list[MemoryLifecycleInspection]: + """Explain retention/quarantine without initializing metadata or pruning receipts. + + Each snapshot uses the same scoped current-read capability as a transition. + Malformed stored state becomes a quarantine explanation, allowing operators + to locate affected identifiers without treating corruption as low usefulness. + Inspection does not authorize a recovery or silently reinterpret a policy. + """ + now = utc_time(now) + + def reduce(memory_id, metadata): + common = { + "memory_id": memory_id, + "cube_id": cube_id, + "eligible": self.is_dream(metadata), + "managed": LIFECYCLE_METADATA_KEY in metadata, + "graph_status": metadata.get("status"), + } + try: + state = self._state(memory_id, cube_id, metadata, now) + if state is None: + return NodeMetadataUpdate(None, MemoryLifecycleInspection(**common)) + decision = decide_maintenance( + state, self.config, now=now, graph_status=metadata.get("status", "") + ) + except LifecycleError: + return NodeMetadataUpdate( + None, + MemoryLifecycleInspection( + **common, quarantined=True, reason="invalid_lifecycle" + ), + ) + report = MemoryLifecycleInspection( + **common, + next_action=decision.action, + reason=decision.reason, + policy_fingerprint=state.policy_fingerprint, + revision=state.revision, + created_at=state.created_at, + last_hit_at=state.last_hit_at, + hit_count=state.hit_count, + outcome_count=state.outcome_count, + pinned=state.pinned, + invalidated_by_feedback=state.invalidated_by_feedback, + strength=decision.strength, + usefulness=decision.usefulness, + receipt_count=len(state.receipts), + receipt_capacity=self.config.receipt_capacity, + ) + return NodeMetadataUpdate(None, report) + + rows = self.graph.mutate_nodes_metadata(ids, reduce, user_name=cube_id) + return [ + row.result + if row.found + else MemoryLifecycleInspection( + memory_id=row.memory_id, cube_id=cube_id, found=False, reason="not_found" + ) + for row in rows + ] + + def maintain( + self, ids: list[str], *, cube_id: str, now: datetime, dry_run: bool = False + ) -> list[MemoryLifecycleOutcome]: + """Revalidate current evidence under locks before any maintenance transition.""" + now = utc_time(now) + + def reduce(memory_id, metadata): + state = self._state(memory_id, cube_id, metadata, now) + if state is None: + return NodeMetadataUpdate( + None, MemoryLifecycleOutcome(memory_id, "skip", "not_dream") + ) + decision = decide_maintenance( + state, self.config, now=now, graph_status=metadata.get("status", "") + ) + fields = None + if decision.action == "archive" and not state.protect_archived_status: + archived = LifecycleState.model_validate( + { + **state.model_dump(), + "status": "archived", + "protect_archived_status": True, + "archived_at": now, + "archive_reason": decision.reason, + "revision": state.revision + 1, + } + ) + fields = self._fields(archived) + # A retained legacy node can acquire its initial envelope, but a dry + # run never initializes state or prunes/deduplicates receipts. + elif decision.action == "keep" and LIFECYCLE_METADATA_KEY not in metadata: + fields = self._fields(state) + outcome = MemoryLifecycleOutcome( + memory_id, + decision.action, + decision.reason, + bool(fields) and not dry_run, + decision.strength, + decision.usefulness, + eligible=True, + evidence_revision=state.revision, + ) + return NodeMetadataUpdate(None if dry_run else fields, outcome) + + rows = self.graph.mutate_nodes_metadata(ids, reduce, user_name=cube_id) + return [ + row.result + if row.found + else MemoryLifecycleOutcome(row.memory_id, "missing", "not_found") + for row in rows + ] + + def _transition( + self, + memory_id: str, + cube_id: str, + now: datetime, + transition: Callable[ + [LifecycleState, dict[str, Any]], tuple[LifecycleState, dict[str, Any], str] + ], + ) -> MemoryLifecycleOutcome: + now = utc_time(now) + + def reduce(item, metadata): + state = self._state(item, cube_id, metadata, now) + if state is None: + return NodeMetadataUpdate(None, MemoryLifecycleOutcome(item, "skip", "not_dream")) + updated, extra, action = transition(state, metadata) + fields = {**self._fields(updated), **extra} + return NodeMetadataUpdate( + fields, MemoryLifecycleOutcome(item, action, action, updated != state) + ) + + row = self.graph.mutate_nodes_metadata([memory_id], reduce, user_name=cube_id)[0] + return ( + row.result if row.found else MemoryLifecycleOutcome(memory_id, "missing", "not_found") + ) + + def archive( + self, memory_id: str, *, cube_id: str, reason: str, now: datetime + ) -> MemoryLifecycleOutcome: + """Explicitly archive a Dream memory without pretending this is causal attribution.""" + now = utc_time(now) + if not reason.strip(): + raise LifecycleError("Explicit archive requires a reason") + + def transition(state, metadata): + if metadata.get("status") not in ("activated", "archived"): + raise LifecycleError("Cannot archive a memory with an unrelated external status") + if state.protect_archived_status: + return state, {}, "archive" + updated = LifecycleState.model_validate( + { + **state.model_dump(), + "status": "archived", + "protect_archived_status": True, + "archived_at": now, + "archive_reason": reason, + "revision": state.revision + 1, + } + ) + return updated, {}, "archive" + + return self._transition(memory_id, cube_id, now, transition) + + def restore( + self, + memory_id: str, + *, + cube_id: str, + reason: str, + now: datetime, + override_invalidation: bool = False, + ) -> MemoryLifecycleOutcome: + """Restore only lifecycle-owned archives with an explicit revalidation reason.""" + now = utc_time(now) + + def transition(state, metadata): + if not state.protect_archived_status or metadata.get("status") != "archived": + raise LifecycleError("Only lifecycle-owned archives can be restored") + updated = restore_state( + state, + self.config, + now=now, + reason=reason, + override_invalidation=override_invalidation, + ) + return ( + updated, + {"status": "activated", "restored_at": now.isoformat(), "restore_reason": reason}, + "restore", + ) + + return self._transition(memory_id, cube_id, now, transition) + + def set_pinned( + self, memory_id: str, *, cube_id: str, pinned: bool, now: datetime + ) -> MemoryLifecycleOutcome: + """Exempt a memory from passive retention decisions; invalidation still wins.""" + if not isinstance(pinned, bool): + raise LifecycleError("Pinned must be a boolean") + + def transition(state, metadata): + updated = ( + state + if state.pinned == pinned + else LifecycleState.model_validate( + {**state.model_dump(), "pinned": pinned, "revision": state.revision + 1} + ) + ) + return updated, {}, "pin" if pinned else "unpin" + + return self._transition(memory_id, cube_id, now, transition) diff --git a/src/memos/dream/maintenance.py b/src/memos/dream/maintenance.py index db859c011..000c26da7 100644 --- a/src/memos/dream/maintenance.py +++ b/src/memos/dream/maintenance.py @@ -1,56 +1,29 @@ -"""Dream memory lifecycle maintenance. +"""Opt-in Dream lifecycle maintenance on the native atomic PostgreSQL backend. -This module is responsible for periodic cleanup of Dream-produced memories. -It is intentionally left unimplemented as a contribution entry point. +Configure CommunityDreamPlugin through configs.dream_lifecycle, explicitly +register cube scopes, then call plugin.lifecycle.run_maintenance(cube_id). +A positive interval_seconds enables periodic maintenance of those same scopes; +zero keeps it manual. No new HTTP route or implicit tenant enumeration is added. -──────────────────────────────────────────────────────────────────────────── -CONTRIBUTION GUIDE — Dream Memory Maintenance -──────────────────────────────────────────────────────────────────────────── - -Dream memories are not permanent. Each Dream-produced memory carries -`DreamMemoryLifecycle` metadata (see types.py) and should be periodically -evaluated for continued relevance. - -Cleanup rules to implement: - -1. STALE — Long time not hit - - If `last_hit_at` is older than a configurable TTL (e.g. 7 days), - the memory should be decayed or archived. - - Suggested approach: query graph_db for nodes where - metadata.source == "dream" and lifecycle.last_hit_at < threshold. - -2. LOW USEFULNESS — Hit but not helpful - - If the memory is retrieved during queries but `usefulness_score` - remains below a threshold, it is noise rather than signal. - - Suggested approach: integrate with the retrieval layer to update - usefulness_score when a Dream memory is included in a response - but receives low relevance feedback. - -3. INVALIDATED — Overturned by feedback - - If `invalidated_by_feedback` is set to True (e.g. by the feedback - API or a contradicting new memory), archive immediately. - - Suggested approach: register a hook on the feedback endpoint to - mark conflicting Dream memories. - -Implementation hints: - -- Create a `DreamMaintenanceTask` that runs on a scheduler interval - (e.g. daily) or is triggered by a hook. -- Use `graph_db.update_node(id, None, {"status": "archived", ...})` to - archive memories that fail the above checks. -- Consider a gradual decay: lower confidence over time rather than - hard-deleting on first miss. -- The `DreamMemoryLifecycle` model in types.py already provides the - fields you need: `last_hit_at`, `hit_count`, `usefulness_score`, - `invalidated_by_feedback`, `status`. - -Entry point suggestion: - - class DreamMaintenanceStrategy: - def run_maintenance(self, *, user_name: str, mem_cube_id: str) -> MaintenanceReport: - ... - -Register it in plugin.py and wire it to a scheduler interval or a -dedicated API endpoint (e.g. POST /dream/maintenance/run). -──────────────────────────────────────────────────────────────────────────── +The public runner can also be used directly with DreamLifecycleService. Pass +stable event IDs and original timestamps to the service for SDK observation; +automatic final-result notifications cover the decorated API search path only. """ + +from memos.dream.lifecycle.runner import ( + DreamMaintenanceRunner, + MaintenanceLimits, + MaintenanceReport, +) +from memos.dream.lifecycle.runtime import DreamLifecycleRuntime, LifecycleRuntimeConfig +from memos.dream.lifecycle.service import DreamLifecycleService + + +__all__ = [ + "DreamLifecycleRuntime", + "DreamLifecycleService", + "DreamMaintenanceRunner", + "LifecycleRuntimeConfig", + "MaintenanceLimits", + "MaintenanceReport", +] diff --git a/src/memos/dream/pipeline/persistence.py b/src/memos/dream/pipeline/persistence.py index 1ab5adcdc..cf97f0aea 100644 --- a/src/memos/dream/pipeline/persistence.py +++ b/src/memos/dream/pipeline/persistence.py @@ -2,7 +2,7 @@ import logging -from datetime import datetime +from datetime import datetime, timezone from typing import Any from memos.dream.hook_defs import DreamH @@ -172,6 +172,15 @@ def _dispatch_action( dream_metadata=dream_metadata, ) + def _lifecycle_metadata(self, memory_id: str, cube_id: str, metadata: dict) -> dict: + """Initialize opted-in evidence as part of the native node write.""" + service = self.context.get("shared", {}).get("dream_lifecycle") + if service is None: + return metadata + return service.metadata_for_new( + memory_id, cube_id, metadata, now=datetime.now(timezone.utc) + ) + def _write_memory_create(self, *, action, text_mem, cube_id, dream_metadata) -> None: """Create a new memory node from Dream reasoning output.""" graph_db = self.context.get("shared", {}).get("graph_db") @@ -191,6 +200,7 @@ def _write_memory_create(self, *, action, text_mem, cube_id, dream_metadata) -> **dream_metadata, **action.metadata, } + metadata = self._lifecycle_metadata(node_id, cube_id, metadata) graph_db.add_node(node_id, action.new_content, metadata, user_name=cube_id) logger.info( "[Dream] created memory node: id=%s, type=%s", @@ -213,8 +223,7 @@ def _write_memory_update(self, *, action, text_mem, cube_id, dream_metadata) -> try: graph_db.update_node( action.target_memory_id, - action.new_content, - metadata, + {**metadata, "memory": action.new_content}, user_name=cube_id, ) logger.info("[Dream] updated memory: id=%s", action.target_memory_id) @@ -240,13 +249,13 @@ def _write_memory_merge(self, *, action, text_mem, cube_id, dream_metadata) -> N **dream_metadata, **action.metadata, } + metadata = self._lifecycle_metadata(merged_id, cube_id, metadata) graph_db.add_node(merged_id, action.new_content, metadata, user_name=cube_id) for src_id in action.source_memory_ids: try: graph_db.update_node( src_id, - None, {"status": "merged", "merged_into": merged_id}, user_name=cube_id, ) @@ -262,9 +271,18 @@ def _write_memory_archive(self, *, action, text_mem, cube_id, dream_metadata) -> return try: + lifecycle = self.context.get("shared", {}).get("dream_lifecycle") + if lifecycle is not None: + outcome = lifecycle.archive( + action.target_memory_id, + cube_id=cube_id, + reason=action.rationale, + now=datetime.now(timezone.utc), + ) + if outcome.action != "skip": + return graph_db.update_node( action.target_memory_id, - None, { "status": "archived", "archived_at": datetime.utcnow().isoformat(), @@ -303,6 +321,7 @@ def _persist_diary( mem_cube_id=mem_cube_id, signal_snapshot=signal_snapshot, ) + metadata = self._lifecycle_metadata(entry.diary_id, cube_id, metadata) try: graph_db.add_node( entry.diary_id, diff --git a/src/memos/dream/plugin.py b/src/memos/dream/plugin.py index ef3291d53..8fc2c1528 100644 --- a/src/memos/dream/plugin.py +++ b/src/memos/dream/plugin.py @@ -12,6 +12,8 @@ on_memory_items_after_fine_extract, ) from memos.dream.hooks import on_add_signal, on_dream_execute +from memos.dream.lifecycle.contracts import LifecycleError +from memos.dream.lifecycle.runtime import DreamLifecycleRuntime, LifecycleRuntimeConfig from memos.dream.pipeline import ( AbstractDreamPipeline, ConsolidationReasoning, @@ -49,6 +51,11 @@ class CommunityDreamPlugin(MemOSPlugin): enabled_by_default = False def on_load(self) -> None: + if getattr(self, "_loaded", False): + return + self._loaded = True + self._owned_hooks = [] + self.lifecycle: DreamLifecycleRuntime | None = None self.context: dict[str, Any] = {"shared": {}, "configs": {}} self.signal_store = DreamSignalStore() self.heuristic_enricher = DreamHeuristicEnricher() @@ -64,20 +71,52 @@ def on_load(self) -> None: # Hook registration happens at load time because scheduler-triggered Dream # execution does not depend on FastAPI route binding. - self.register_hook(H.DREAM_EXECUTE, partial(on_dream_execute, self)) - self.register_hook(H.SEARCH_MEMORY_RESULTS, self.search_extension.merge_context_recall) - self.register_hook(H.ADD_AFTER, partial(on_add_signal, self)) - self.register_hook( + self._register_owned_hook(H.DREAM_EXECUTE, partial(on_dream_execute, self)) + self._register_owned_hook( + H.SEARCH_MEMORY_RESULTS, self.search_extension.merge_context_recall + ) + self._register_owned_hook(H.ADD_AFTER, partial(on_add_signal, self)) + self._register_owned_hook( H.MEMORY_ITEMS_AFTER_FINE_EXTRACT, partial(on_memory_items_after_fine_extract, self), ) logger.info("[Dream] plugin loaded") + def _register_owned_hook(self, name, callback) -> None: + self.register_hook(name, callback) + self._owned_hooks.append((name, callback)) + def init_components(self, context: dict) -> None: # Keep the mutable context reference directly. The server bootstrap updates # scheduler handles later, and the plugin should see those changes in-place. + if self.lifecycle is not None: + if context is self.context: + return + raise LifecycleError( + "Shutdown the active lifecycle runtime before rebinding components" + ) + config = LifecycleRuntimeConfig.model_validate( + context.get("configs", {}).get("dream_lifecycle") or {} + ) + runtime = None + shared = context.get("shared", {}) + if config.enabled: + if shared.get("dream_lifecycle") is not None: + raise LifecycleError("A Dream lifecycle service is already registered") + graph = shared.get("graph_db") + if graph is None: + raise LifecycleError("Dream lifecycle requires a shared graph backend") + runtime = DreamLifecycleRuntime(graph, config) self.context = context self.pipeline.bind_context(context) + if runtime is not None: + self.lifecycle = runtime + shared["dream_lifecycle"] = runtime.service + self._register_owned_hook(H.SEARCH_COMPLETED, runtime.observer.on_search_completed) + self._register_owned_hook( + H.MEMORY_FEEDBACK_COMMITTED, runtime.observer.on_feedback_committed + ) + runtime.start() def init_app(self) -> None: self.register_router(create_trigger_router(self)) @@ -85,6 +124,21 @@ def init_app(self) -> None: logger.info("[Dream] plugin initialized") def on_shutdown(self) -> None: + from memos.plugins.hooks import unregister_hook + + for name, callback in self._owned_hooks: + unregister_hook(name, callback) + self._owned_hooks.clear() + if self.lifecycle is not None: + if not self.lifecycle.shutdown(): + raise LifecycleError( + "Dream maintenance is still stopping; retain its runtime until complete" + ) + shared = self.context.get("shared", {}) + if shared.get("dream_lifecycle") is self.lifecycle.service: + shared.pop("dream_lifecycle") + self.lifecycle = None + self._loaded = False self.context = {"shared": {}, "configs": {}} self.heuristic_enricher = None logger.info("[Dream] plugin shutdown") diff --git a/src/memos/graph_dbs/base.py b/src/memos/graph_dbs/base.py index 0bc4a54f8..bd2e9b166 100644 --- a/src/memos/graph_dbs/base.py +++ b/src/memos/graph_dbs/base.py @@ -1,17 +1,89 @@ import re from abc import ABC, abstractmethod +from collections.abc import Callable +from copy import deepcopy +from dataclasses import dataclass from typing import Any, Literal +from memos.exceptions import MemoryError as MemOSMemoryError + + +MANAGED_LIFECYCLE_KEY = "_memos_dream_lifecycle_v1" +MANAGED_LIFECYCLE_VERSION = 1 + + +@dataclass(frozen=True) +class NodeMetadataUpdate: + """Pure metadata patch and caller result; None requests a read-only operation.""" + + fields: dict[str, Any] | None + result: Any = None + + +@dataclass(frozen=True) +class NodeMetadataResult: + """Outcome of one scoped, transactionally committed metadata reduction.""" + + memory_id: str + found: bool + changed: bool = False + result: Any = None + + +def preserve_managed_metadata(current: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]: + """Keep server-owned lifecycle evidence across ordinary graph writes. + + The storage layer deliberately knows only the reserved envelope and its + archive guard, avoiding a dependency on Dream policy. Unknown/malformed + envelopes retain their current status until an explicit lifecycle repair. + """ + merged = deepcopy(incoming) + if MANAGED_LIFECYCLE_KEY in current: + state = current[MANAGED_LIFECYCLE_KEY] + merged[MANAGED_LIFECYCLE_KEY] = deepcopy(state) + recognized_unprotected = ( + isinstance(state, dict) + and type(state.get("version")) is int + and state["version"] == MANAGED_LIFECYCLE_VERSION + and state.get("protect_archived_status") is False + ) + if not recognized_unprotected: + if "status" in current: + merged["status"] = current["status"] + else: + merged.pop("status", None) + return merged + # Pattern for valid field names: alphanumeric and underscores, must start with letter or underscore _VALID_FIELD_NAME_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") class BaseGraphDB(ABC): - """ - Abstract base class for a graph database interface used in a memory-augmented RAG system. - """ + """Graph storage interface for memory-augmented retrieval.""" + + supports_atomic_lifecycle = False + + def mutate_nodes_metadata( + self, + ids: list[str], + reducer: Callable[[str, dict[str, Any]], NodeMetadataUpdate], + *, + user_name: str, + ) -> list[NodeMetadataResult]: + """Reduce current scoped metadata under locks, or reject unsupported backends. + + Reducers must be pure and bounded: no network calls, recursive database + access or external side effects. A reducer failure rolls back the batch. + """ + raise MemOSMemoryError("This graph backend lacks atomic lifecycle metadata mutation") + + def list_lifecycle_nodes( + self, *, user_name: str, after: str | None = None, limit: int = 100 + ) -> list[dict[str, Any]]: + """Read one scoped, bounded keyset page without memory content/embeddings.""" + raise MemOSMemoryError("This graph backend lacks lifecycle keyset pagination") @staticmethod def _validate_return_fields(return_fields: list[str] | None) -> list[str]: diff --git a/src/memos/graph_dbs/postgres.py b/src/memos/graph_dbs/postgres.py index 2820a8b54..0c372c9be 100644 --- a/src/memos/graph_dbs/postgres.py +++ b/src/memos/graph_dbs/postgres.py @@ -15,13 +15,23 @@ import re import time -from contextlib import suppress +from collections.abc import Callable +from contextlib import contextmanager, suppress +from copy import deepcopy from datetime import datetime, timezone from typing import Any, Literal from memos.configs.graph_db import PostgresGraphDBConfig from memos.dependency import require_python_package -from memos.graph_dbs.base import BaseGraphDB +from memos.exceptions import MemoryError as MemOSMemoryError +from memos.graph_dbs.base import ( + MANAGED_LIFECYCLE_KEY, + MANAGED_LIFECYCLE_VERSION, + BaseGraphDB, + NodeMetadataResult, + NodeMetadataUpdate, + preserve_managed_metadata, +) from memos.log import get_logger @@ -45,6 +55,7 @@ def _prepare_node_metadata(metadata: dict[str, Any]) -> dict[str, Any]: class PostgresGraphDB(BaseGraphDB): """PostgreSQL + pgvector implementation of a graph memory store.""" + supports_atomic_lifecycle = True supports_lightweight_vector_search = True prefer_lightweight_vector_search = True @@ -451,15 +462,30 @@ def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None: with conn.cursor() as cur: cur.execute( f""" - INSERT INTO {memories} + INSERT INTO {memories} AS existing (id, memory, properties, embedding, user_name, created_at, updated_at) VALUES {values_sql} ON CONFLICT (user_name, id) DO UPDATE SET memory = EXCLUDED.memory, - properties = EXCLUDED.properties, + properties = CASE + WHEN existing.properties ? '{MANAGED_LIFECYCLE_KEY}' THEN + (CASE + WHEN jsonb_typeof(existing.properties->'{MANAGED_LIFECYCLE_KEY}'->'version') = 'number' + AND existing.properties->'{MANAGED_LIFECYCLE_KEY}'->>'version' = '{MANAGED_LIFECYCLE_VERSION}' + AND existing.properties->'{MANAGED_LIFECYCLE_KEY}'->'protect_archived_status' = 'false'::jsonb + THEN EXCLUDED.properties + ELSE (EXCLUDED.properties - 'status') || + CASE WHEN existing.properties ? 'status' + THEN jsonb_build_object('status', existing.properties->'status') + ELSE '{{}}'::jsonb END + END) || jsonb_build_object( + '{MANAGED_LIFECYCLE_KEY}', existing.properties->'{MANAGED_LIFECYCLE_KEY}' + ) + ELSE EXCLUDED.properties + END, embedding = EXCLUDED.embedding, updated_at = EXCLUDED.updated_at - WHERE "Memory".user_name = EXCLUDED.user_name + WHERE existing.user_name = EXCLUDED.user_name """, params, ) @@ -467,46 +493,167 @@ def add_nodes_batch(self, nodes: list[dict[str, Any]], user_name: str) -> None: self._put_conn(conn) def update_node(self, id: str, fields: dict[str, Any], user_name: str | None = None) -> None: - """Update node fields.""" - user_name = user_name or self.user_name - if not fields: - return - fields = fields.copy() + """Merge current properties under a row lock without erasing lifecycle evidence.""" + from psycopg2 import sql - # Get current node - current = self.get_node(id, user_name=user_name) - if not current: + if not fields: return - - # Merge properties - props = current.get("metadata", {}).copy() + user_name = user_name or self.user_name + fields = deepcopy(fields) embedding = fields.pop("embedding", None) - memory = fields.pop("memory", current.get("memory", "")) - props.update(fields) - props["updated_at"] = datetime.now(timezone.utc).isoformat() - memories = self._physical_table("memories", user_name) + with self._metadata_transaction() as conn, conn.cursor() as cur: + table = sql.SQL(self._physical_table("memories", user_name, conn)) + cur.execute( + sql.SQL( + "SELECT memory, properties FROM {} WHERE id = %s AND user_name = %s FOR UPDATE" + ).format(table), + (id, user_name), + ) + row = cur.fetchone() + if row is None: + return + memory = fields.pop("memory", row[0]) + properties = preserve_managed_metadata(row[1], {**row[1], **fields}) + properties["updated_at"] = datetime.now(timezone.utc).isoformat() + parameters = [memory, json.dumps(properties, allow_nan=False)] + embedding_clause = sql.SQL("") + if embedding: + embedding_clause = sql.SQL(f", embedding = %s::halfvec({self._vector_dim()})") + parameters.append(embedding) + parameters.extend([id, user_name]) + cur.execute( + sql.SQL( + "UPDATE {} SET memory = %s, properties = %s::jsonb{}" + ", updated_at = NOW() WHERE id = %s AND user_name = %s" + ).format(table, embedding_clause), + parameters, + ) + @contextmanager + def _metadata_transaction(self): + """Borrow one connection, roll back failures, and restore pool state.""" conn = self._get_conn() + reusable = True try: - with conn.cursor() as cur: - if embedding: - cur.execute( - f""" - UPDATE {memories} - SET memory = %s, properties = %s, embedding = %s::halfvec({self._vector_dim()}), updated_at = NOW() - WHERE id = %s AND user_name = %s - """, - (memory, json.dumps(props), embedding, id, user_name), - ) - else: + conn.autocommit = False + yield conn + conn.commit() + except BaseException: + try: + conn.rollback() + except Exception: + reusable = False + raise + finally: + if reusable: + try: + conn.autocommit = True + except Exception: + reusable = False + if reusable: + self._put_conn(conn) + else: + with suppress(Exception): + self.pool.putconn(conn, close=True) + with suppress(Exception): + conn.close() + + @staticmethod + def _lifecycle_scope(user_name: str) -> None: + if not isinstance(user_name, str) or not user_name.strip(): + raise MemOSMemoryError("Lifecycle operations require an explicit scope") + + def mutate_nodes_metadata( + self, + ids: list[str], + reducer: Callable[[str, dict[str, Any]], NodeMetadataUpdate], + *, + user_name: str, + ) -> list[NodeMetadataResult]: + """Atomically reduce a bounded scoped batch using stable row-lock ordering. + + Missing nodes return found=False. Every callback sees a detached current + snapshot. None/empty patches do not update timestamps; callback or SQL + failure rolls back all prior updates in this batch. + """ + from psycopg2 import sql + + self._lifecycle_scope(user_name) + if len(ids) > 1000 or any(not isinstance(item, str) or not item for item in ids): + raise MemOSMemoryError("Lifecycle batches require at most 1000 nonempty IDs") + ordered = list(dict.fromkeys(ids)) + if not ordered: + return [] + results = {} + with self._metadata_transaction() as conn, conn.cursor() as cur: + table = sql.SQL(self._physical_table("memories", user_name, conn)) + cur.execute( + sql.SQL( + "SELECT id, properties, created_at, updated_at FROM {} " + 'WHERE user_name = %s AND id = ANY(%s) ORDER BY id COLLATE "C" ' + "FOR UPDATE" + ).format(table), + (user_name, ordered), + ) + rows = cur.fetchall() + for memory_id, properties, created_at, updated_at in rows: + snapshot = deepcopy(properties) + snapshot.setdefault("created_at", created_at.isoformat()) + snapshot.setdefault("updated_at", updated_at.isoformat()) + patch = reducer(memory_id, snapshot) + if not isinstance(patch, NodeMetadataUpdate): + raise MemOSMemoryError("Metadata reducer must return NodeMetadataUpdate") + if patch.fields is not None and not isinstance(patch.fields, dict): + raise MemOSMemoryError("Metadata patch must be a dictionary or None") + changed = bool(patch.fields) and any( + key not in properties or properties[key] != value + for key, value in patch.fields.items() + ) + if changed: + merged = {**properties, **patch.fields} cur.execute( - f""" - UPDATE {memories} - SET memory = %s, properties = %s, updated_at = NOW() - WHERE id = %s AND user_name = %s - """, - (memory, json.dumps(props), id, user_name), + sql.SQL( + "UPDATE {} SET properties = %s::jsonb, " + "updated_at = NOW() WHERE id = %s AND user_name = %s" + ).format(table), + (json.dumps(merged, allow_nan=False), memory_id, user_name), ) + results[memory_id] = NodeMetadataResult( + memory_id, True, bool(changed), patch.result + ) + return [results.get(item, NodeMetadataResult(item, False)) for item in ordered] + + def list_lifecycle_nodes( + self, *, user_name: str, after: str | None = None, limit: int = 100 + ) -> list[dict[str, Any]]: + """Fetch metadata-only Dream candidates in a stable scoped keyset page.""" + from psycopg2 import sql + + self._lifecycle_scope(user_name) + if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000: + raise MemOSMemoryError("Lifecycle page limit must be between 1 and 1000") + if after is not None and (not isinstance(after, str) or not after): + raise MemOSMemoryError("Lifecycle cursor must be a nonempty ID") + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + sql.SQL(""" + SELECT id, properties, created_at, updated_at FROM {table} + WHERE user_name = %s AND (%s::text IS NULL OR id COLLATE "C" > %s) + AND (properties ? %s OR properties->>'source' = 'dream' + OR properties->'internal_info' ? 'dream') + ORDER BY id COLLATE "C" LIMIT %s + """).format(table=sql.SQL(self._physical_table("memories", user_name, conn))), + (user_name, after, after, MANAGED_LIFECYCLE_KEY, limit), + ) + result = [] + for memory_id, properties, created_at, updated_at in cur.fetchall(): + metadata = dict(properties) + metadata.setdefault("created_at", created_at.isoformat()) + metadata.setdefault("updated_at", updated_at.isoformat()) + result.append({"id": memory_id, "metadata": metadata}) + return result finally: self._put_conn(conn) diff --git a/src/memos/mem_feedback/feedback.py b/src/memos/mem_feedback/feedback.py index 18ebb0b49..72ddc05b1 100644 --- a/src/memos/mem_feedback/feedback.py +++ b/src/memos/mem_feedback/feedback.py @@ -4,7 +4,7 @@ import re import uuid -from datetime import datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Literal from tenacity import retry, stop_after_attempt, wait_random_exponential @@ -12,6 +12,7 @@ from memos.configs.memory import MemFeedbackConfig from memos.context.context import ContextThreadPoolExecutor from memos.dependency import require_python_package +from memos.dream.lifecycle.contracts import detach_lifecycle_identity from memos.embedders.factory import EmbedderFactory, OllamaEmbedder from memos.graph_dbs.factory import GraphStoreFactory, PolarDBGraphDB from memos.llms.factory import AzureLLM, LLMFactory, OllamaLLM, OpenAILLM @@ -34,7 +35,7 @@ ) from memos.memories.textual.tree_text_memory.retrieve.retrieve_utils import StopwordManager from memos.plugins.hook_defs import H -from memos.plugins.hooks import trigger_single_hook +from memos.plugins.hooks import notify_hook, trigger_single_hook if TYPE_CHECKING: @@ -240,12 +241,17 @@ def _single_add_operation( to_add_memory = new_memory_item.model_copy(deep=True) to_add_memory.id = str(uuid.uuid4()) + fresh_metadata = detach_lifecycle_identity( + to_add_memory.metadata.model_dump(), + source_memory_id=(old_memory_item or new_memory_item).id, + ) + to_add_memory.metadata = type(to_add_memory.metadata).model_validate(fresh_metadata) if to_add_memory.metadata.memory_type == "PreferenceMemory": to_add_memory.metadata.preference = new_memory_item.memory - to_add_memory.metadata.created_at = to_add_memory.metadata.updated_at = ( - datetime.now().isoformat() - ) + to_add_memory.metadata.created_at = to_add_memory.metadata.updated_at = datetime.now( + timezone.utc + ).isoformat() to_add_memory.metadata.background = new_memory_item.metadata.background to_add_memory.metadata.sources = new_memory_item.metadata.sources @@ -328,6 +334,14 @@ def _single_update_operation( f"[Memory Feedback UPDATE] Updated:{item_id} | history appended | memory_type: {old_memory_item.metadata.memory_type}" ) + notify_hook( + H.MEMORY_FEEDBACK_COMMITTED, + cube_id=user_name, + memory_ids=[old_memory_item.id], + action="superseded" if item_id != old_memory_item.id else "corrected", + event_id="feedback:" + uuid.uuid4().hex, + at=datetime.now(timezone.utc), + ) return { "id": item_id, "text": new_memory_item.memory, @@ -386,6 +400,10 @@ def _single_update_operation_with_versions( if archived_item and archived_metadata: try: + if archived_item.id != old_memory_item.id: + archived_metadata = detach_lifecycle_identity( + archived_metadata, source_memory_id=old_memory_item.id + ) self.graph_store.add_node( id=archived_item.id, memory=archived_item.memory, diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index a874ac77b..f92d3d291 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -136,7 +136,7 @@ class TextualMemoryMetadata(BaseModel): default=None, description="A numeric score (float between 0 and 100) indicating how certain you are about the accuracy or reliability of the memory.", ) - source: Literal["conversation", "retrieved", "web", "file", "system"] | None = Field( + source: Literal["conversation", "retrieved", "web", "file", "system", "dream"] | None = Field( default=None, description="The origin of the memory" ) tags: list[str] | None = Field( @@ -335,6 +335,13 @@ class TextualMemoryItem(BaseModel): @field_validator("id") @classmethod def _validate_id(cls, v: str) -> str: + # Native Dream writers namespace UUIDs to distinguish actions, diaries + # and Context nodes. Keep validating their UUID payload, rather than + # admitting arbitrary graph identifiers into the textual memory API. + for prefix in ("dream_mem_", "dream_merged_", "dream_diary_", "ctx_"): + if v.startswith(prefix): + uuid.UUID(v[len(prefix) :]) + return v uuid.UUID(v) return v diff --git a/src/memos/plugins/hook_defs.py b/src/memos/plugins/hook_defs.py index f955da0af..c7f2ced43 100644 --- a/src/memos/plugins/hook_defs.py +++ b/src/memos/plugins/hook_defs.py @@ -71,6 +71,8 @@ class H: # @hookable("search") — SearchHandler.handle_search_memories SEARCH_BEFORE = "search.before" SEARCH_AFTER = "search.after" + SEARCH_COMPLETED = "search.completed" + MEMORY_FEEDBACK_COMMITTED = "memory.feedback.committed" # Search extension point before core threshold/dedup/rerank processing. SEARCH_MEMORY_RESULTS = "search.memory_results" @@ -183,3 +185,10 @@ class H: "scheduler_context", ], ) + + +define_hook( + H.MEMORY_FEEDBACK_COMMITTED, + description="Observe explicitly affected IDs after a successful feedback update", + params=["cube_id", "memory_ids", "action", "event_id", "at"], +) diff --git a/src/memos/plugins/hooks.py b/src/memos/plugins/hooks.py index d5b64ce82..a1886421c 100644 --- a/src/memos/plugins/hooks.py +++ b/src/memos/plugins/hooks.py @@ -6,8 +6,12 @@ import logging from collections import defaultdict +from copy import deepcopy +from dataclasses import dataclass +from datetime import datetime, timezone from functools import wraps from typing import TYPE_CHECKING, Any +from uuid import uuid4 if TYPE_CHECKING: @@ -37,6 +41,48 @@ def register_hook(name: str, callback: Callable) -> None: ) +def unregister_hook(name: str, callback: Callable) -> None: + """Remove one owned registration, retaining other plugins' callbacks.""" + callbacks = _hooks.get(name, []) + for index, registered in enumerate(callbacks): + if registered is callback: + del callbacks[index] + break + if not callbacks: + _hooks.pop(name, None) + + +@dataclass(frozen=True) +class HookDelivery: + """Synchronous notification delivery outcomes, separate from business success.""" + + attempted: int = 0 + failed: int = 0 + + +def notify_hook(name: str, **kwargs: Any) -> HookDelivery: + """Notify observers with separate detached snapshots and ignore return values. + + Unlike pipes, notifications cannot mutate the returned payload or another + observer's view. Delivery failures are logged/counted, not transactionally + coupled to a preceding successful operation. No snapshot is made when idle. + """ + from memos.plugins.hook_defs import get_hook_spec + + spec = get_hook_spec(name) + if spec is None or spec.pipe_key is not None: + raise ValueError("Notifications require a declared non-pipe hook") + callbacks = tuple(_hooks.get(name, ())) + failed = 0 + for callback in callbacks: + try: + callback(**deepcopy(kwargs)) + except Exception: + failed += 1 + logger.exception("Notification %s callback failed", name) + return HookDelivery(len(callbacks), failed) + + def register_hooks(names: list[str], callback: Callable) -> None: """Batch-register the same callback to multiple hook points.""" for name in names: @@ -122,6 +168,12 @@ def hookable(name: str): pipe_key="result", ) + define_hook( + f"{name}.completed", + description=f"Observe the final {name} result after every result pipe", + params=["request", "result", "event_id", "at"], + ) + def decorator(func): if asyncio.iscoroutinefunction(func): @@ -132,6 +184,14 @@ async def async_wrapper(self, request, *args, **kwargs): result = await func(self, request, *args, **kwargs) rv = trigger_hook(f"{name}.after", request=request, result=result) result = rv if rv is not None else result + if _hooks.get(f"{name}.completed"): + notify_hook( + f"{name}.completed", + request=request, + result=result, + event_id=f"{name}:" + uuid4().hex, + at=datetime.now(timezone.utc), + ) return result return async_wrapper @@ -143,6 +203,14 @@ def sync_wrapper(self, request, *args, **kwargs): result = func(self, request, *args, **kwargs) rv = trigger_hook(f"{name}.after", request=request, result=result) result = rv if rv is not None else result + if _hooks.get(f"{name}.completed"): + notify_hook( + f"{name}.completed", + request=request, + result=result, + event_id=f"{name}:" + uuid4().hex, + at=datetime.now(timezone.utc), + ) return result return sync_wrapper diff --git a/tests/dream/conftest.py b/tests/dream/conftest.py new file mode 100644 index 000000000..db325016e --- /dev/null +++ b/tests/dream/conftest.py @@ -0,0 +1,45 @@ +"""Optional isolated native PostgreSQL fixture for Dream lifecycle tests.""" + +import os + +from uuid import uuid4 + +import pytest + +from memos.configs.graph_db import PostgresGraphDBConfig +from memos.graph_dbs.postgres import PostgresGraphDB + + +@pytest.fixture +def graph(): + dsn = os.environ.get("MEMOS_TEST_PG_DSN") + if not dsn: + pytest.skip("MEMOS_TEST_PG_DSN not configured") + psycopg2 = pytest.importorskip("psycopg2") + from psycopg2 import sql + from psycopg2.extensions import parse_dsn + + settings = parse_dsn(dsn) + schema = "dream_lifecycle_" + uuid4().hex + config = PostgresGraphDBConfig( + host=settings.get("host", "localhost"), + port=int(settings.get("port", 5432)), + user=settings["user"], + password=settings.get("password", ""), + db_name=settings["dbname"], + schema_name=schema, + user_name="cube-a", + embedding_dimension=3, + shard_count=2, + fulltext_config="simple", + ) + backend = PostgresGraphDB(config) + yield backend + backend.close() + conn = psycopg2.connect(dsn) + conn.autocommit = True + with conn.cursor() as cur: + for remainder in range(config.shard_count): + shard = f"{schema}_graph_{remainder}" + cur.execute(sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(sql.Identifier(shard))) + conn.close() diff --git a/tests/dream/test_lifecycle_observation.py b/tests/dream/test_lifecycle_observation.py new file mode 100644 index 000000000..86912bd82 --- /dev/null +++ b/tests/dream/test_lifecycle_observation.py @@ -0,0 +1,324 @@ +"""Final-result notifications and successful native feedback attribution.""" + +import logging + +from datetime import datetime, timezone +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from memos.api.handlers.search_handler import SearchHandler +from memos.api.product_models import APISearchRequest +from memos.dream.lifecycle import LIFECYCLE_METADATA_KEY, LifecycleConfig +from memos.dream.lifecycle.observation import DreamLifecycleObserver +from memos.dream.lifecycle.service import DreamLifecycleService +from memos.mem_feedback.feedback import MemFeedback +from memos.memories.textual.item import TextualMemoryItem +from memos.plugins import hooks +from memos.plugins.hook_defs import H, get_hook_spec + + +@pytest.fixture(autouse=True) +def restore_hooks(): + before = {key: list(value) for key, value in hooks._hooks.items()} + hooks._hooks.clear() + yield + hooks._hooks.clear() + hooks._hooks.update(before) + + +def new_node(graph): + item = str(uuid4()) + graph.add_node( + item, "Synthetic source", {"source": "dream", "status": "activated"}, user_name="cube-a" + ) + return item + + +def observer(graph): + return DreamLifecycleObserver(DreamLifecycleService(graph, LifecycleConfig())) + + +def test_real_search_counts_only_final_returned_ids_after_every_pipe(graph): + kept, removed, injected = new_node(graph), new_node(graph), new_node(graph) + view = SimpleNamespace( + search_memories=lambda request: { + "text_mem": [ + { + "cube_id": "cube-a", + "memories": [ + { + "id": item, + "memory": "Synthetic", + "metadata": {"relativity": 1, "memory_type": "LongTermMemory"}, + } + for item in (kept, removed) + ], + } + ] + } + ) + handler = SearchHandler.__new__(SearchHandler) + handler.logger = logging.getLogger("lifecycle-search-test") + handler.deps = SimpleNamespace(reranker=None) + handler._build_cube_view = lambda request: view + + def final_pipe(request, result): + result.data["text_mem"][0]["memories"] = [{"id": kept}, {"id": kept}] + return result + + def malicious_notification(**kwargs): + kwargs["result"].data["text_mem"][0]["memories"] = [{"id": injected}] + + bridge = observer(graph) + hooks.register_hook(H.SEARCH_AFTER, final_pipe) + hooks.register_hook(H.SEARCH_COMPLETED, malicious_notification) + hooks.register_hook(H.SEARCH_COMPLETED, bridge.on_search_completed) + response = handler.handle_search_memories( + APISearchRequest( + user_id="user-a", + readable_cube_ids=["cube-a"], + query="Fixture", + relativity=0, + dedup="no", + top_k=2, + ) + ) + assert [item["id"] for item in response.data["text_mem"][0]["memories"]] == [kept, kept] + assert ( + graph.get_node(kept, user_name="cube-a")["metadata"][LIFECYCLE_METADATA_KEY]["hit_count"] + == 1 + ) + assert LIFECYCLE_METADATA_KEY not in graph.get_node(removed, user_name="cube-a")["metadata"] + assert LIFECYCLE_METADATA_KEY not in graph.get_node(injected, user_name="cube-a")["metadata"] + assert bridge.delivery_snapshot().accepted == 1 + + +def test_explicit_delivery_replay_keeps_identity_and_foreign_buckets_are_excluded(graph): + item = new_node(graph) + bridge = observer(graph) + request = APISearchRequest(user_id="user-a", readable_cube_ids=["cube-a"], query="Fixture") + payload = { + "text_mem": [ + {"cube_id": "cube-a", "memories": [{"id": item}]}, + {"cube_id": "foreign", "memories": [{"id": item}]}, + ] + } + now = datetime.now(timezone.utc) + bridge.observe_returned(request=request, results=payload, event_id="stable-response", at=now) + bridge.observe_returned(request=request, results=payload, event_id="stable-response", at=now) + snapshot = bridge.delivery_snapshot() + assert snapshot.accepted == 1 and snapshot.duplicates == 1 + + +def test_native_successful_feedback_notifies_only_explicit_affected_id(graph): + old_id, other_id = new_node(graph), new_node(graph) + bridge = observer(graph) + hooks.register_hook(H.MEMORY_FEEDBACK_COMMITTED, bridge.on_feedback_committed) + feedback = MemFeedback.__new__(MemFeedback) + feedback.mem_reader = SimpleNamespace(memory_version_switch="off") + feedback.graph_store = graph + feedback._single_add_operation = lambda *args: {"id": new_node(graph)} + old = TextualMemoryItem(id=old_id, memory="Before", metadata={"memory_type": "LongTermMemory"}) + new = TextualMemoryItem(memory="Corrected", metadata={"memory_type": "LongTermMemory"}) + feedback._single_update_operation(old, new, "user-a", "cube-a") + metadata = graph.get_node(old_id, user_name="cube-a")["metadata"] + assert metadata[LIFECYCLE_METADATA_KEY]["invalidated_by_feedback"] + assert LIFECYCLE_METADATA_KEY not in graph.get_node(other_id, user_name="cube-a")["metadata"] + assert get_hook_spec(H.MEMORY_VERSION_APPLY_FEEDBACK_UPDATE).pipe_key is None + assert len(hooks._hooks.get(H.MEMORY_VERSION_APPLY_FEEDBACK_UPDATE, [])) == 0 + + +def test_failed_native_feedback_emits_no_success_notification(graph, monkeypatch): + item = new_node(graph) + calls = [] + hooks.register_hook(H.MEMORY_FEEDBACK_COMMITTED, lambda **kwargs: calls.append(kwargs)) + feedback = MemFeedback.__new__(MemFeedback) + feedback.mem_reader = SimpleNamespace(memory_version_switch="off") + feedback.graph_store = graph + monkeypatch.setattr( + graph, + "update_node", + lambda *args, **kwargs: (_ for _ in ()).throw(OSError("Synthetic failed write")), + ) + old = TextualMemoryItem(id=item, memory="Before", metadata={"memory_type": "WorkingMemory"}) + new = TextualMemoryItem( + memory="Corrected", metadata={"memory_type": "WorkingMemory", "embedding": [0, 0, 1]} + ) + with pytest.raises(OSError): + feedback._single_update_operation(old, new, "user-a", "cube-a") + assert calls == [] + + +def test_delivery_failure_is_reported_without_mutating_response(graph): + item = new_node(graph) + graph.update_node(item, {LIFECYCLE_METADATA_KEY: {"version": 99}}, user_name="cube-a") + bridge = observer(graph) + hooks.register_hook(H.SEARCH_COMPLETED, bridge.on_search_completed) + from memos.api.product_models import SearchResponse + + request = APISearchRequest(user_id="user-a", readable_cube_ids=["cube-a"], query="Fixture") + response = SearchResponse( + message="Fixture", data={"text_mem": [{"cube_id": "cube-a", "memories": [{"id": item}]}]} + ) + delivery = hooks.notify_hook( + H.SEARCH_COMPLETED, + request=request, + result=response, + event_id="failed-delivery", + at=datetime.now(timezone.utc), + ) + assert delivery.failed == 1 and bridge.delivery_snapshot().failed == 1 + assert response.data["text_mem"][0]["memories"] == [{"id": item}] + + +def test_delayed_notification_checks_receipts_against_delivery_clock(graph): + from datetime import timedelta + + from memos.dream.lifecycle import LifecycleError + + item = new_node(graph) + bridge = observer(graph) + request = APISearchRequest(user_id="user-a", readable_cube_ids=["cube-a"], query="Fixture") + old = datetime.now(timezone.utc) - timedelta(days=2) + graph.update_node( + item, {"created_at": (old - timedelta(days=1)).isoformat()}, user_name="cube-a" + ) + with pytest.raises(LifecycleError, match="too old"): + bridge.observe_returned( + request=request, + results={"text_mem": [{"cube_id": "cube-a", "memories": [{"id": item}]}]}, + event_id="expired-response", + at=old, + ) + assert LIFECYCLE_METADATA_KEY not in graph.get_node(item, user_name="cube-a")["metadata"] + + +def test_real_feedback_replacement_gets_independent_lifecycle_through_manager(graph): + from memos.dream.lifecycle import EventKind, LifecycleEvent + from memos.dream.lifecycle.runner import DreamMaintenanceRunner + from memos.memories.textual.tree_text_memory.organize.manager import MemoryManager + + now = datetime.now(timezone.utc) + service = DreamLifecycleService(graph, LifecycleConfig()) + bridge = DreamLifecycleObserver(service) + hooks.register_hook(H.MEMORY_FEEDBACK_COMMITTED, bridge.on_feedback_committed) + item = str(uuid4()) + metadata = service.metadata_for_new( + item, + "cube-a", + { + "source": "system", + "status": "activated", + "memory_type": "LongTermMemory", + "confidence": 0.7, + }, + now=now, + ) + graph.add_node(item, "Before", metadata, user_name="cube-a") + service.observe( + [ + LifecycleEvent( + memory_id=item, + cube_id="cube-a", + event_id="old-hit", + kind=EventKind.RETRIEVED, + at=now, + ) + ], + now=now, + ) + old = TextualMemoryItem(**graph.get_node(item, user_name="cube-a")) + new = TextualMemoryItem(memory="Corrected", metadata={"memory_type": "LongTermMemory"}) + manager = MemoryManager.__new__(MemoryManager) + manager.graph_store = graph + manager.reorganizer = SimpleNamespace( + add_message=lambda _message: None, + wait_until_current_task_done=lambda: None, + stop=lambda: None, + ) + manager._cleanup_working_memory = lambda _scope: None + feedback = MemFeedback.__new__(MemFeedback) + feedback.graph_store = graph + feedback.mem_reader = SimpleNamespace(memory_version_switch="off") + feedback.memory_manager = manager + feedback._retry_db_operation = lambda callback: callback() + result = feedback._single_update_operation(old, new, "user-a", "cube-a") + replacement = graph.get_node(result["id"], user_name="cube-a")["metadata"] + assert result["id"] != item + assert LIFECYCLE_METADATA_KEY not in replacement + assert replacement["confidence"] == 0.7 and replacement["source"] == "system" + inspection = service.inspect([result["id"]], cube_id="cube-a", now=datetime.now(timezone.utc))[ + 0 + ] + assert inspection.eligible and not inspection.quarantined and inspection.hit_count == 0 + service.observe( + [ + LifecycleEvent( + memory_id=result["id"], + cube_id="cube-a", + event_id="new-hit", + kind=EventKind.RETRIEVED, + at=datetime.now(timezone.utc), + ) + ], + now=datetime.now(timezone.utc), + ) + replacement = graph.get_node(result["id"], user_name="cube-a")["metadata"] + assert replacement[LIFECYCLE_METADATA_KEY]["memory_id"] == result["id"] + assert replacement[LIFECYCLE_METADATA_KEY]["hit_count"] == 1 + assert len(replacement[LIFECYCLE_METADATA_KEY]["receipts"]) == 1 + assert graph.get_node(item, user_name="cube-a")["metadata"][LIFECYCLE_METADATA_KEY][ + "invalidated_by_feedback" + ] + assert ( + DreamMaintenanceRunner(service) + .run(cube_id="cube-a", now=datetime.now(timezone.utc)) + .complete + ) + + +def test_native_version_archive_copy_does_not_inherit_current_identity(graph): + now = datetime.now(timezone.utc) + service = DreamLifecycleService(graph, LifecycleConfig()) + item = str(uuid4()) + metadata = service.metadata_for_new( + item, + "cube-a", + {"source": "dream", "status": "activated", "memory_type": "LongTermMemory"}, + now=now, + ) + graph.add_node(item, "Original", metadata, user_name="cube-a") + old = TextualMemoryItem(id=item, memory="Original", metadata=metadata) + archive = old.model_copy(deep=True) + archive.id = str(uuid4()) + archive.metadata.status = "archived" + + def version_provider(**kwargs): + return old, archive, archive.metadata.model_dump(), {"memory": "Corrected"} + + hooks.register_hook(H.MEMORY_VERSION_APPLY_FEEDBACK_UPDATE, version_provider) + feedback = MemFeedback.__new__(MemFeedback) + feedback.graph_store = graph + feedback._retry_db_operation = lambda callback: callback() + feedback._single_update_operation_with_versions( + old, TextualMemoryItem(memory="Corrected"), "cube-a" + ) + archived = graph.get_node(archive.id, user_name="cube-a")["metadata"] + assert LIFECYCLE_METADATA_KEY not in archived + report = service.inspect([archive.id], cube_id="cube-a", now=datetime.now(timezone.utc))[0] + assert not report.quarantined and report.next_action == "skip" + + +@pytest.mark.parametrize("prefix", ["dream_mem_", "dream_merged_", "dream_diary_", "ctx_"]) +def test_native_dream_identifiers_remain_uuid_validated(prefix): + identifier = prefix + uuid4().hex + assert ( + TextualMemoryItem(id=identifier, memory="Native", metadata={"source": "dream"}).id + == identifier + ) + with pytest.raises(ValueError): + TextualMemoryItem(id=prefix + "not-a-uuid", memory="Invalid") + with pytest.raises(ValueError): + TextualMemoryItem(id="unknown_" + uuid4().hex, memory="Unknown prefix") diff --git a/tests/dream/test_lifecycle_policy.py b/tests/dream/test_lifecycle_policy.py new file mode 100644 index 000000000..f360e5260 --- /dev/null +++ b/tests/dream/test_lifecycle_policy.py @@ -0,0 +1,218 @@ +"""Observable lifecycle policy contracts with fixed clocks and literal oracles.""" + +import math + +from datetime import datetime, timedelta, timezone + +import pytest + +from memos.dream.lifecycle import ( + EventKind, + LifecycleConfig, + LifecycleError, + LifecycleEvent, + apply_event, + decide_maintenance, + initialize_state, + restore_state, +) + + +NOW = datetime(2026, 1, 10, tzinfo=timezone.utc) + + +def config(**overrides): + return LifecycleConfig( + **{ + "half_life_seconds": 86400, + "never_hit_grace_seconds": 86400, + "archive_strength_threshold": 0.2, + "event_max_age_seconds": 86400, + "receipt_capacity": 8, + **overrides, + } + ) + + +def state(*, created_at=NOW, **kwargs): + return initialize_state( + memory_id="memory-a", + cube_id="cube-a", + created_at=created_at, + config=config(), + **kwargs, + ) + + +def event(event_id="event-a", *, at=NOW, kind=EventKind.RETRIEVED, **kwargs): + return LifecycleEvent( + event_id=event_id, + memory_id="memory-a", + cube_id="cube-a", + at=at, + kind=kind, + **kwargs, + ) + + +def test_grace_and_half_life_are_based_on_original_time_not_sweep_count(): + original = state() + first = decide_maintenance(original, config(), now=NOW + timedelta(days=1)) + assert first.action == "keep" + assert first.strength == pytest.approx(0.5) + third_day = decide_maintenance(original, config(), now=NOW + timedelta(days=3)) + assert third_day.action == "archive" + assert third_day.strength == pytest.approx(0.125) + assert decide_maintenance(original, config(), now=NOW + timedelta(days=3)) == third_day + assert original.status == "active" + + +def test_never_hit_grace_prevents_low_prior_premature_archive(): + original = state(base_strength=0.01) + decision = decide_maintenance(original, config(), now=NOW + timedelta(hours=23)) + assert decision.action == "keep" + assert decision.reason == "never_hit_grace" + + +def test_duplicate_observation_does_not_reinforce_or_change_revision(): + observed = apply_event(state(), event(), config(), now=NOW) + duplicate = apply_event(observed, event(), config(), now=NOW + timedelta(minutes=1)) + assert duplicate == observed + assert duplicate.hit_count == 1 + assert duplicate.last_hit_at == NOW + + +def test_conflicting_event_identity_fails_without_mutating_evidence(): + observed = apply_event(state(), event(), config(), now=NOW) + with pytest.raises(LifecycleError, match="conflict"): + apply_event(observed, event(kind=EventKind.POSITIVE), config(), now=NOW) + assert observed.hit_count == 1 + assert observed.positive_mass == 0 + + +def test_delayed_hit_does_not_move_last_hit_back_or_receive_arrival_time(): + original = state(created_at=NOW - timedelta(days=2)) + newer = apply_event(original, event(at=NOW - timedelta(hours=1)), config(), now=NOW) + older = apply_event(newer, event("older", at=NOW - timedelta(hours=3)), config(), now=NOW) + assert older.hit_count == 2 + assert older.last_hit_at == NOW - timedelta(hours=1) + + +def test_delayed_outcome_has_event_time_weight(): + original = state(created_at=NOW - timedelta(days=2)) + result = apply_event( + original, event(kind=EventKind.POSITIVE, at=NOW - timedelta(hours=12)), config(), now=NOW + ) + assert result.positive_mass == pytest.approx(2**-0.5) + assert result.mass_anchor_at == NOW + + +def test_retrieval_never_counts_as_confirmed_usefulness(): + result = apply_event(state(), event(), config(), now=NOW) + assert result.positive_mass == 0 + assert result.negative_mass == 0 + assert decide_maintenance(result, config(), now=NOW).usefulness == 0.5 + + +def test_pinned_memory_still_obeys_explicit_invalidation(): + original = state(pinned=True) + assert decide_maintenance(original, config(), now=NOW + timedelta(days=90)).action == "keep" + invalid = apply_event( + original, event(kind=EventKind.INVALIDATED, reason="corrected fact"), config(), now=NOW + ) + decision = decide_maintenance(invalid, config(), now=NOW) + assert decision.action == "archive" + assert decision.reason == "feedback_invalidated" + + +def test_recovery_needs_explicit_override_for_invalidated_memory(): + invalid = apply_event( + state(), event(kind=EventKind.INVALIDATED, reason="corrected fact"), config(), now=NOW + ) + with pytest.raises(LifecycleError, match="override"): + restore_state(invalid, config(), now=NOW, reason="operator request") + restored = restore_state( + invalid, config(), now=NOW, reason="verified correction", override_invalidation=True + ) + assert not restored.invalidated_by_feedback + assert restored.status == "active" + assert not restored.protect_archived_status + assert restored.restore_reason == "verified correction" + + +def test_live_receipts_are_not_evicted_to_accept_another_event(): + settings = config(receipt_capacity=1) + original = initialize_state( + memory_id="memory-a", cube_id="cube-a", created_at=NOW, config=settings + ) + first = apply_event(original, event(), settings, now=NOW) + with pytest.raises(LifecycleError, match="capacity"): + apply_event(first, event("new"), settings, now=NOW) + assert first.hit_count == 1 + + +def test_expired_receipts_can_be_pruned_but_old_events_cannot_be_replayed(): + settings = config(receipt_capacity=1) + original = initialize_state( + memory_id="memory-a", cube_id="cube-a", created_at=NOW, config=settings + ) + first = apply_event(original, event(), settings, now=NOW) + later = NOW + timedelta(days=2) + second = apply_event(first, event("new", at=later), settings, now=later) + assert second.hit_count == 2 + assert len(second.receipts) == 1 + with pytest.raises(LifecycleError, match="old"): + apply_event(second, event(), settings, now=later) + + +@pytest.mark.parametrize("overrides", [{"memory_id": "other"}, {"cube_id": "other"}]) +def test_event_scope_mismatch_is_rejected(overrides): + bad = event().model_copy(update=overrides) + with pytest.raises(LifecycleError, match="scope"): + apply_event(state(), bad, config(), now=NOW) + + +def test_future_event_cannot_extend_retention_into_future(): + observed = apply_event(state(), event(at=NOW + timedelta(seconds=30)), config(), now=NOW) + assert observed.last_hit_at == NOW + with pytest.raises(LifecycleError, match="future"): + apply_event(state(), event(at=NOW + timedelta(days=2)), config(), now=NOW) + + +def test_legacy_naive_creation_is_utc_but_new_event_requires_timezone(): + original = state(created_at=datetime(2026, 1, 10)) + assert original.created_at == NOW + with pytest.raises(ValueError): + event(at=datetime(2026, 1, 10)) + + +@pytest.mark.parametrize( + "field,value", + [ + ("half_life_seconds", 0), + ("receipt_capacity", 0), + ("archive_strength_threshold", float("nan")), + ("retrieval_reinforcement", -1), + ], +) +def test_invalid_policy_parameters_fail_before_mutation(field, value): + with pytest.raises(ValueError): + config(**{field: value}) + + +def test_policy_change_cannot_reinterpret_accumulated_masses(): + observed = apply_event(state(), event(kind=EventKind.POSITIVE), config(), now=NOW) + with pytest.raises(LifecycleError, match="policy"): + apply_event(observed, event("next"), config(half_life_seconds=172800), now=NOW) + + +def test_external_archived_status_is_not_automatically_reactivated(): + decision = decide_maintenance(state(), config(), now=NOW, graph_status="archived") + assert decision.action == "skip" + assert decision.reason == "externally_archived" + + +def test_retention_works_without_python_311_only_math_functions(monkeypatch): + monkeypatch.delattr(math, "exp2", raising=False) + decision = decide_maintenance(state(), config(), now=NOW + timedelta(days=1)) + assert decision.strength == pytest.approx(0.5) diff --git a/tests/dream/test_lifecycle_postgres.py b/tests/dream/test_lifecycle_postgres.py new file mode 100644 index 000000000..e80a01923 --- /dev/null +++ b/tests/dream/test_lifecycle_postgres.py @@ -0,0 +1,254 @@ +"""Real PostgreSQL lifecycle ownership tests; enable with MEMOS_TEST_PG_DSN.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest + +from memos.dream.lifecycle import LIFECYCLE_METADATA_KEY, LifecycleConfig, initialize_state +from memos.exceptions import MemoryError as MemOSMemoryError +from memos.graph_dbs.base import NodeMetadataUpdate + + +def add(graph, *, archived=False): + memory_id = "memory-" + uuid4().hex + state = initialize_state( + memory_id=memory_id, + cube_id="cube-a", + created_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + config=LifecycleConfig(), + ).model_copy( + update={"protect_archived_status": archived, "status": "archived" if archived else "active"} + ) + graph.add_node( + memory_id, + "Synthetic memory", + { + "source": "dream", + "status": "archived" if archived else "activated", + LIFECYCLE_METADATA_KEY: state.model_dump(mode="json"), + "counter": 0, + }, + user_name="cube-a", + ) + return memory_id + + +def test_concurrent_atomic_updates_keep_every_committed_observation(graph): + memory_id = add(graph) + + def increment(_): + return graph.mutate_nodes_metadata( + [memory_id], + lambda _id, metadata: NodeMetadataUpdate({"counter": metadata["counter"] + 1}), + user_name="cube-a", + ) + + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(increment, range(16))) + assert all(result[0].found and result[0].changed for result in results) + assert graph.get_node(memory_id, user_name="cube-a")["metadata"]["counter"] == 16 + + +def test_stale_ordinary_write_cannot_erase_committed_lifecycle_hit(graph): + memory_id = add(graph) + stale = graph.get_node(memory_id, user_name="cube-a")["metadata"] + + def hit(_id, metadata): + life = {**metadata[LIFECYCLE_METADATA_KEY], "hit_count": 1, "revision": 1} + return NodeMetadataUpdate({LIFECYCLE_METADATA_KEY: life}) + + graph.mutate_nodes_metadata([memory_id], hit, user_name="cube-a") + graph.update_node(memory_id, {**stale, "memory": "Content changed"}, user_name="cube-a") + result = graph.get_node(memory_id, user_name="cube-a") + assert result["memory"] == "Content changed" + assert result["metadata"][LIFECYCLE_METADATA_KEY]["hit_count"] == 1 + assert result["metadata"][LIFECYCLE_METADATA_KEY]["revision"] == 1 + + +def test_context_style_upsert_preserves_invalidated_archive_and_evidence(graph): + memory_id = add(graph, archived=True) + previous = graph.get_node(memory_id, user_name="cube-a")["metadata"][LIFECYCLE_METADATA_KEY] + graph.add_node( + memory_id, + "Refreshed context", + {"source": "system", "status": "activated"}, + user_name="cube-a", + ) + result = graph.get_node(memory_id, user_name="cube-a") + assert result["memory"] == "Refreshed context" + assert result["metadata"]["status"] == "archived" + assert result["metadata"][LIFECYCLE_METADATA_KEY] == previous + + +def test_ordinary_status_write_cannot_restore_protected_archive(graph): + memory_id = add(graph, archived=True) + graph.update_node( + memory_id, {"status": "activated", LIFECYCLE_METADATA_KEY: {}}, user_name="cube-a" + ) + result = graph.get_node(memory_id, user_name="cube-a") + assert result["metadata"]["status"] == "archived" + assert result["metadata"][LIFECYCLE_METADATA_KEY]["protect_archived_status"] is True + + +def test_atomic_capability_can_perform_deliberate_recovery(graph): + memory_id = add(graph, archived=True) + + def recover(_id, metadata): + life = { + **metadata[LIFECYCLE_METADATA_KEY], + "protect_archived_status": False, + "status": "active", + } + return NodeMetadataUpdate( + {LIFECYCLE_METADATA_KEY: life, "status": "activated"}, result="restored" + ) + + result = graph.mutate_nodes_metadata([memory_id], recover, user_name="cube-a") + assert result[0].result == "restored" + assert graph.get_node(memory_id, user_name="cube-a")["metadata"]["status"] == "activated" + + +def test_scope_mismatch_cannot_run_a_mutation(graph): + memory_id = add(graph) + + def forbidden(_id, _metadata): + pytest.fail("Foreign-scope mutation was invoked") + + result = graph.mutate_nodes_metadata([memory_id], forbidden, user_name="cube-b") + assert not result[0].found + assert graph.get_node(memory_id, user_name="cube-a")["metadata"]["counter"] == 0 + + +def test_foreign_upsert_cannot_take_over_a_lifecycle_owned_id(graph): + memory_id = add(graph) + graph.add_node(memory_id, "Other tenant's memory", {"source": "dream"}, user_name="cube-b") + assert graph.get_node(memory_id, user_name="cube-a")["memory"] == "Synthetic memory" + assert graph.get_node(memory_id, user_name="cube-b")["memory"] == "Other tenant's memory" + + +def test_callback_failure_rolls_back_the_entire_batch(graph): + memory_ids = sorted([add(graph), add(graph)]) + + def update(memory_id, _metadata): + if memory_id == memory_ids[1]: + raise MemOSMemoryError("Synthetic failed reduction") + return NodeMetadataUpdate({"counter": 42}) + + with pytest.raises(MemOSMemoryError): + graph.mutate_nodes_metadata(memory_ids, update, user_name="cube-a") + assert [ + graph.get_node(item, user_name="cube-a")["metadata"]["counter"] for item in memory_ids + ] == [0, 0] + # A subsequent operation proves rollback/autocommit restoration before pool reuse. + graph.update_node(memory_ids[0], {"counter": 1}, user_name="cube-a") + assert graph.get_node(memory_ids[0], user_name="cube-a")["metadata"]["counter"] == 1 + + +def test_read_only_reducer_does_not_update_metadata_timestamps(graph): + memory_id = add(graph) + before = graph.get_node(memory_id, user_name="cube-a")["metadata"] + result = graph.mutate_nodes_metadata( + [memory_id], + lambda _id, _metadata: NodeMetadataUpdate(None, result="inspect"), + user_name="cube-a", + ) + assert result[0].result == "inspect" + assert not result[0].changed + assert graph.get_node(memory_id, user_name="cube-a")["metadata"] == before + + +def test_keyset_pages_are_bounded_unique_and_scoped(graph): + own = {add(graph) for _ in range(5)} + after = None + seen = [] + while True: + page = graph.list_lifecycle_nodes(user_name="cube-a", after=after, limit=2) + assert len(page) <= 2 + if not page: + break + seen.extend(row["id"] for row in page) + after = page[-1]["id"] + assert own <= set(seen) + assert len(seen) == len(set(seen)) + assert graph.list_lifecycle_nodes(user_name="cube-b", limit=2) == [] + + +def test_legacy_write_semantics_and_embedding_survive_managed_updates(graph): + memory_id = "legacy-" + uuid4().hex + graph.add_node( + memory_id, + "Original", + {"status": "activated", "old": 1, "embedding": [1, 0, 0]}, + user_name="cube-a", + ) + fields = {"memory": "Merged", "new": 2} + graph.update_node(memory_id, fields, user_name="cube-a") + assert fields == {"memory": "Merged", "new": 2} + merged = graph.get_node(memory_id, include_embedding=True, user_name="cube-a") + assert merged["metadata"]["old"] == 1 + assert merged["metadata"]["embedding"] == [1, 0, 0] + graph.add_node( + memory_id, + "Replaced", + {"status": "archived", "new": 3, "embedding": [0, 1, 0]}, + user_name="cube-a", + ) + result = graph.get_node(memory_id, include_embedding=True, user_name="cube-a") + assert result["memory"] == "Replaced" + assert "old" not in result["metadata"] + assert result["metadata"]["status"] == "archived" + assert result["metadata"]["embedding"] == [0, 1, 0] + + +@pytest.mark.parametrize( + "envelope", + [ + None, + [], + {"version": 99}, + {"protect_archived_status": "false"}, + {"version": 99, "protect_archived_status": False}, + {"version": True, "protect_archived_status": False}, + {"protect_archived_status": False}, + {"version": "1", "protect_archived_status": False}, + {"version": 1.0, "protect_archived_status": False}, + ], +) +def test_malformed_envelope_remains_quarantined_across_ordinary_writers(graph, envelope): + memory_id = "quarantine-" + uuid4().hex + graph.add_node( + memory_id, + "Original", + {LIFECYCLE_METADATA_KEY: envelope, "status": "archived"}, + user_name="cube-a", + ) + graph.update_node(memory_id, {"status": "activated"}, user_name="cube-a") + graph.add_node(memory_id, "New", {"status": "activated"}, user_name="cube-a") + result = graph.get_node(memory_id, user_name="cube-a")["metadata"] + assert result[LIFECYCLE_METADATA_KEY] == envelope + assert result["status"] == "archived" + + +def test_invalid_patch_serialization_rolls_back_earlier_rows(graph): + memory_ids = sorted([add(graph), add(graph)]) + with pytest.raises(ValueError): + graph.mutate_nodes_metadata( + memory_ids, + lambda item, _meta: NodeMetadataUpdate( + {"counter": 1 if item == memory_ids[0] else float("nan")} + ), + user_name="cube-a", + ) + assert [ + graph.get_node(item, user_name="cube-a")["metadata"]["counter"] for item in memory_ids + ] == [0, 0] + + +@pytest.mark.parametrize("limit", [0, -1, 1001, True, 1.5]) +def test_invalid_page_limit_rejected_before_query(graph, limit): + with pytest.raises(MemOSMemoryError): + graph.list_lifecycle_nodes(user_name="cube-a", limit=limit) diff --git a/tests/dream/test_lifecycle_runner.py b/tests/dream/test_lifecycle_runner.py new file mode 100644 index 000000000..aea93b56d --- /dev/null +++ b/tests/dream/test_lifecycle_runner.py @@ -0,0 +1,153 @@ +"""Bounded maintenance and durable cursor failure recovery on native storage.""" + +import json + +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +import pytest + +from memos.dream.lifecycle import LIFECYCLE_METADATA_KEY, LifecycleConfig, LifecycleError +from memos.dream.lifecycle.runner import DreamMaintenanceRunner, MaintenanceLimits +from memos.dream.lifecycle.service import DreamLifecycleService + + +NOW = datetime(2026, 2, 1, tzinfo=timezone.utc) + + +def records(graph, count=5): + ids = ["runner-" + uuid4().hex for _ in range(count)] + for item in ids: + graph.add_node( + item, + "Synthetic old memory", + { + "source": "dream", + "status": "activated", + "created_at": (NOW - timedelta(days=30)).isoformat(), + }, + user_name="runner-cube", + ) + return ids + + +def service(graph): + return DreamLifecycleService( + graph, LifecycleConfig(half_life_seconds=86400, never_hit_grace_seconds=0) + ) + + +def test_node_cap_checkpoint_resume_and_quiet_next_sweep(graph, tmp_path): + ids = records(graph) + path = tmp_path / "cursor.json" + runner = DreamMaintenanceRunner(service(graph), checkpoint=path) + first = runner.run( + cube_id="runner-cube", now=NOW, limits=MaintenanceLimits(page_size=2, max_nodes=3) + ) + assert first.scanned == 3 and first.archived == 3 and not first.complete + second = runner.run(cube_id="runner-cube", now=NOW) + assert second.complete + assert all( + graph.get_node(item, user_name="runner-cube")["metadata"]["status"] == "archived" + for item in ids + ) + next_sweep = runner.run(cube_id="runner-cube", now=NOW + timedelta(days=1)) + assert next_sweep.complete and next_sweep.scanned >= 5 and next_sweep.archived == 0 + + +def test_dry_run_does_not_create_checkpoint_lock_or_lifecycle_metadata(graph, tmp_path): + ids = records(graph) + path = tmp_path / "cursor.json" + before = {item: graph.get_node(item, user_name="runner-cube")["metadata"] for item in ids} + runner = DreamMaintenanceRunner(service(graph), checkpoint=path) + report = runner.run(cube_id="runner-cube", now=NOW, dry_run=True) + assert report.would_archive >= len(ids) and report.archived == 0 + assert not list(tmp_path.iterdir()) + assert { + item: graph.get_node(item, user_name="runner-cube")["metadata"] for item in ids + } == before + + +def test_failed_page_does_not_pass_corrupt_row_or_commit_other_page_members(graph, tmp_path): + ids = sorted(records(graph, 2)) + graph.update_node(ids[1], {LIFECYCLE_METADATA_KEY: {"version": 99}}, user_name="runner-cube") + runner = DreamMaintenanceRunner(service(graph), checkpoint=tmp_path / "cursor.json") + report = runner.run(cube_id="runner-cube", now=NOW, limits=MaintenanceLimits(page_size=1000)) + assert report.stop_reason == "page_failed" and report.failed == 1 + assert report.next_after is None + assert graph.get_node(ids[0], user_name="runner-cube")["metadata"]["status"] == "activated" + + +def test_checkpoint_binding_and_corruption_rejected_before_writes(graph, tmp_path): + records(graph) + path = tmp_path / "cursor.json" + runner = DreamMaintenanceRunner(service(graph), checkpoint=path) + runner.run(cube_id="runner-cube", now=NOW, limits=MaintenanceLimits(max_nodes=1)) + with pytest.raises(LifecycleError, match="scope|binding"): + runner.run(cube_id="different-cube", now=NOW) + original = json.loads(path.read_text()) + path.write_text('{"version":1,"version":1}') + with pytest.raises(LifecycleError): + runner.run(cube_id="runner-cube", now=NOW) + path.write_text(json.dumps({**original, "version": 99})) + with pytest.raises(LifecycleError): + runner.run(cube_id="runner-cube", now=NOW) + + +def test_cancel_and_elapsed_budget_are_cooperative_and_do_not_advance(graph, tmp_path): + records(graph) + runner = DreamMaintenanceRunner(service(graph), checkpoint=tmp_path / "cursor.json") + result = runner.run(cube_id="runner-cube", now=NOW, cancelled=lambda: True) + assert result.scanned == 0 and result.stop_reason == "cancelled" + times = iter([0, 10]) + runner = DreamMaintenanceRunner(service(graph), monotonic=lambda: next(times, 10)) + result = runner.run(cube_id="runner-cube", now=NOW, limits=MaintenanceLimits(max_seconds=1)) + assert result.scanned == 0 and result.stop_reason == "time_budget" + + +def test_checkpoint_failure_replays_committed_page_without_duplicate_transitions( + graph, tmp_path, monkeypatch +): + records(graph) + runner = DreamMaintenanceRunner(service(graph), checkpoint=tmp_path / "cursor.json") + original = runner._save_cursor + monkeypatch.setattr( + runner, + "_save_cursor", + lambda _cursor: (_ for _ in ()).throw(OSError("Synthetic disk failure")), + ) + result = runner.run(cube_id="runner-cube", now=NOW) + assert result.stop_reason == "checkpoint_failed" and result.next_after is None + monkeypatch.setattr(runner, "_save_cursor", original) + replay = runner.run(cube_id="runner-cube", now=NOW) + assert replay.complete and replay.archived == 0 + + +def test_reports_bound_decisions_and_checkpoint_exact_evidence_revisions(graph, tmp_path): + records(graph, 5) + path = tmp_path / "cursor.json" + runner = DreamMaintenanceRunner(service(graph), checkpoint=path) + report = runner.run( + cube_id="runner-cube", + now=NOW, + limits=MaintenanceLimits(page_size=2, max_report_decisions=2), + ) + assert report.eligible == 5 and report.archived == 5 + assert len(report.decisions) == 2 and report.omitted_decisions == 3 + assert report.reason_counts == {"stale": 5} + assert report.backend_capability == "atomic_lifecycle_metadata" + assert all(item.evidence_revision == 0 for item in report.decisions) + cursor = json.loads(path.read_text()) + assert cursor["last_page"][0]["reason"] == "stale" + assert cursor["last_page"][0]["evidence_revision"] == 0 + assert "memory" not in cursor["last_page"][0] + + +def test_failed_page_reports_every_retry_id_without_advancing_cursor(graph, tmp_path): + ids = sorted(records(graph, 2)) + graph.update_node(ids[-1], {LIFECYCLE_METADATA_KEY: {"version": 99}}, user_name="runner-cube") + runner = DreamMaintenanceRunner(service(graph), checkpoint=tmp_path / "cursor.json") + report = runner.run(cube_id="runner-cube", now=NOW) + assert report.retry_ids == tuple(ids) + assert report.invalid == 1 and report.failed == 1 + assert report.next_after is None diff --git a/tests/dream/test_lifecycle_runtime.py b/tests/dream/test_lifecycle_runtime.py new file mode 100644 index 000000000..2f4de2feb --- /dev/null +++ b/tests/dream/test_lifecycle_runtime.py @@ -0,0 +1,170 @@ +"""Opt-in plugin ownership, manual maintenance and cooperative worker shutdown.""" + +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from memos.dream.lifecycle import LIFECYCLE_METADATA_KEY, LifecycleError +from memos.dream.plugin import CommunityDreamPlugin +from memos.plugins import hooks +from memos.plugins.hook_defs import H + + +@pytest.fixture(autouse=True) +def restore_hooks(): + before = {key: list(value) for key, value in hooks._hooks.items()} + hooks._hooks.clear() + yield + hooks._hooks.clear() + hooks._hooks.update(before) + + +def test_disabled_lifecycle_has_no_backend_checks_worker_or_shared_mutation(): + plugin = CommunityDreamPlugin() + plugin.on_load() + context = { + "shared": {"graph_db": SimpleNamespace(supports_atomic_lifecycle=False)}, + "configs": {}, + } + plugin.init_components(context) + assert "dream_lifecycle" not in context["shared"] + assert plugin.lifecycle is None + assert not hooks._hooks.get(H.SEARCH_COMPLETED) + plugin.on_shutdown() + assert not any(hooks._hooks.values()) + + +def test_plugin_manual_quiet_cube_and_shutdown_remove_only_owned_hooks(graph, tmp_path): + now = datetime.now(timezone.utc) + item = str(uuid4()) + graph.add_node( + item, + "Synthetic quiet memory", + { + "source": "dream", + "status": "activated", + "created_at": (now - timedelta(days=30)).isoformat(), + }, + user_name="quiet-cube", + ) + + def other(**kwargs): + return None + + hooks.register_hook(H.ADD_AFTER, other) + plugin = CommunityDreamPlugin() + plugin.on_load() + context = { + "shared": {"graph_db": graph}, + "configs": { + "dream_lifecycle": { + "enabled": True, + "cube_ids": ["quiet-cube"], + "checkpoint_directory": str(tmp_path), + "policy": {"half_life_seconds": 86400, "never_hit_grace_seconds": 0}, + } + }, + } + plugin.init_components(context) + assert context["shared"]["dream_lifecycle"] is plugin.lifecycle.service + report = plugin.lifecycle.run_maintenance("quiet-cube", now=now) + assert report.archived == 1 and report.complete + assert ( + graph.get_node(item, user_name="quiet-cube")["metadata"][LIFECYCLE_METADATA_KEY]["status"] + == "archived" + ) + plugin.init_components(context) + assert len(hooks._hooks[H.SEARCH_COMPLETED]) == 1 + plugin.on_shutdown() + assert hooks._hooks[H.ADD_AFTER] == [other] + assert not hooks._hooks.get(H.SEARCH_COMPLETED) + assert "dream_lifecycle" not in context["shared"] + plugin.on_load() + assert len(hooks._hooks[H.DREAM_EXECUTE]) == 1 + plugin.on_shutdown() + + +def test_enabled_unsupported_backend_fails_explicitly_without_registering_observers(): + plugin = CommunityDreamPlugin() + plugin.on_load() + with pytest.raises(LifecycleError, match="atomic"): + plugin.init_components( + { + "shared": {"graph_db": SimpleNamespace(supports_atomic_lifecycle=False)}, + "configs": {"dream_lifecycle": {"enabled": True}}, + } + ) + assert not hooks._hooks.get(H.SEARCH_COMPLETED) + plugin.on_shutdown() + + +def test_periodic_runtime_checks_quiet_registered_scope_and_stops(graph, monkeypatch): + import threading + + from memos.dream.lifecycle.runtime import DreamLifecycleRuntime, LifecycleRuntimeConfig + + now = datetime.now(timezone.utc) + item = str(uuid4()) + graph.add_node( + item, + "Quiet", + { + "source": "dream", + "status": "activated", + "created_at": (now - timedelta(days=30)).isoformat(), + }, + user_name="quiet-cube", + ) + runtime = DreamLifecycleRuntime( + graph, + LifecycleRuntimeConfig( + enabled=True, + cube_ids=("quiet-cube",), + interval_seconds=0.01, + policy={"half_life_seconds": 86400, "never_hit_grace_seconds": 0}, + ), + ) + finished = threading.Event() + original = runtime.run_maintenance + + def run(*args, **kwargs): + result = original(*args, **kwargs) + finished.set() + return result + + monkeypatch.setattr(runtime, "run_maintenance", run) + runtime.start() + assert finished.wait(timeout=5) + assert runtime.shutdown(timeout=5) + assert graph.get_node(item, user_name="quiet-cube")["metadata"]["status"] == "archived" + assert not runtime.worker_alive + + +def test_shutdown_does_not_claim_inflight_manual_operation_stopped(graph, monkeypatch): + import threading + + from memos.dream.lifecycle.runtime import DreamLifecycleRuntime, LifecycleRuntimeConfig + + runtime = DreamLifecycleRuntime( + graph, LifecycleRuntimeConfig(enabled=True, cube_ids=("quiet-cube",)) + ) + entered, release = threading.Event(), threading.Event() + original = graph.list_lifecycle_nodes + + def blocking_read(**kwargs): + entered.set() + assert release.wait(timeout=5) + return original(**kwargs) + + monkeypatch.setattr(graph, "list_lifecycle_nodes", blocking_read) + thread = threading.Thread(target=lambda: runtime.run_maintenance("quiet-cube")) + thread.start() + try: + assert entered.wait(timeout=5) + assert runtime.shutdown(timeout=0) is False + finally: + release.set() + thread.join(timeout=5) + assert runtime.shutdown(timeout=1) is True diff --git a/tests/dream/test_lifecycle_service.py b/tests/dream/test_lifecycle_service.py new file mode 100644 index 000000000..e15773c6b --- /dev/null +++ b/tests/dream/test_lifecycle_service.py @@ -0,0 +1,243 @@ +"""Public lifecycle service and native persistence contracts.""" + +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +import pytest + +from memos.dream.lifecycle import ( + LIFECYCLE_METADATA_KEY, + EventKind, + LifecycleConfig, + LifecycleError, + LifecycleEvent, +) +from memos.dream.lifecycle.service import DreamLifecycleService +from memos.dream.pipeline.persistence import DreamPersistence +from memos.dream.types import DreamAction, DreamActionType, DreamResult, TargetMemoryType + + +NOW = datetime(2026, 1, 15, tzinfo=timezone.utc) + + +def node(graph, source="dream", **extra): + memory_id = "service-" + uuid4().hex + graph.add_node( + memory_id, + "Synthetic source", + { + "source": source, + "status": "activated", + "created_at": (NOW - timedelta(days=14)).isoformat(), + "confidence": 0.3, + **extra, + }, + user_name="cube-a", + ) + return memory_id + + +def event(memory_id, identity="retrieval-1", kind=EventKind.RETRIEVED, at=NOW): + return LifecycleEvent( + memory_id=memory_id, + cube_id="cube-a", + event_id=identity, + kind=kind, + at=at, + reason="Explicit fixture feedback" if kind == EventKind.INVALIDATED else "", + ) + + +def test_observation_is_atomic_deduplicated_and_keeps_raw_confidence(graph): + memory_id = node(graph) + service = DreamLifecycleService(graph, LifecycleConfig()) + report = service.observe([event(memory_id), event(memory_id)], now=NOW) + assert report.accepted == 1 and report.duplicates == 1 + metadata = graph.get_node(memory_id, user_name="cube-a")["metadata"] + assert metadata["confidence"] == 0.3 + assert metadata[LIFECYCLE_METADATA_KEY]["hit_count"] == 1 + assert service.observe([event(memory_id)], now=NOW).duplicates == 1 + + +def test_non_dream_and_missing_nodes_do_not_gain_lifecycle_state(graph): + memory_id = node(graph, source="user") + service = DreamLifecycleService(graph, LifecycleConfig()) + report = service.observe([event(memory_id), event("missing")], now=NOW) + assert report.skipped == 1 and report.missing == 1 + assert LIFECYCLE_METADATA_KEY not in graph.get_node(memory_id, user_name="cube-a")["metadata"] + + +def test_duplicate_conflict_rolls_back_all_events_in_batch(graph): + first, second = node(graph), node(graph) + service = DreamLifecycleService(graph, LifecycleConfig()) + service.observe([event(second)], now=NOW) + with pytest.raises(LifecycleError, match="identity conflict"): + service.observe([event(first), event(second, kind=EventKind.NEGATIVE)], now=NOW) + assert LIFECYCLE_METADATA_KEY not in graph.get_node(first, user_name="cube-a")["metadata"] + + +def test_dry_run_repeated_sweep_and_explicit_restore(graph): + memory_id = node(graph) + service = DreamLifecycleService( + graph, LifecycleConfig(half_life_seconds=86400, never_hit_grace_seconds=0) + ) + before = graph.get_node(memory_id, user_name="cube-a")["metadata"] + outcome = service.maintain([memory_id], cube_id="cube-a", now=NOW, dry_run=True)[0] + assert outcome.action == "archive" and outcome.reason == "stale" + assert graph.get_node(memory_id, user_name="cube-a")["metadata"] == before + service.maintain([memory_id], cube_id="cube-a", now=NOW) + archived = graph.get_node(memory_id, user_name="cube-a")["metadata"] + service.maintain([memory_id], cube_id="cube-a", now=NOW) + assert graph.get_node(memory_id, user_name="cube-a")["metadata"] == archived + service.restore(memory_id, cube_id="cube-a", reason="Revalidated by caller", now=NOW) + assert graph.get_node(memory_id, user_name="cube-a")["metadata"]["status"] == "activated" + assert service.maintain([memory_id], cube_id="cube-a", now=NOW)[0].action == "keep" + + +def test_explicit_invalidation_overrides_pin_and_requires_restore_override(graph): + memory_id = node(graph) + service = DreamLifecycleService(graph, LifecycleConfig()) + service.set_pinned(memory_id, cube_id="cube-a", pinned=True, now=NOW) + service.observe([event(memory_id, kind=EventKind.INVALIDATED)], now=NOW) + assert graph.get_node(memory_id, user_name="cube-a")["metadata"]["status"] == "archived" + with pytest.raises(LifecycleError): + service.restore(memory_id, cube_id="cube-a", reason="Try", now=NOW) + service.restore( + memory_id, + cube_id="cube-a", + reason="Explicit revalidation", + now=NOW, + override_invalidation=True, + ) + assert graph.get_node(memory_id, user_name="cube-a")["metadata"]["status"] == "activated" + + +def test_corrupt_metadata_is_quarantined_and_foreign_scope_rejected(graph): + memory_id = node(graph, **{LIFECYCLE_METADATA_KEY: {"version": 99}}) + service = DreamLifecycleService(graph, LifecycleConfig()) + before = graph.get_node(memory_id, user_name="cube-a")["metadata"] + with pytest.raises(LifecycleError): + service.maintain([memory_id], cube_id="cube-a", now=NOW) + assert graph.get_node(memory_id, user_name="cube-a")["metadata"] == before + assert service.maintain([memory_id], cube_id="cube-b", now=NOW)[0].action == "missing" + + +def test_native_persistence_archive_update_merge_use_graph_field_contract(graph): + first, second = node(graph), node(graph) + persistence = DreamPersistence() + persistence.bind_context({"shared": {"graph_db": graph}}) + + def persist(action): + persistence.persist( + results=[DreamResult(cluster_id="native", actions=[action])], + text_mem=object(), + cube_id="cube-a", + mem_cube_id="cube-a", + user_id="user-a", + signal_snapshot=None, + ) + + args = { + "target_memory_type": TargetMemoryType.LONG_TERM, + "rationale": "Synthetic explicit instruction", + "confidence": 1, + } + persist( + DreamAction( + action_type=DreamActionType.UPDATE, + target_memory_id=first, + new_content="Updated", + **args, + ) + ) + assert graph.get_node(first, user_name="cube-a")["memory"] == "Updated" + persist( + DreamAction( + action_type=DreamActionType.MERGE, + source_memory_ids=[second], + new_content="Merged", + **args, + ) + ) + assert graph.get_node(second, user_name="cube-a")["metadata"]["status"] == "merged" + persist(DreamAction(action_type=DreamActionType.ARCHIVE, target_memory_id=first, **args)) + assert graph.get_node(first, user_name="cube-a")["metadata"]["status"] == "archived" + + +def test_opted_in_persistence_initializes_service_state_without_confidence_rewrite(graph): + service = DreamLifecycleService(graph, LifecycleConfig()) + persistence = DreamPersistence() + persistence.bind_context({"shared": {"graph_db": graph, "dream_lifecycle": service}}) + persistence.persist( + results=[ + DreamResult( + cluster_id="native", + actions=[ + DreamAction( + action_type=DreamActionType.CREATE, + target_memory_type=TargetMemoryType.LONG_TERM, + new_content="New memory", + rationale="Synthetic", + confidence=0.7, + ) + ], + ) + ], + text_mem=object(), + cube_id="cube-a", + mem_cube_id="cube-a", + user_id="user-a", + signal_snapshot=None, + ) + records = graph.list_lifecycle_nodes(user_name="cube-a", limit=1000) + created = [row for row in records if row["metadata"].get("confidence") == 0.7] + assert len(created) == 1 + assert created[0]["metadata"][LIFECYCLE_METADATA_KEY]["base_strength"] == 1 + from memos.memories.textual.item import TextualMemoryItem + + native_item = TextualMemoryItem(**graph.get_node(created[0]["id"], user_name="cube-a")) + assert native_item.metadata.source == "dream" + + +def test_inspection_explains_quarantine_and_retention_without_initializing_or_writing(graph): + good = node(graph) + corrupt = node(graph, **{LIFECYCLE_METADATA_KEY: {"version": 99}}) + unrelated = node(graph, source="user") + service = DreamLifecycleService( + graph, LifecycleConfig(half_life_seconds=86400, never_hit_grace_seconds=0) + ) + before = graph.get_node(good, user_name="cube-a")["metadata"] + reports = service.inspect([good, corrupt, unrelated, "missing"], cube_id="cube-a", now=NOW) + assert reports[0].next_action == "archive" and reports[0].reason == "stale" + assert reports[0].managed is False and reports[0].receipt_count == 0 + assert reports[1].quarantined and reports[1].next_action == "skip" + assert reports[2].eligible is False and reports[3].found is False + assert graph.get_node(good, user_name="cube-a")["metadata"] == before + + +def test_malformed_temporal_and_ownership_invariants_are_quarantined(graph): + from memos.dream.lifecycle import initialize_state + + service = DreamLifecycleService(graph, LifecycleConfig()) + for patch in [ + {"last_hit_at": NOW + timedelta(days=1), "hit_count": 1}, + {"status": "archived", "protect_archived_status": False}, + {"invalidated_by_feedback": True}, + {"hit_count": 1, "last_hit_at": None}, + ]: + item = node(graph) + state = initialize_state( + memory_id=item, + cube_id="cube-a", + config=service.config, + created_at=NOW - timedelta(days=1), + ) + graph.update_node( + item, + {LIFECYCLE_METADATA_KEY: state.model_copy(update=patch).model_dump(mode="json")}, + user_name="cube-a", + ) + before = graph.get_node(item, user_name="cube-a")["metadata"] + with pytest.raises(LifecycleError): + service.maintain([item], cube_id="cube-a", now=NOW) + assert graph.get_node(item, user_name="cube-a")["metadata"] == before