Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/memos/memories/textual/item.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from datetime import datetime
from typing import Any, Literal

from pydantic import BaseModel, ConfigDict, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator


ALLOWED_ROLES = {"user", "assistant", "system"}
Expand Down Expand Up @@ -159,14 +159,14 @@ class TextualMemoryMetadata(BaseModel):
description="Internal algorithm metadata reserved for system use.",
)

@field_validator("internal_info", mode="before")
@field_validator("internal_info", "info", mode="before")
@classmethod
def _decode_internal_info(cls, value: Any) -> Any:
def _decode_dict_metadata(cls, value: Any, validation_info: ValidationInfo) -> Any:
# Neo4j stores dictionaries as JSON strings; decode them before model validation.
if isinstance(value, str):
value = json.loads(value)
if not isinstance(value, dict):
raise ValueError("internal_info JSON must decode to an object")
raise ValueError(f"{validation_info.field_name} JSON must decode to an object")
return value

model_config = ConfigDict(extra="allow")
Expand Down
115 changes: 115 additions & 0 deletions tests/graph_dbs/test_neo4j_internal_info_roundtrip.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
"""Legacy info decoding shares the existing strict metadata validation boundary."""

import json

from types import SimpleNamespace
from unittest.mock import MagicMock
from uuid import uuid4

import pytest

from pydantic import ValidationError

from memos.graph_dbs.neo4j import (
Neo4jGraphDB,
_flatten_info_fields,
_sanitize_neo4j_metadata,
)
from memos.graph_dbs.neo4j_community import Neo4jCommunityGraphDB
from memos.memories.textual.item import TextualMemoryItem


@pytest.fixture(params=["neo4j", "community", "community_batch"])
def parse_nodes(request):
cls = Neo4jGraphDB if request.param == "neo4j" else Neo4jCommunityGraphDB
db = cls.__new__(cls)
if isinstance(db, Neo4jCommunityGraphDB):
db.vec_db = MagicMock()
db.vec_db.get_by_id.return_value = None
db.vec_db.get_by_ids.return_value = []
if request.param == "community_batch":
return db._parse_nodes

def parse(nodes):
return [db._parse_node(node) for node in nodes]

return parse


def make_node():
return {
"id": str(uuid4()),
"memory": "Synthetic legacy memory",
"memory_type": "LongTermMemory",
"internal_info": json.dumps({"chunk_index": 0}),
"legacy_flat_key": "preserved",
}


@pytest.mark.parametrize(
("stored_info", "expected"),
[
(None, None),
({}, {}),
({"nested": {"values": [True, None]}}, {"nested": {"values": [True, None]}}),
("{}", {}),
(
' {"nested": {"text": "历史数据", "values": [true, null]}}\n',
{"nested": {"text": "历史数据", "values": [True, None]}},
),
],
)
def test_legacy_info_validates_on_all_graph_read_paths(parse_nodes, stored_info, expected):
node = make_node()
node["info"] = stored_info
parsed = parse_nodes([node])[0]

item = TextualMemoryItem.from_dict(parsed)

assert item.id == node["id"]
assert item.memory == node["memory"]
assert item.metadata.info == expected
assert item.metadata.internal_info == {"chunk_index": 0}
assert item.metadata.legacy_flat_key == "preserved"
assert node["info"] == stored_info


@pytest.mark.parametrize("field", ["internal_info", "info"])
@pytest.mark.parametrize("value", ["{not json}", "plain text", "[]", "null", "1", '""'])
def test_corrupt_dict_metadata_is_rejected_without_dropping_it(parse_nodes, field, value):
node = make_node()
node[field] = value
parsed = parse_nodes([node])[0]

assert parsed["metadata"][field] == value
with pytest.raises(ValidationError):
TextualMemoryItem.from_dict(parsed)


def test_missing_info_and_modern_flattened_data_remain_readable(parse_nodes):
node = make_node()
node.update(_sanitize_neo4j_metadata(_flatten_info_fields({"info": {"client_key": "value"}})))

parsed = parse_nodes([node])[0]
item = TextualMemoryItem.from_dict(parsed)

assert "info" not in parsed["metadata"]
assert item.metadata.info is None
assert item.metadata.client_key == "value"


def test_community_batch_preserves_embedding_and_info_alignment():
db = Neo4jCommunityGraphDB.__new__(Neo4jCommunityGraphDB)
db.vec_db = MagicMock()
nodes = [make_node(), make_node()]
for index, node in enumerate(nodes):
node["info"] = json.dumps({"index": index})
db.vec_db.get_by_ids.return_value = [
SimpleNamespace(id=nodes[1]["id"], vector=[0.2]),
SimpleNamespace(id=nodes[0]["id"], vector=[0.1]),
]

items = [TextualMemoryItem.from_dict(node) for node in db._parse_nodes(nodes)]

assert [item.metadata.info for item in items] == [{"index": 0}, {"index": 1}]
assert [item.metadata.embedding for item in items] == [[0.1], [0.2]]
29 changes: 18 additions & 11 deletions tests/memories/textual/test_item.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,38 +26,42 @@
PreferenceTextualMemoryMetadata,
],
)
def test_internal_info_decodes_json_object(metadata_cls):
@pytest.mark.parametrize("field", ["internal_info", "info"])
def test_dict_metadata_decodes_json_object(metadata_cls, field):
value = {"ingest_batch_id": "test-batch", "nested": {"chunk_index": 0, "values": [True, None]}}

metadata = metadata_cls(internal_info=json.dumps(value))
metadata = metadata_cls(**{field: " " + json.dumps(value) + "\n"})

assert metadata.internal_info == value
assert metadata.model_dump()["internal_info"] == value
assert getattr(metadata, field) == value
assert metadata.model_dump()[field] == value


@pytest.mark.parametrize("value", [None, {}, {"ingest_batch_id": "test-batch", "nested": [1, 2]}])
def test_internal_info_preserves_existing_values(value):
@pytest.mark.parametrize("field", ["internal_info", "info"])
def test_dict_metadata_preserves_existing_values(value, field):
original = copy.deepcopy(value)

metadata = TextualMemoryMetadata(internal_info=value)
metadata = TextualMemoryMetadata(**{field: value})

assert metadata.internal_info == original
assert getattr(metadata, field) == original
assert value == original


@pytest.mark.parametrize(
"value",
["{", "not-json", "[]", "null", "1", "true", '"text"', json.dumps('{"batch": "test"}')],
)
def test_internal_info_rejects_malformed_json_and_encoded_non_objects(value):
@pytest.mark.parametrize("field", ["internal_info", "info"])
def test_dict_metadata_rejects_malformed_json_and_encoded_non_objects(value, field):
with pytest.raises(ValidationError):
TextualMemoryMetadata(internal_info=value)
TextualMemoryMetadata(**{field: value})


@pytest.mark.parametrize("value", [[], [["batch", "test"]], 1, False])
def test_internal_info_preserves_rejection_of_invalid_types(value):
@pytest.mark.parametrize("field", ["internal_info", "info"])
def test_dict_metadata_preserves_rejection_of_invalid_types(value, field):
with pytest.raises(ValidationError):
TextualMemoryMetadata(internal_info=value)
TextualMemoryMetadata(**{field: value})


def test_search_item_from_dict_decodes_internal_info_without_changing_other_fields():
Expand All @@ -68,6 +72,7 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel
"metadata": {
"memory_type": "LongTermMemory",
"internal_info": json.dumps(value),
"info": json.dumps({"client_metadata": True}),
"relativity": 0.75,
"sources": [{"type": "chat", "role": "user", "content": "Synthetic source."}],
"session_id": "test-session",
Expand All @@ -78,6 +83,7 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel
item = TextualMemoryItem.from_dict(record)

assert item.metadata.internal_info == value
assert item.metadata.info == {"client_metadata": True}
assert item.id == record["id"]
assert item.memory == record["memory"]
assert item.metadata.sources[0].content == "Synthetic source."
Expand All @@ -88,4 +94,5 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel
# Graph storage serializes dictionaries as JSON strings again on the next write.
stored = item.to_dict()
stored["metadata"]["internal_info"] = json.dumps(stored["metadata"]["internal_info"])
stored["metadata"]["info"] = json.dumps(stored["metadata"]["info"])
assert TextualMemoryItem.from_dict(stored).to_dict() == item.to_dict()
Loading