diff --git a/src/memos/memories/textual/item.py b/src/memos/memories/textual/item.py index f92d3d291..fb9c903da 100644 --- a/src/memos/memories/textual/item.py +++ b/src/memos/memories/textual/item.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator ALLOWED_ROLES = {"user", "assistant", "system"} @@ -159,14 +159,14 @@ class TextualMemoryMetadata(BaseModel): description="Internal algorithm metadata reserved for system use.", ) - @field_validator("internal_info", mode="before") + @field_validator("internal_info", "info", mode="before") @classmethod - def _decode_internal_info(cls, value: Any) -> Any: + def _decode_dict_metadata(cls, value: Any, validation_info: ValidationInfo) -> Any: # Neo4j stores dictionaries as JSON strings; decode them before model validation. if isinstance(value, str): value = json.loads(value) if not isinstance(value, dict): - raise ValueError("internal_info JSON must decode to an object") + raise ValueError(f"{validation_info.field_name} JSON must decode to an object") return value model_config = ConfigDict(extra="allow") diff --git a/tests/graph_dbs/test_neo4j_internal_info_roundtrip.py b/tests/graph_dbs/test_neo4j_internal_info_roundtrip.py new file mode 100644 index 000000000..15ab30f31 --- /dev/null +++ b/tests/graph_dbs/test_neo4j_internal_info_roundtrip.py @@ -0,0 +1,115 @@ +"""Legacy info decoding shares the existing strict metadata validation boundary.""" + +import json + +from types import SimpleNamespace +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest + +from pydantic import ValidationError + +from memos.graph_dbs.neo4j import ( + Neo4jGraphDB, + _flatten_info_fields, + _sanitize_neo4j_metadata, +) +from memos.graph_dbs.neo4j_community import Neo4jCommunityGraphDB +from memos.memories.textual.item import TextualMemoryItem + + +@pytest.fixture(params=["neo4j", "community", "community_batch"]) +def parse_nodes(request): + cls = Neo4jGraphDB if request.param == "neo4j" else Neo4jCommunityGraphDB + db = cls.__new__(cls) + if isinstance(db, Neo4jCommunityGraphDB): + db.vec_db = MagicMock() + db.vec_db.get_by_id.return_value = None + db.vec_db.get_by_ids.return_value = [] + if request.param == "community_batch": + return db._parse_nodes + + def parse(nodes): + return [db._parse_node(node) for node in nodes] + + return parse + + +def make_node(): + return { + "id": str(uuid4()), + "memory": "Synthetic legacy memory", + "memory_type": "LongTermMemory", + "internal_info": json.dumps({"chunk_index": 0}), + "legacy_flat_key": "preserved", + } + + +@pytest.mark.parametrize( + ("stored_info", "expected"), + [ + (None, None), + ({}, {}), + ({"nested": {"values": [True, None]}}, {"nested": {"values": [True, None]}}), + ("{}", {}), + ( + ' {"nested": {"text": "历史数据", "values": [true, null]}}\n', + {"nested": {"text": "历史数据", "values": [True, None]}}, + ), + ], +) +def test_legacy_info_validates_on_all_graph_read_paths(parse_nodes, stored_info, expected): + node = make_node() + node["info"] = stored_info + parsed = parse_nodes([node])[0] + + item = TextualMemoryItem.from_dict(parsed) + + assert item.id == node["id"] + assert item.memory == node["memory"] + assert item.metadata.info == expected + assert item.metadata.internal_info == {"chunk_index": 0} + assert item.metadata.legacy_flat_key == "preserved" + assert node["info"] == stored_info + + +@pytest.mark.parametrize("field", ["internal_info", "info"]) +@pytest.mark.parametrize("value", ["{not json}", "plain text", "[]", "null", "1", '""']) +def test_corrupt_dict_metadata_is_rejected_without_dropping_it(parse_nodes, field, value): + node = make_node() + node[field] = value + parsed = parse_nodes([node])[0] + + assert parsed["metadata"][field] == value + with pytest.raises(ValidationError): + TextualMemoryItem.from_dict(parsed) + + +def test_missing_info_and_modern_flattened_data_remain_readable(parse_nodes): + node = make_node() + node.update(_sanitize_neo4j_metadata(_flatten_info_fields({"info": {"client_key": "value"}}))) + + parsed = parse_nodes([node])[0] + item = TextualMemoryItem.from_dict(parsed) + + assert "info" not in parsed["metadata"] + assert item.metadata.info is None + assert item.metadata.client_key == "value" + + +def test_community_batch_preserves_embedding_and_info_alignment(): + db = Neo4jCommunityGraphDB.__new__(Neo4jCommunityGraphDB) + db.vec_db = MagicMock() + nodes = [make_node(), make_node()] + for index, node in enumerate(nodes): + node["info"] = json.dumps({"index": index}) + db.vec_db.get_by_ids.return_value = [ + SimpleNamespace(id=nodes[1]["id"], vector=[0.2]), + SimpleNamespace(id=nodes[0]["id"], vector=[0.1]), + ] + + items = [TextualMemoryItem.from_dict(node) for node in db._parse_nodes(nodes)] + + assert [item.metadata.info for item in items] == [{"index": 0}, {"index": 1}] + assert [item.metadata.embedding for item in items] == [[0.1], [0.2]] diff --git a/tests/memories/textual/test_item.py b/tests/memories/textual/test_item.py index 71c206742..1238eb4d7 100644 --- a/tests/memories/textual/test_item.py +++ b/tests/memories/textual/test_item.py @@ -26,22 +26,24 @@ PreferenceTextualMemoryMetadata, ], ) -def test_internal_info_decodes_json_object(metadata_cls): +@pytest.mark.parametrize("field", ["internal_info", "info"]) +def test_dict_metadata_decodes_json_object(metadata_cls, field): value = {"ingest_batch_id": "test-batch", "nested": {"chunk_index": 0, "values": [True, None]}} - metadata = metadata_cls(internal_info=json.dumps(value)) + metadata = metadata_cls(**{field: " " + json.dumps(value) + "\n"}) - assert metadata.internal_info == value - assert metadata.model_dump()["internal_info"] == value + assert getattr(metadata, field) == value + assert metadata.model_dump()[field] == value @pytest.mark.parametrize("value", [None, {}, {"ingest_batch_id": "test-batch", "nested": [1, 2]}]) -def test_internal_info_preserves_existing_values(value): +@pytest.mark.parametrize("field", ["internal_info", "info"]) +def test_dict_metadata_preserves_existing_values(value, field): original = copy.deepcopy(value) - metadata = TextualMemoryMetadata(internal_info=value) + metadata = TextualMemoryMetadata(**{field: value}) - assert metadata.internal_info == original + assert getattr(metadata, field) == original assert value == original @@ -49,15 +51,17 @@ def test_internal_info_preserves_existing_values(value): "value", ["{", "not-json", "[]", "null", "1", "true", '"text"', json.dumps('{"batch": "test"}')], ) -def test_internal_info_rejects_malformed_json_and_encoded_non_objects(value): +@pytest.mark.parametrize("field", ["internal_info", "info"]) +def test_dict_metadata_rejects_malformed_json_and_encoded_non_objects(value, field): with pytest.raises(ValidationError): - TextualMemoryMetadata(internal_info=value) + TextualMemoryMetadata(**{field: value}) @pytest.mark.parametrize("value", [[], [["batch", "test"]], 1, False]) -def test_internal_info_preserves_rejection_of_invalid_types(value): +@pytest.mark.parametrize("field", ["internal_info", "info"]) +def test_dict_metadata_preserves_rejection_of_invalid_types(value, field): with pytest.raises(ValidationError): - TextualMemoryMetadata(internal_info=value) + TextualMemoryMetadata(**{field: value}) def test_search_item_from_dict_decodes_internal_info_without_changing_other_fields(): @@ -68,6 +72,7 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel "metadata": { "memory_type": "LongTermMemory", "internal_info": json.dumps(value), + "info": json.dumps({"client_metadata": True}), "relativity": 0.75, "sources": [{"type": "chat", "role": "user", "content": "Synthetic source."}], "session_id": "test-session", @@ -78,6 +83,7 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel item = TextualMemoryItem.from_dict(record) assert item.metadata.internal_info == value + assert item.metadata.info == {"client_metadata": True} assert item.id == record["id"] assert item.memory == record["memory"] assert item.metadata.sources[0].content == "Synthetic source." @@ -88,4 +94,5 @@ def test_search_item_from_dict_decodes_internal_info_without_changing_other_fiel # Graph storage serializes dictionaries as JSON strings again on the next write. stored = item.to_dict() stored["metadata"]["internal_info"] = json.dumps(stored["metadata"]["internal_info"]) + stored["metadata"]["info"] = json.dumps(stored["metadata"]["info"]) assert TextualMemoryItem.from_dict(stored).to_dict() == item.to_dict()