From e79304ea3d3cd9d5279fdaa9f7f4126d848fcca3 Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Sat, 3 Oct 2026 17:21:13 +0800 Subject: [PATCH 1/3] fix: stop sources duplication across chunked memory items (#2453) MultiModalStructMemReader._split_large_memory_item was attaching the parent item's full `sources` list to every chunk, and _build_window_from_items then concatenated them without dedup. The result: a message split into N chunks produced memory items whose `sources` list held N byte-identical copies of the original message. Production data showed 27% of memories affected, bloating storage, retrieval context (advanced_searcher re-injects sources into prompts verbatim), and provenance tooling. Fix is two-layer: * In _split_large_memory_item, attach `sources` only to chunk 0 (the owning chunk) and record source_chunk_index=0 in internal_info. Non-owning chunks keep ingest_batch_id + chunk_index + chunk_total for lineage discoverability. * In _build_window_from_items, dedup `all_sources` by a stable (type, role, message_id, chat_time, doc_path, content) signature so future regressions or multi-parser overlap also cannot inflate the list. Added 8 regression tests in tests/mem_reader/test_sources_dedup.py covering the propagation fix, the dedup fix, role-detection preservation, and an end-to-end _concat_multi_modal_memories invariant. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/memos/mem_reader/multi_modal_struct.py | 66 ++++- tests/mem_reader/test_sources_dedup.py | 330 +++++++++++++++++++++ 2 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 tests/mem_reader/test_sources_dedup.py diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index c46331f1d..1a1733149 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -31,6 +31,38 @@ logger = log.get_logger(__name__) +def _source_signature(source: Any) -> tuple: + """Stable, hashable signature for a `SourceMessage` (or dict) used by + `_build_window_from_items` to deduplicate sources across constituent + items of a sliding window. + + Two sources that reference the same origin message — same role, same + content, same locators — collapse to the same signature regardless of + object identity. See issue #2453. + + For objects that cannot be introspected (unexpected shapes) fall back + to Python object identity so a mystery object is at worst treated as + unique. + """ + if hasattr(source, "model_dump"): + try: + data = source.model_dump() + except Exception: # pragma: no cover — defensive + return (id(source),) + elif isinstance(source, dict): + data = source + else: + return (id(source),) + return ( + data.get("type"), + data.get("role"), + data.get("message_id"), + data.get("chat_time"), + data.get("doc_path"), + data.get("content"), + ) + + class MultiModalStructMemReader(SimpleStructMemReader): """Multimodal implementation of MemReader that inherits from SimpleStructMemReader.""" @@ -159,6 +191,20 @@ def _create_chunk_item(chunk_idx: int, chunk): "chunk_index": chunk_idx, "chunk_total": chunk_total, } + # Attach `sources` to the owning chunk only (issue #2453). + # If every chunk inherited the full parent `sources` list, the + # sliding-window aggregator (`_build_window_from_items`) would + # concatenate N copies of the same `SourceMessage` into one + # window — bloating storage, retrieval context, and provenance + # dedup. The parent sources belong to one semantic origin, so + # chunk 0 owns them; chunks 1..N-1 are discoverable via + # ingest_batch_id + chunk_index. + is_owning_chunk = chunk_idx == 0 + if is_owning_chunk: + chunk_sources = item.metadata.sources or [] + chunk_internal_info["source_chunk_index"] = 0 + else: + chunk_sources = [] # Create a new memory item for each chunk, preserving original metadata split_item = self._make_memory_item( value=chunk_text, @@ -166,7 +212,7 @@ def _create_chunk_item(chunk_idx: int, chunk): memory_type=item.metadata.memory_type, tags=item.metadata.tags or [], key=item.metadata.key, - sources=item.metadata.sources or [], + sources=chunk_sources, background=item.metadata.background or "", need_embed=False, ) @@ -332,6 +378,14 @@ def _build_window_from_items( # Collect all memory texts and sources memory_texts = [] all_sources = [] + # Dedup sources by a stable signature (issue #2453): a sliding + # window can legitimately contain multiple items that share the + # same origin SourceMessage (e.g. several chunks of one long + # input, or two parsers that both reference the same raw doc). + # Without dedup the window — and every downstream fine item — + # ends up carrying N identical copies, bloating storage and + # re-injecting the same original text N times at retrieval. + seen_source_sigs: set = set() roles = set() aggregated_file_ids: list[str] = [] ingest_batch_ids: set[str] = set() @@ -346,6 +400,16 @@ def _build_window_from_items( item_sources = [item_sources] for source in item_sources: + sig = _source_signature(source) + if sig in seen_source_sigs: + # Still extract role even if we drop the duplicate, + # so memory_type classification stays correct. + if hasattr(source, "role") and source.role: + roles.add(source.role) + elif isinstance(source, dict) and source.get("role"): + roles.add(source.get("role")) + continue + seen_source_sigs.add(sig) # Add source to all_sources all_sources.append(source) diff --git a/tests/mem_reader/test_sources_dedup.py b/tests/mem_reader/test_sources_dedup.py new file mode 100644 index 000000000..447da98c1 --- /dev/null +++ b/tests/mem_reader/test_sources_dedup.py @@ -0,0 +1,330 @@ +"""Regression tests for issue #2453: + `sources` array must not be duplicated across chunks when a large memory + item is split. + +Covers two layers of the fix: + * `_split_large_memory_item` attaches the parent `sources` list only to + the chunk-0 ("owning") sub-item. Non-owning chunks get `sources=[]`. + * `_build_window_from_items` deduplicates `sources` by a stable + (type, role, message_id, chat_time, doc_path, content) signature so + that even if upstream re-introduces duplicates, the aggregated window + carries at most one copy per unique origin message. +""" + +from __future__ import annotations + +import unittest + +from unittest.mock import MagicMock, patch + +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) + + +def _make_fast_item( + memory: str = "User said hello", + user_id: str = "u1", + session_id: str = "sess1", + role: str = "user", + sources: list | None = None, + internal_info: dict | None = None, +) -> TextualMemoryItem: + if sources is None: + sources = [ + SourceMessage( + type="chat", + role=role, + content=memory, + message_id="msg-1", + chat_time="2026-10-03T09:00:00Z", + ) + ] + return TextualMemoryItem( + memory=memory, + metadata=TreeNodeTextualMemoryMetadata( + user_id=user_id, + session_id=session_id, + memory_type="LongTermMemory", + sources=sources, + internal_info=internal_info, + ), + ) + + +def _fake_make_memory_item( + *, + value, + info, + memory_type, + tags, + key, + sources, + background, + need_embed, +): + """Thin stand-in for `_make_memory_item` that avoids touching the real + embedder.""" + return TextualMemoryItem( + memory=value, + metadata=TreeNodeTextualMemoryMetadata( + user_id=info.get("user_id", ""), + session_id=info.get("session_id", ""), + memory_type=memory_type, + tags=tags, + key=key, + sources=sources, + background=background, + ), + ) + + +class TestSplitLargeItemSourcesNotDuplicated(unittest.TestCase): + """`_split_large_memory_item` must not propagate full sources list to + every chunk — only to the first (owning) chunk.""" + + def setUp(self): + with patch.object(MultiModalStructMemReader, "__init__", lambda self, *a, **kw: None): + self.reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + self.reader.chunker = MagicMock() + self.reader._make_memory_item = MagicMock(side_effect=_fake_make_memory_item) + self.reader._count_tokens = MagicMock(return_value=9999) + + def test_only_first_chunk_inherits_sources(self): + """Given a fast item with 1 source split into 4 chunks, chunk 0 + carries the source; chunks 1..3 carry sources=[]. Total sources + across chunks == len(parent.sources).""" + self.reader.chunker.chunk.return_value = [ + "chunk 1", + "chunk 2", + "chunk 3", + "chunk 4", + ] + + source = SourceMessage( + type="chat", + role="user", + content="a" * 1456, + message_id="msg-1456", + chat_time="2026-10-03T09:00:00Z", + ) + parent = _make_fast_item("a" * 1456, sources=[source]) + + chunks = self.reader._split_large_memory_item(parent, max_tokens=10) + + self.assertEqual(len(chunks), 4, "chunker returned 4 chunks") + + # Chunk 0 is the owning chunk — carries the full source list. + self.assertEqual(len(chunks[0].metadata.sources), 1) + self.assertIs(chunks[0].metadata.sources[0], source) + + # Chunks 1..3 carry no sources (prevents multiplication). + for i, chunk in enumerate(chunks[1:], start=1): + self.assertEqual( + len(chunk.metadata.sources), + 0, + f"chunk {i} should have 0 sources, got {len(chunk.metadata.sources)}", + ) + + # Total count across all chunks == original source count. + total = sum(len(c.metadata.sources) for c in chunks) + self.assertEqual( + total, + 1, + f"Expected total sources across chunks == 1, got {total}. " + "Each chunk should NOT copy the full parent sources list.", + ) + + def test_owning_chunk_marker_in_internal_info(self): + """Owning chunk (chunk_index=0) must have source_chunk_index=0 in + internal_info; non-owning chunks must NOT have this key.""" + self.reader.chunker.chunk.return_value = ["chunk 1", "chunk 2", "chunk 3"] + + parent = _make_fast_item("x" * 500, internal_info={"origin": "doc"}) + + chunks = self.reader._split_large_memory_item(parent, max_tokens=10) + + self.assertEqual(len(chunks), 3) + self.assertEqual(chunks[0].metadata.internal_info.get("source_chunk_index"), 0) + for chunk in chunks[1:]: + self.assertNotIn("source_chunk_index", chunk.metadata.internal_info or {}) + + # Lineage keys preserved on every chunk. + for chunk in chunks: + self.assertIn("ingest_batch_id", chunk.metadata.internal_info) + self.assertIn("chunk_index", chunk.metadata.internal_info) + self.assertIn("chunk_total", chunk.metadata.internal_info) + self.assertEqual(chunk.metadata.internal_info["chunk_total"], 3) + # Pre-existing internal_info keys propagate. + self.assertEqual(chunk.metadata.internal_info["origin"], "doc") + + def test_split_bails_before_marking_when_only_one_chunk(self): + """If chunker returns only 1 chunk the split path still returns 1 + item carrying the original sources — no new source_chunk_index + marker required.""" + self.reader.chunker.chunk.return_value = ["only chunk"] + source = SourceMessage(type="chat", role="user", content="hi") + parent = _make_fast_item("hi", sources=[source]) + + chunks = self.reader._split_large_memory_item(parent, max_tokens=10) + + self.assertEqual(len(chunks), 1) + self.assertEqual(chunks[0].metadata.sources, [source]) + + +class TestBuildWindowDedupesSources(unittest.TestCase): + """`_build_window_from_items` must deduplicate sources so that a + sliding window containing N chunks that share one original source ends + up with 1 source, not N.""" + + def setUp(self): + with patch.object(MultiModalStructMemReader, "__init__", lambda self, *a, **kw: None): + self.reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + self.reader.embedder = MagicMock() + self.reader.embedder.embed.return_value = [[0.1] * 8] + + def test_window_dedupes_identical_sources(self): + """4 chunks each carrying the SAME SourceMessage must yield a + window whose sources list has length 1 (not 4).""" + shared_source = SourceMessage( + type="chat", + role="user", + content="hello world", + message_id="m-1", + chat_time="2026-10-03T09:00:00Z", + ) + items = [_make_fast_item(f"chunk {i}", sources=[shared_source]) for i in range(4)] + + window = self.reader._build_window_from_items( + items, {"user_id": "u1", "session_id": "sess1"} + ) + + self.assertIsNotNone(window) + self.assertEqual( + len(window.metadata.sources), + 1, + f"Expected 1 deduped source, got {len(window.metadata.sources)} — " + "window is re-duplicating identical sources.", + ) + + def test_window_dedupes_structurally_identical_but_distinct_objects(self): + """Two separate SourceMessage instances with identical fields + must also dedupe — not just reference-equal ones.""" + s1 = SourceMessage( + type="chat", + role="user", + content="hello world", + message_id="m-1", + chat_time="2026-10-03T09:00:00Z", + ) + s2 = SourceMessage( + type="chat", + role="user", + content="hello world", + message_id="m-1", + chat_time="2026-10-03T09:00:00Z", + ) + self.assertIsNot(s1, s2, "test needs two distinct SourceMessage objects") + + items = [ + _make_fast_item("part A", sources=[s1]), + _make_fast_item("part B", sources=[s2]), + ] + + window = self.reader._build_window_from_items( + items, {"user_id": "u1", "session_id": "sess1"} + ) + + self.assertIsNotNone(window) + self.assertEqual(len(window.metadata.sources), 1) + + def test_window_keeps_distinct_sources(self): + """Legitimately distinct sources must NOT be deduped.""" + s1 = SourceMessage(type="chat", role="user", content="first", message_id="m-1") + s2 = SourceMessage(type="chat", role="user", content="second", message_id="m-2") + + items = [ + _make_fast_item("first", sources=[s1]), + _make_fast_item("second", sources=[s2]), + ] + + window = self.reader._build_window_from_items( + items, {"user_id": "u1", "session_id": "sess1"} + ) + + self.assertIsNotNone(window) + self.assertEqual(len(window.metadata.sources), 2) + contents = {s.content for s in window.metadata.sources} + self.assertEqual(contents, {"first", "second"}) + + def test_window_role_detection_still_works_after_dedup(self): + """Dedup must not break the role-based memory_type assignment + (UserMemory when sources only contain `user` role).""" + source = SourceMessage(type="chat", role="user", content="hi", message_id="m-1") + items = [_make_fast_item(f"c{i}", sources=[source]) for i in range(3)] + + window = self.reader._build_window_from_items( + items, {"user_id": "u1", "session_id": "sess1"} + ) + + self.assertIsNotNone(window) + self.assertEqual(window.metadata.memory_type, "UserMemory") + + +class TestEndToEndSourcesCountInvariant(unittest.TestCase): + """End-to-end: a fast item fed through `_concat_multi_modal_memories` + produces fast-mode windows whose sources are not duplicated.""" + + def setUp(self): + with patch.object(MultiModalStructMemReader, "__init__", lambda self, *a, **kw: None): + self.reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + self.reader.embedder = MagicMock() + self.reader.embedder.embed.return_value = [[0.1] * 8] + self.reader.chunker = MagicMock() + self.reader._make_memory_item = MagicMock(side_effect=_fake_make_memory_item) + self.reader.chat_window_max_tokens = 10 + + def test_long_message_produces_windows_with_single_source(self): + """Reproduces the issue scenario: a single long message gets + chunked into N; the resulting fast-mode windows must carry 1 + source each, not N.""" + # Simulate a long input: tokens high enough to trigger split. + token_calls = {"n": 0} + + def fake_count(_text: str) -> int: + token_calls["n"] += 1 + # First call = whole-item token check in _concat → returns big + # subsequent calls (per-window cur_text) stay below the cap so + # the sliding window happily puts all chunks into one window. + return 999 if token_calls["n"] == 1 else 1 + + self.reader._count_tokens = MagicMock(side_effect=fake_count) + self.reader.chunker.chunk.return_value = ["c1", "c2", "c3", "c4"] + + shared_source = SourceMessage( + type="chat", + role="user", + content="x" * 1456, + message_id="msg-xxl", + chat_time="2026-10-03T09:00:00Z", + ) + parent = _make_fast_item("x" * 1456, sources=[shared_source]) + + windows = self.reader._concat_multi_modal_memories([parent], max_tokens=10) + + self.assertTrue(len(windows) >= 1) + for w in windows: + # The key assertion: no source multiplication. + self.assertLessEqual( + len(w.metadata.sources), + 1, + f"Window carries {len(w.metadata.sources)} sources; " + "expected ≤ 1 for a single-origin long message.", + ) + + +if __name__ == "__main__": + unittest.main() From 73621315b028232af3b75179361af75b12f58cfa Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Sat, 3 Oct 2026 17:38:09 +0800 Subject: [PATCH 2/3] fix(mem_reader): address OCR review on sources dedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four follow-ups from the open code review on #2454: 1. `_source_signature` now folds in all non-None keys returned by `model_dump` instead of only the six known fields. SourceMessage declares `extra="allow"`, so two paragraphs from the same document that differ only in an extra locator (page, offset, span, …) and share `message_id=None` would otherwise collapse incorrectly after dedup. Unhashable extras fall back to object identity so we never silently merge distinct sources. 2. `test_long_message_produces_windows_with_single_source` additionally asserts that at least one window actually carries the owning source. The pre-existing `≤ 1` check alone would pass a buggy implementation that dropped the source entirely. 3. `test_split_bails_before_marking_when_only_one_chunk` now matches the implementation: chunk 0 is the owning chunk even when chunk_total==1, so `source_chunk_index=0` is set and the test asserts it. Docstring updated. 4. `test_window_role_detection_still_works_after_dedup` additionally asserts `len(window.metadata.sources) == 1` so a dedup regression that strips every source cannot slip through on role defaulting. Also adds `test_window_keeps_sources_distinguished_by_extra_fields` to guard the first fix against future regressions. Co-Authored-By: Claude Opus 4.7 --- src/memos/mem_reader/multi_modal_struct.py | 37 ++++++---- tests/mem_reader/test_sources_dedup.py | 83 ++++++++++++++++++++-- 2 files changed, 100 insertions(+), 20 deletions(-) diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index 1a1733149..c87a350c0 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -37,12 +37,21 @@ def _source_signature(source: Any) -> tuple: items of a sliding window. Two sources that reference the same origin message — same role, same - content, same locators — collapse to the same signature regardless of - object identity. See issue #2453. - - For objects that cannot be introspected (unexpected shapes) fall back - to Python object identity so a mystery object is at worst treated as - unique. + content, same locators, AND same extra provenance (url, page, offset, + span, local_confidence, …) — collapse to the same signature regardless + of object identity. See issue #2453. + + `SourceMessage` declares ``model_config = ConfigDict(extra="allow")`` so + callers can attach arbitrary provenance attributes. The signature + therefore folds in every non-None key returned by ``model_dump`` (not + just the six known fields): two paragraphs from the same document that + differ only in ``page`` / ``offset`` must remain distinct after dedup. + + For objects that cannot be introspected (unexpected shapes) or that + carry unhashable extra values (e.g. a ``dict`` or ``list`` in a + provenance field), fall back to Python object identity so a mystery + source is at worst treated as unique rather than being incorrectly + collapsed with another. """ if hasattr(source, "model_dump"): try: @@ -53,14 +62,14 @@ def _source_signature(source: Any) -> tuple: data = source else: return (id(source),) - return ( - data.get("type"), - data.get("role"), - data.get("message_id"), - data.get("chat_time"), - data.get("doc_path"), - data.get("content"), - ) + # Fully-deterministic signature across all fields, including extras + # allowed by SourceMessage's ConfigDict(extra="allow"). + try: + return tuple(sorted((k, v) for k, v in data.items() if v is not None)) + except TypeError: + # Unhashable value in an extra field (e.g. a dict / list) — fall + # back to identity so we never silently collapse distinct sources. + return (id(source),) class MultiModalStructMemReader(SimpleStructMemReader): diff --git a/tests/mem_reader/test_sources_dedup.py b/tests/mem_reader/test_sources_dedup.py index 447da98c1..5d54f5035 100644 --- a/tests/mem_reader/test_sources_dedup.py +++ b/tests/mem_reader/test_sources_dedup.py @@ -162,9 +162,11 @@ def test_owning_chunk_marker_in_internal_info(self): self.assertEqual(chunk.metadata.internal_info["origin"], "doc") def test_split_bails_before_marking_when_only_one_chunk(self): - """If chunker returns only 1 chunk the split path still returns 1 - item carrying the original sources — no new source_chunk_index - marker required.""" + """If the chunker returns only 1 chunk the split path still returns + 1 item carrying the original sources. The single chunk IS the + owning chunk (chunk_index=0), so ``source_chunk_index=0`` is also + recorded in ``internal_info`` — this makes the owning-chunk marker + uniformly discoverable regardless of chunk_total.""" self.reader.chunker.chunk.return_value = ["only chunk"] source = SourceMessage(type="chat", role="user", content="hi") parent = _make_fast_item("hi", sources=[source]) @@ -173,6 +175,12 @@ def test_split_bails_before_marking_when_only_one_chunk(self): self.assertEqual(len(chunks), 1) self.assertEqual(chunks[0].metadata.sources, [source]) + # Single chunk is also the owning chunk — marker must be present + # so downstream consumers can locate the source-carrying chunk by + # internal_info["source_chunk_index"] without a chunk_total probe. + self.assertEqual( + chunks[0].metadata.internal_info.get("source_chunk_index"), 0 + ) class TestBuildWindowDedupesSources(unittest.TestCase): @@ -260,9 +268,56 @@ def test_window_keeps_distinct_sources(self): contents = {s.content for s in window.metadata.sources} self.assertEqual(contents, {"first", "second"}) + def test_window_keeps_sources_distinguished_by_extra_fields(self): + """SourceMessage declares extra='allow', so two paragraphs from the + same document that differ only in an extra locator (page, offset, + span, …) must remain distinct after dedup. Both have + ``message_id=None`` and identical core fields, so a 6-field + signature would collapse them — regressing provenance granularity. + See issue #2453 OCR follow-up.""" + s1 = SourceMessage( + type="doc", + role=None, + content="paragraph body", + message_id=None, + doc_path="docs/handbook.md", + page=12, + ) + s2 = SourceMessage( + type="doc", + role=None, + content="paragraph body", + message_id=None, + doc_path="docs/handbook.md", + page=37, + ) + + items = [ + _make_fast_item("first", sources=[s1]), + _make_fast_item("second", sources=[s2]), + ] + + window = self.reader._build_window_from_items( + items, {"user_id": "u1", "session_id": "sess1"} + ) + + self.assertIsNotNone(window) + self.assertEqual( + len(window.metadata.sources), + 2, + "Sources that differ only in extra fields (e.g. page) must not " + "be collapsed by the signature — provenance granularity lost.", + ) + pages = {getattr(s, "page", None) for s in window.metadata.sources} + self.assertEqual(pages, {12, 37}) + def test_window_role_detection_still_works_after_dedup(self): """Dedup must not break the role-based memory_type assignment - (UserMemory when sources only contain `user` role).""" + (UserMemory when sources only contain `user` role) AND must retain + the owning source. Without the sources-count assertion, a buggy + dedup that strips the user-role source entirely would still leave + ``roles`` empty — role detection then defaults away from + ``assistant`` and the test passes as a false positive.""" source = SourceMessage(type="chat", role="user", content="hi", message_id="m-1") items = [_make_fast_item(f"c{i}", sources=[source]) for i in range(3)] @@ -271,6 +326,11 @@ def test_window_role_detection_still_works_after_dedup(self): ) self.assertIsNotNone(window) + self.assertEqual( + len(window.metadata.sources), + 1, + "Dedup must retain exactly 1 source, not drop all", + ) self.assertEqual(window.metadata.memory_type, "UserMemory") @@ -316,12 +376,23 @@ def fake_count(_text: str) -> int: windows = self.reader._concat_multi_modal_memories([parent], max_tokens=10) self.assertTrue(len(windows) >= 1) + # At least one window must carry the owning source. If dedup + # over-collapses (or the owning chunk is dropped entirely), + # every window ends up with `sources=[]` and the ≤ 1 check + # alone would silently pass — defeating the regression guard. + windows_with_sources = [w for w in windows if w.metadata.sources] + self.assertGreaterEqual( + len(windows_with_sources), + 1, + "At least one window must carry the owning source; no window had any " + "source — possible regression where the source is being dropped entirely.", + ) for w in windows: # The key assertion: no source multiplication. self.assertLessEqual( - len(w.metadata.sources), + len(w.metadata.sources or []), 1, - f"Window carries {len(w.metadata.sources)} sources; " + f"Window carries {len(w.metadata.sources or [])} sources; " "expected ≤ 1 for a single-origin long message.", ) From 9aca576db15024e3dcc771f8a2b2d71790999aa2 Mon Sep 17 00:00:00 2001 From: Wenqiang Wei <46308778+endxxxx@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:05:35 +0800 Subject: [PATCH 3/3] fix: preserve provenance when deduplicating window sources --- src/memos/mem_reader/multi_modal_struct.py | 97 ++--- tests/mem_reader/test_sources_dedup.py | 434 +++------------------ 2 files changed, 90 insertions(+), 441 deletions(-) diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index c87a350c0..c7dd5e76c 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -4,6 +4,7 @@ import traceback import uuid +from collections.abc import Hashable from typing import TYPE_CHECKING, Any from memos import log @@ -15,7 +16,11 @@ from memos.mem_reader.read_skill_memory.process_skill_memory import process_skill_memory_fine from memos.mem_reader.simple_struct import PROMPT_DICT, SimpleStructMemReader from memos.mem_reader.utils import parse_json_result -from memos.memories.textual.item import TextualMemoryItem, TreeNodeTextualMemoryMetadata +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) from memos.plugins.hook_defs import H from memos.plugins.hooks import trigger_hook, trigger_single_hook from memos.templates.mem_reader_prompts import MEMORY_MERGE_PROMPT_EN, MEMORY_MERGE_PROMPT_ZH @@ -31,45 +36,26 @@ logger = log.get_logger(__name__) -def _source_signature(source: Any) -> tuple: - """Stable, hashable signature for a `SourceMessage` (or dict) used by - `_build_window_from_items` to deduplicate sources across constituent - items of a sliding window. - - Two sources that reference the same origin message — same role, same - content, same locators, AND same extra provenance (url, page, offset, - span, local_confidence, …) — collapse to the same signature regardless - of object identity. See issue #2453. - - `SourceMessage` declares ``model_config = ConfigDict(extra="allow")`` so - callers can attach arbitrary provenance attributes. The signature - therefore folds in every non-None key returned by ``model_dump`` (not - just the six known fields): two paragraphs from the same document that - differ only in ``page`` / ``offset`` must remain distinct after dedup. - - For objects that cannot be introspected (unexpected shapes) or that - carry unhashable extra values (e.g. a ``dict`` or ``list`` in a - provenance field), fall back to Python object identity so a mystery - source is at worst treated as unique rather than being incorrectly - collapsed with another. - """ - if hasattr(source, "model_dump"): - try: - data = source.model_dump() - except Exception: # pragma: no cover — defensive - return (id(source),) - elif isinstance(source, dict): - data = source - else: - return (id(source),) - # Fully-deterministic signature across all fields, including extras - # allowed by SourceMessage's ConfigDict(extra="allow"). +def _freeze_source_value(value: Any) -> Hashable: + """Ignore mapping order and preserve sequence order in provenance.""" + if isinstance(value, dict): + return dict, frozenset((key, _freeze_source_value(item)) for key, item in value.items()) + if isinstance(value, list | tuple): + return type(value), tuple(_freeze_source_value(item) for item in value) + return type(value), value + + +def _source_signature(source: SourceMessage | dict[str, Any]) -> Hashable: + """Include all supplied provenance fields when deduplicating a window.""" try: - return tuple(sorted((k, v) for k, v in data.items() if v is not None)) - except TypeError: - # Unhashable value in an extra field (e.g. a dict / list) — fall - # back to identity so we never silently collapse distinct sources. - return (id(source),) + data = source.model_dump() if isinstance(source, SourceMessage) else source + signature = frozenset( + (key, _freeze_source_value(value)) for key, value in data.items() if value is not None + ) + except (AttributeError, TypeError, ValueError, RecursionError): + # Keep opaque provenance distinct unless it is the same object. + return id(source) + return signature class MultiModalStructMemReader(SimpleStructMemReader): @@ -200,20 +186,6 @@ def _create_chunk_item(chunk_idx: int, chunk): "chunk_index": chunk_idx, "chunk_total": chunk_total, } - # Attach `sources` to the owning chunk only (issue #2453). - # If every chunk inherited the full parent `sources` list, the - # sliding-window aggregator (`_build_window_from_items`) would - # concatenate N copies of the same `SourceMessage` into one - # window — bloating storage, retrieval context, and provenance - # dedup. The parent sources belong to one semantic origin, so - # chunk 0 owns them; chunks 1..N-1 are discoverable via - # ingest_batch_id + chunk_index. - is_owning_chunk = chunk_idx == 0 - if is_owning_chunk: - chunk_sources = item.metadata.sources or [] - chunk_internal_info["source_chunk_index"] = 0 - else: - chunk_sources = [] # Create a new memory item for each chunk, preserving original metadata split_item = self._make_memory_item( value=chunk_text, @@ -221,7 +193,8 @@ def _create_chunk_item(chunk_idx: int, chunk): memory_type=item.metadata.memory_type, tags=item.metadata.tags or [], key=item.metadata.key, - sources=chunk_sources, + # A chunk can land in a different window from the first chunk. + sources=item.metadata.sources or [], background=item.metadata.background or "", need_embed=False, ) @@ -387,14 +360,8 @@ def _build_window_from_items( # Collect all memory texts and sources memory_texts = [] all_sources = [] - # Dedup sources by a stable signature (issue #2453): a sliding - # window can legitimately contain multiple items that share the - # same origin SourceMessage (e.g. several chunks of one long - # input, or two parsers that both reference the same raw doc). - # Without dedup the window — and every downstream fine item — - # ends up carrying N identical copies, bloating storage and - # re-injecting the same original text N times at retrieval. - seen_source_sigs: set = set() + # Keep one copy of each source per window while preserving its first position. + seen_source_sigs: set[Hashable] = set() roles = set() aggregated_file_ids: list[str] = [] ingest_batch_ids: set[str] = set() @@ -411,12 +378,6 @@ def _build_window_from_items( for source in item_sources: sig = _source_signature(source) if sig in seen_source_sigs: - # Still extract role even if we drop the duplicate, - # so memory_type classification stays correct. - if hasattr(source, "role") and source.role: - roles.add(source.role) - elif isinstance(source, dict) and source.get("role"): - roles.add(source.get("role")) continue seen_source_sigs.add(sig) # Add source to all_sources diff --git a/tests/mem_reader/test_sources_dedup.py b/tests/mem_reader/test_sources_dedup.py index 5d54f5035..36322af18 100644 --- a/tests/mem_reader/test_sources_dedup.py +++ b/tests/mem_reader/test_sources_dedup.py @@ -1,22 +1,8 @@ -"""Regression tests for issue #2453: - `sources` array must not be duplicated across chunks when a large memory - item is split. +from unittest.mock import Mock -Covers two layers of the fix: - * `_split_large_memory_item` attaches the parent `sources` list only to - the chunk-0 ("owning") sub-item. Non-owning chunks get `sources=[]`. - * `_build_window_from_items` deduplicates `sources` by a stable - (type, role, message_id, chat_time, doc_path, content) signature so - that even if upstream re-introduces duplicates, the aggregated window - carries at most one copy per unique origin message. -""" - -from __future__ import annotations - -import unittest - -from unittest.mock import MagicMock, patch +import pytest +from memos.chunkers.simple_chunker import SimpleTextSplitter from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader from memos.memories.textual.item import ( SourceMessage, @@ -25,377 +11,79 @@ ) -def _make_fast_item( - memory: str = "User said hello", - user_id: str = "u1", - session_id: str = "sess1", - role: str = "user", - sources: list | None = None, - internal_info: dict | None = None, -) -> TextualMemoryItem: - if sources is None: - sources = [ - SourceMessage( - type="chat", - role=role, - content=memory, - message_id="msg-1", - chat_time="2026-10-03T09:00:00Z", - ) - ] - return TextualMemoryItem( - memory=memory, - metadata=TreeNodeTextualMemoryMetadata( - user_id=user_id, - session_id=session_id, - memory_type="LongTermMemory", - sources=sources, - internal_info=internal_info, - ), - ) +@pytest.fixture +def reader(): + reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + reader.chunker = SimpleTextSplitter(chunk_size=4, chunk_overlap=0) + reader._count_tokens = len + reader.embedder = Mock() + reader.embedder.embed.side_effect = lambda texts: [[0.1] for _ in texts] + return reader -def _fake_make_memory_item( - *, - value, - info, - memory_type, - tags, - key, - sources, - background, - need_embed, -): - """Thin stand-in for `_make_memory_item` that avoids touching the real - embedder.""" +def make_item(text: str, *sources: SourceMessage) -> TextualMemoryItem: return TextualMemoryItem( - memory=value, + memory=text, metadata=TreeNodeTextualMemoryMetadata( - user_id=info.get("user_id", ""), - session_id=info.get("session_id", ""), - memory_type=memory_type, - tags=tags, - key=key, - sources=sources, - background=background, + user_id="test-user", + session_id="test-session", + memory_type="UserMemory", + sources=list(sources), ), ) -class TestSplitLargeItemSourcesNotDuplicated(unittest.TestCase): - """`_split_large_memory_item` must not propagate full sources list to - every chunk — only to the first (owning) chunk.""" - - def setUp(self): - with patch.object(MultiModalStructMemReader, "__init__", lambda self, *a, **kw: None): - self.reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) - self.reader.chunker = MagicMock() - self.reader._make_memory_item = MagicMock(side_effect=_fake_make_memory_item) - self.reader._count_tokens = MagicMock(return_value=9999) - - def test_only_first_chunk_inherits_sources(self): - """Given a fast item with 1 source split into 4 chunks, chunk 0 - carries the source; chunks 1..3 carry sources=[]. Total sources - across chunks == len(parent.sources).""" - self.reader.chunker.chunk.return_value = [ - "chunk 1", - "chunk 2", - "chunk 3", - "chunk 4", - ] - - source = SourceMessage( - type="chat", - role="user", - content="a" * 1456, - message_id="msg-1456", - chat_time="2026-10-03T09:00:00Z", - ) - parent = _make_fast_item("a" * 1456, sources=[source]) - - chunks = self.reader._split_large_memory_item(parent, max_tokens=10) - - self.assertEqual(len(chunks), 4, "chunker returned 4 chunks") - - # Chunk 0 is the owning chunk — carries the full source list. - self.assertEqual(len(chunks[0].metadata.sources), 1) - self.assertIs(chunks[0].metadata.sources[0], source) - - # Chunks 1..3 carry no sources (prevents multiplication). - for i, chunk in enumerate(chunks[1:], start=1): - self.assertEqual( - len(chunk.metadata.sources), - 0, - f"chunk {i} should have 0 sources, got {len(chunk.metadata.sources)}", - ) - - # Total count across all chunks == original source count. - total = sum(len(c.metadata.sources) for c in chunks) - self.assertEqual( - total, - 1, - f"Expected total sources across chunks == 1, got {total}. " - "Each chunk should NOT copy the full parent sources list.", - ) - - def test_owning_chunk_marker_in_internal_info(self): - """Owning chunk (chunk_index=0) must have source_chunk_index=0 in - internal_info; non-owning chunks must NOT have this key.""" - self.reader.chunker.chunk.return_value = ["chunk 1", "chunk 2", "chunk 3"] - - parent = _make_fast_item("x" * 500, internal_info={"origin": "doc"}) - - chunks = self.reader._split_large_memory_item(parent, max_tokens=10) - - self.assertEqual(len(chunks), 3) - self.assertEqual(chunks[0].metadata.internal_info.get("source_chunk_index"), 0) - for chunk in chunks[1:]: - self.assertNotIn("source_chunk_index", chunk.metadata.internal_info or {}) - - # Lineage keys preserved on every chunk. - for chunk in chunks: - self.assertIn("ingest_batch_id", chunk.metadata.internal_info) - self.assertIn("chunk_index", chunk.metadata.internal_info) - self.assertIn("chunk_total", chunk.metadata.internal_info) - self.assertEqual(chunk.metadata.internal_info["chunk_total"], 3) - # Pre-existing internal_info keys propagate. - self.assertEqual(chunk.metadata.internal_info["origin"], "doc") - - def test_split_bails_before_marking_when_only_one_chunk(self): - """If the chunker returns only 1 chunk the split path still returns - 1 item carrying the original sources. The single chunk IS the - owning chunk (chunk_index=0), so ``source_chunk_index=0`` is also - recorded in ``internal_info`` — this makes the owning-chunk marker - uniformly discoverable regardless of chunk_total.""" - self.reader.chunker.chunk.return_value = ["only chunk"] - source = SourceMessage(type="chat", role="user", content="hi") - parent = _make_fast_item("hi", sources=[source]) - - chunks = self.reader._split_large_memory_item(parent, max_tokens=10) - - self.assertEqual(len(chunks), 1) - self.assertEqual(chunks[0].metadata.sources, [source]) - # Single chunk is also the owning chunk — marker must be present - # so downstream consumers can locate the source-carrying chunk by - # internal_info["source_chunk_index"] without a chunk_total probe. - self.assertEqual( - chunks[0].metadata.internal_info.get("source_chunk_index"), 0 - ) - - -class TestBuildWindowDedupesSources(unittest.TestCase): - """`_build_window_from_items` must deduplicate sources so that a - sliding window containing N chunks that share one original source ends - up with 1 source, not N.""" - - def setUp(self): - with patch.object(MultiModalStructMemReader, "__init__", lambda self, *a, **kw: None): - self.reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) - self.reader.embedder = MagicMock() - self.reader.embedder.embed.return_value = [[0.1] * 8] - - def test_window_dedupes_identical_sources(self): - """4 chunks each carrying the SAME SourceMessage must yield a - window whose sources list has length 1 (not 4).""" - shared_source = SourceMessage( - type="chat", - role="user", - content="hello world", - message_id="m-1", - chat_time="2026-10-03T09:00:00Z", - ) - items = [_make_fast_item(f"chunk {i}", sources=[shared_source]) for i in range(4)] - - window = self.reader._build_window_from_items( - items, {"user_id": "u1", "session_id": "sess1"} - ) - - self.assertIsNotNone(window) - self.assertEqual( - len(window.metadata.sources), - 1, - f"Expected 1 deduped source, got {len(window.metadata.sources)} — " - "window is re-duplicating identical sources.", - ) - - def test_window_dedupes_structurally_identical_but_distinct_objects(self): - """Two separate SourceMessage instances with identical fields - must also dedupe — not just reference-equal ones.""" - s1 = SourceMessage( - type="chat", - role="user", - content="hello world", - message_id="m-1", - chat_time="2026-10-03T09:00:00Z", - ) - s2 = SourceMessage( - type="chat", - role="user", - content="hello world", - message_id="m-1", - chat_time="2026-10-03T09:00:00Z", - ) - self.assertIsNot(s1, s2, "test needs two distinct SourceMessage objects") - - items = [ - _make_fast_item("part A", sources=[s1]), - _make_fast_item("part B", sources=[s2]), - ] - - window = self.reader._build_window_from_items( - items, {"user_id": "u1", "session_id": "sess1"} - ) - - self.assertIsNotNone(window) - self.assertEqual(len(window.metadata.sources), 1) - - def test_window_keeps_distinct_sources(self): - """Legitimately distinct sources must NOT be deduped.""" - s1 = SourceMessage(type="chat", role="user", content="first", message_id="m-1") - s2 = SourceMessage(type="chat", role="user", content="second", message_id="m-2") - - items = [ - _make_fast_item("first", sources=[s1]), - _make_fast_item("second", sources=[s2]), - ] - - window = self.reader._build_window_from_items( - items, {"user_id": "u1", "session_id": "sess1"} - ) - - self.assertIsNotNone(window) - self.assertEqual(len(window.metadata.sources), 2) - contents = {s.content for s in window.metadata.sources} - self.assertEqual(contents, {"first", "second"}) - - def test_window_keeps_sources_distinguished_by_extra_fields(self): - """SourceMessage declares extra='allow', so two paragraphs from the - same document that differ only in an extra locator (page, offset, - span, …) must remain distinct after dedup. Both have - ``message_id=None`` and identical core fields, so a 6-field - signature would collapse them — regressing provenance granularity. - See issue #2453 OCR follow-up.""" - s1 = SourceMessage( - type="doc", - role=None, - content="paragraph body", - message_id=None, - doc_path="docs/handbook.md", - page=12, - ) - s2 = SourceMessage( - type="doc", - role=None, - content="paragraph body", - message_id=None, - doc_path="docs/handbook.md", - page=37, - ) - - items = [ - _make_fast_item("first", sources=[s1]), - _make_fast_item("second", sources=[s2]), - ] - - window = self.reader._build_window_from_items( - items, {"user_id": "u1", "session_id": "sess1"} - ) - - self.assertIsNotNone(window) - self.assertEqual( - len(window.metadata.sources), - 2, - "Sources that differ only in extra fields (e.g. page) must not " - "be collapsed by the signature — provenance granularity lost.", - ) - pages = {getattr(s, "page", None) for s in window.metadata.sources} - self.assertEqual(pages, {12, 37}) - - def test_window_role_detection_still_works_after_dedup(self): - """Dedup must not break the role-based memory_type assignment - (UserMemory when sources only contain `user` role) AND must retain - the owning source. Without the sources-count assertion, a buggy - dedup that strips the user-role source entirely would still leave - ``roles`` empty — role detection then defaults away from - ``assistant`` and the test passes as a false positive.""" - source = SourceMessage(type="chat", role="user", content="hi", message_id="m-1") - items = [_make_fast_item(f"c{i}", sources=[source]) for i in range(3)] - - window = self.reader._build_window_from_items( - items, {"user_id": "u1", "session_id": "sess1"} - ) - - self.assertIsNotNone(window) - self.assertEqual( - len(window.metadata.sources), - 1, - "Dedup must retain exactly 1 source, not drop all", - ) - self.assertEqual(window.metadata.memory_type, "UserMemory") - +def test_window_deduplicates_nested_provenance_in_source_order(reader): + source = SourceMessage( + type="doc", + doc_path="sample.md", + content="A paragraph.", + file_info={"locator": {"pages": [1, 2], "offset": 4}}, + span=[{"start": 4, "end": 16}], + ) + duplicate = SourceMessage( + type="doc", + doc_path="sample.md", + content="A paragraph.", + file_info={"locator": {"offset": 4, "pages": [1, 2]}}, + span=[{"end": 16, "start": 4}], + ) + different_span = duplicate.model_copy(update={"span": [{"start": 20, "end": 32}]}) + different_page_order = duplicate.model_copy( + update={"file_info": {"locator": {"pages": [2, 1], "offset": 4}}} + ) -class TestEndToEndSourcesCountInvariant(unittest.TestCase): - """End-to-end: a fast item fed through `_concat_multi_modal_memories` - produces fast-mode windows whose sources are not duplicated.""" + window = reader._build_window_from_items( + [ + make_item("first", source), + make_item("second", duplicate, different_span, different_page_order), + ], + {}, + ) - def setUp(self): - with patch.object(MultiModalStructMemReader, "__init__", lambda self, *a, **kw: None): - self.reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) - self.reader.embedder = MagicMock() - self.reader.embedder.embed.return_value = [[0.1] * 8] - self.reader.chunker = MagicMock() - self.reader._make_memory_item = MagicMock(side_effect=_fake_make_memory_item) - self.reader.chat_window_max_tokens = 10 + assert window.metadata.sources == [source, different_span, different_page_order] - def test_long_message_produces_windows_with_single_source(self): - """Reproduces the issue scenario: a single long message gets - chunked into N; the resulting fast-mode windows must carry 1 - source each, not N.""" - # Simulate a long input: tokens high enough to trigger split. - token_calls = {"n": 0} - def fake_count(_text: str) -> int: - token_calls["n"] += 1 - # First call = whole-item token check in _concat → returns big - # subsequent calls (per-window cur_text) stay below the cap so - # the sliding window happily puts all chunks into one window. - return 999 if token_calls["n"] == 1 else 1 +def test_long_message_keeps_one_source_per_window(reader): + text = "aaaabbbbccccdddd" + source = SourceMessage(type="chat", role="user", content=text, message_id="msg-1") - self.reader._count_tokens = MagicMock(side_effect=fake_count) - self.reader.chunker.chunk.return_value = ["c1", "c2", "c3", "c4"] + windows = reader._concat_multi_modal_memories( + [make_item(text, source)], max_tokens=10, overlap=0 + ) - shared_source = SourceMessage( - type="chat", - role="user", - content="x" * 1456, - message_id="msg-xxl", - chat_time="2026-10-03T09:00:00Z", - ) - parent = _make_fast_item("x" * 1456, sources=[shared_source]) + assert [window.memory for window in windows] == ["aaaabbbb", "ccccdddd"] + assert [window.metadata.sources for window in windows] == [[source], [source]] + assert [window.metadata.memory_type for window in windows] == ["UserMemory", "UserMemory"] - windows = self.reader._concat_multi_modal_memories([parent], max_tokens=10) - self.assertTrue(len(windows) >= 1) - # At least one window must carry the owning source. If dedup - # over-collapses (or the owning chunk is dropped entirely), - # every window ends up with `sources=[]` and the ≤ 1 check - # alone would silently pass — defeating the regression guard. - windows_with_sources = [w for w in windows if w.metadata.sources] - self.assertGreaterEqual( - len(windows_with_sources), - 1, - "At least one window must carry the owning source; no window had any " - "source — possible regression where the source is being dropped entirely.", - ) - for w in windows: - # The key assertion: no source multiplication. - self.assertLessEqual( - len(w.metadata.sources or []), - 1, - f"Window carries {len(w.metadata.sources or [])} sources; " - "expected ≤ 1 for a single-origin long message.", - ) +def test_window_keeps_identical_text_from_distinct_messages(reader): + first = SourceMessage(role="user", content="yes", message_id="msg-1") + second = SourceMessage(role="assistant", content="yes", message_id="msg-2") + window = reader._build_window_from_items( + [make_item("first", first), make_item("second", second, first.model_copy())], {} + ) -if __name__ == "__main__": - unittest.main() + assert window.metadata.sources == [first, second] + assert window.metadata.memory_type == "LongTermMemory"