Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 74 additions & 1 deletion src/memos/mem_reader/multi_modal_struct.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,47 @@
logger = log.get_logger(__name__)


def _source_signature(source: Any) -> tuple:
"""Stable, hashable signature for a `SourceMessage` (or dict) used by
`_build_window_from_items` to deduplicate sources across constituent
items of a sliding window.

Two sources that reference the same origin message — same role, same
content, same locators, AND same extra provenance (url, page, offset,
span, local_confidence, …) — collapse to the same signature regardless
of object identity. See issue #2453.

`SourceMessage` declares ``model_config = ConfigDict(extra="allow")`` so
callers can attach arbitrary provenance attributes. The signature
therefore folds in every non-None key returned by ``model_dump`` (not
just the six known fields): two paragraphs from the same document that
differ only in ``page`` / ``offset`` must remain distinct after dedup.

For objects that cannot be introspected (unexpected shapes) or that
carry unhashable extra values (e.g. a ``dict`` or ``list`` in a
provenance field), fall back to Python object identity so a mystery
source is at worst treated as unique rather than being incorrectly
collapsed with another.
"""
if hasattr(source, "model_dump"):
try:
data = source.model_dump()
except Exception: # pragma: no cover — defensive
return (id(source),)
elif isinstance(source, dict):
data = source
else:
return (id(source),)
# Fully-deterministic signature across all fields, including extras
# allowed by SourceMessage's ConfigDict(extra="allow").
try:
return tuple(sorted((k, v) for k, v in data.items() if v is not None))
except TypeError:
# Unhashable value in an extra field (e.g. a dict / list) — fall
# back to identity so we never silently collapse distinct sources.
return (id(source),)


class MultiModalStructMemReader(SimpleStructMemReader):
"""Multimodal implementation of MemReader that inherits from
SimpleStructMemReader."""
Expand Down Expand Up @@ -159,14 +200,28 @@ def _create_chunk_item(chunk_idx: int, chunk):
"chunk_index": chunk_idx,
"chunk_total": chunk_total,
}
# Attach `sources` to the owning chunk only (issue #2453).
# If every chunk inherited the full parent `sources` list, the
# sliding-window aggregator (`_build_window_from_items`) would
# concatenate N copies of the same `SourceMessage` into one
# window — bloating storage, retrieval context, and provenance
# dedup. The parent sources belong to one semantic origin, so
# chunk 0 owns them; chunks 1..N-1 are discoverable via
# ingest_batch_id + chunk_index.
is_owning_chunk = chunk_idx == 0
if is_owning_chunk:
chunk_sources = item.metadata.sources or []
chunk_internal_info["source_chunk_index"] = 0
else:
chunk_sources = []
# Create a new memory item for each chunk, preserving original metadata
split_item = self._make_memory_item(
value=chunk_text,
info=chunk_info,
memory_type=item.metadata.memory_type,
tags=item.metadata.tags or [],
key=item.metadata.key,
sources=item.metadata.sources or [],
sources=chunk_sources,
background=item.metadata.background or "",
need_embed=False,
)
Expand Down Expand Up @@ -332,6 +387,14 @@ def _build_window_from_items(
# Collect all memory texts and sources
memory_texts = []
all_sources = []
# Dedup sources by a stable signature (issue #2453): a sliding
# window can legitimately contain multiple items that share the
# same origin SourceMessage (e.g. several chunks of one long
# input, or two parsers that both reference the same raw doc).
# Without dedup the window — and every downstream fine item —
# ends up carrying N identical copies, bloating storage and
# re-injecting the same original text N times at retrieval.
seen_source_sigs: set = set()
roles = set()
aggregated_file_ids: list[str] = []
ingest_batch_ids: set[str] = set()
Expand All @@ -346,6 +409,16 @@ def _build_window_from_items(
item_sources = [item_sources]

for source in item_sources:
sig = _source_signature(source)
if sig in seen_source_sigs:
# Still extract role even if we drop the duplicate,
# so memory_type classification stays correct.
if hasattr(source, "role") and source.role:
roles.add(source.role)
elif isinstance(source, dict) and source.get("role"):
roles.add(source.get("role"))
continue
seen_source_sigs.add(sig)
# Add source to all_sources
all_sources.append(source)

Expand Down
Loading
Loading