diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index c46331f1d..e38831df1 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -9,6 +9,7 @@ from memos import log from memos.configs.mem_reader import MultiModalStructMemReaderConfig from memos.context.context import ContextThreadPoolExecutor +from memos.exceptions import EmbedderError from memos.mem_reader.read_multi_modal import MultiModalParser, detect_lang from memos.mem_reader.read_multi_modal.base import _derive_key from memos.mem_reader.read_pref_memory.process_preference_memory import process_preference_fine @@ -89,28 +90,68 @@ def __init__(self, config: MultiModalStructMemReaderConfig): ) def _embed_memory_items(self, items: list[TextualMemoryItem]) -> None: - """Compute embeddings for a list of memory items in-place. - - Attempts a single batch call first; falls back to per-item calls if the - batch fails. Errors are logged but never raised so callers always - continue normally. - """ - valid = [w for w in items if w and w.memory] + """Assign vectors only after the whole batch has usable results.""" + valid = [item for item in items if item.memory] if not valid: return - texts = [w.memory for w in valid] + texts = [item.memory for item in valid] try: embeddings = self.embedder.embed(texts) - for w, emb in zip(valid, embeddings, strict=True): - w.metadata.embedding = emb - except Exception as e: - logger.error(f"[MultiModalStruct] Error batch computing embeddings: {e}") - logger.warning("[EMBED_FALLBACK] batch_size=%d", len(texts)) - for w in valid: + if len(embeddings) != len(valid) or any(not vector for vector in embeddings): + raise EmbedderError("Embedding batch returned missing vectors") + except Exception: + logger.warning("Embedding batch failed; retrying %d items individually", len(valid)) + embeddings = [] + for text in texts: try: - w.metadata.embedding = self.embedder.embed([w.memory])[0] - except Exception as e2: - logger.error(f"[MultiModalStruct] Error computing embedding for item: {e2}") + [vector] = self.embedder.embed([text]) + if not vector: + raise EmbedderError("Embedding response contains an empty vector") + except Exception: + raise EmbedderError("Could not embed a memory window") from None + embeddings.append(vector) + + for item, vector in zip(valid, embeddings, strict=True): + item.metadata.embedding = vector + + def _fits_embedding_window(self, text: str, max_tokens: int) -> bool: + config = getattr(self.embedder, "config", None) + limit = getattr(config, "max_tokens", None) + # BaseEmbedder enforces its configured limit with a character slice. + if type(limit) is int and limit > 0 and len(text) > limit: + return False + return self._count_tokens(text) <= max_tokens + + def _split_embedding_text(self, text: str, max_tokens: int) -> list[str]: + """Split oversized text without losing any suffix to provider truncation.""" + pieces = [] + while text and not self._fits_embedding_window(text, max_tokens): + low, high, best = 1, len(text) - 1, 0 + while low <= high: + middle = (low + high) // 2 + if self._fits_embedding_window(text[:middle], max_tokens): + best = middle + low = middle + 1 + else: + high = middle - 1 + if not best: + raise EmbedderError("Memory input cannot fit the embedding window") + # Prefer a nearby word or punctuation boundary when it fits. + start = best // 2 + boundary = max( + ( + start + match.end() + for match in re.finditer(r"[\s。!?;,.!?;]", text[start:best]) + ), + default=best, + ) + if self._fits_embedding_window(text[:boundary], max_tokens): + best = boundary + pieces.append(text[:best]) + text = text[best:] + if text: + pieces.append(text) + return pieces def _split_large_memory_item( self, item: TextualMemoryItem, max_tokens: int @@ -129,13 +170,23 @@ def _split_large_memory_item( if not item_text: return [item] - item_tokens = self._count_tokens(item_text) - if item_tokens <= max_tokens: + if self._fits_embedding_window(item_text, max_tokens): return [item] - # Use chunker to split the text try: - chunks = self.chunker.chunk(item_text) + try: + chunks = [ + chunk if isinstance(chunk, str) else chunk.text + for chunk in self.chunker.chunk(item_text) + ] + except Exception: + chunks = [] + # Keep configured boundaries when the chunker preserves the whole source. + if "".join(chunks) != item_text: + chunks = [item_text] + chunks = [ + piece for chunk in chunks for piece in self._split_embedding_text(chunk, max_tokens) + ] split_items = [] source_info = dict(item.metadata.info or {}) source_internal_info = dict(item.metadata.internal_info or {}) @@ -143,11 +194,7 @@ def _split_large_memory_item( chunk_total = len(chunks) def _create_chunk_item(chunk_idx: int, chunk): - # Different chunkers are not fully consistent: - # some return Chunk-like objects with `.text`, while others return raw strings. - chunk_text = chunk.text if hasattr(chunk, "text") else chunk - if not chunk_text or not chunk_text.strip(): - return None + chunk_text = chunk chunk_info = { "user_id": item.metadata.user_id, "session_id": item.metadata.session_id, @@ -184,12 +231,11 @@ def _create_chunk_item(chunk_idx: int, chunk): if split_item is not None: split_items.append(split_item) - return split_items if split_items else [item] - except Exception as e: - logger.warning( - f"[MultiModalStruct] Failed to split large memory item: {e}. Returning original item." - ) - return [item] + return split_items + except EmbedderError: + raise + except Exception: + raise EmbedderError("Could not split memory into embedding windows") from None def _concat_multi_modal_memories( self, all_memory_items: list[TextualMemoryItem], max_tokens=None, overlap=200 @@ -208,50 +254,19 @@ def _concat_multi_modal_memories( max_tokens = max_tokens or self.chat_window_max_tokens - # Split large memory items before processing - processed_items = [] - # control whether to parallel chunk large memory items - parallel_chunking = True - - if parallel_chunking: - # parallel chunk large memory items, but keep the original order - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - # Create a list to hold futures with their original index - futures = [] - for idx, item in enumerate(all_memory_items): - if (item.memory or "") and self._count_tokens(item.memory) > max_tokens: - future = executor.submit(self._split_large_memory_item, item, max_tokens) - futures.append( - (idx, future, True) - ) # True indicates this item needs splitting - else: - futures.append((idx, item, False)) # False indicates no splitting needed - - # Process results in original order - temp_results = [None] * len(all_memory_items) - for idx, future_or_item, needs_splitting in futures: - if needs_splitting: - # Wait for the future to complete and get the split items - split_items = future_or_item.result() - temp_results[idx] = split_items - else: - # No splitting needed, use the original item - temp_results[idx] = [future_or_item] - - # Flatten the results while preserving order - for items in temp_results: - processed_items.extend(items) - else: - # serial chunk large memory items - for item in all_memory_items: - item_text = item.memory or "" - item_tokens = self._count_tokens(item_text) - if item_tokens > max_tokens: - # Split the large item into multiple chunks - split_items = self._split_large_memory_item(item, max_tokens) - processed_items.extend(split_items) - else: - processed_items.append(item) + # Preserve input order and submit only oversized items to the pool. + with ContextThreadPoolExecutor(max_workers=4) as executor: + splits = [ + [item] + if self._fits_embedding_window(item.memory or "", max_tokens) + else executor.submit(self._split_large_memory_item, item, max_tokens) + for item in all_memory_items + ] + processed_items = [ + item + for split in splits + for item in (split if isinstance(split, list) else split.result()) + ] # If only one item after processing, compute embedding and return if len(processed_items) == 1: @@ -272,34 +287,22 @@ def _concat_multi_modal_memories( **(first_item.metadata.info or {}), } - for _idx, item in enumerate(processed_items): + for item in processed_items: item_text = item.memory or "" - # Ensure line ends with newline (same format as simple_struct) - line = item_text if item_text.endswith("\n") else f"{item_text}\n" - - # Check if adding this item would exceed max_tokens (same logic as _iter_chat_windows) - # Note: After splitting large items, each item should be <= max_tokens, - # but we still check to handle edge cases - if self._count_tokens(cur_text + line) > max_tokens and cur_text: - # Yield current window + if cur_text and not self._fits_embedding_window(cur_text + item_text, max_tokens): window = self._build_window_from_items(buf_items, info) if window: windows.append(window) - # Keep overlap: remove items until remaining tokens <= overlap - # (same logic as _iter_chat_windows) - while ( - buf_items - and self._count_tokens("".join([it.memory or "" for it in buf_items])) > overlap + while buf_items and ( + self._count_tokens(cur_text) > overlap + or not self._fits_embedding_window(cur_text + item_text, max_tokens) ): buf_items.pop(0) - # Recalculate cur_text from remaining items - cur_text = "".join([it.memory or "" for it in buf_items]) + cur_text = "".join(part.memory or "" for part in buf_items) - # Add item to current window buf_items.append(item) - # Recalculate cur_text from all items in buffer (same as _iter_chat_windows) - cur_text = "".join([it.memory or "" for it in buf_items]) + cur_text += item_text # Yield final window if any items remain if buf_items: @@ -1293,6 +1296,8 @@ def _read_memory( res_memory = future.result() if res_memory is not None: memory_list.append(res_memory) + except EmbedderError: + raise except Exception as e: logger.error(f"Task failed with exception: {e}") logger.error(traceback.format_exc()) diff --git a/tests/mem_reader/test_embed_budget_split.py b/tests/mem_reader/test_embed_budget_split.py new file mode 100644 index 000000000..0d67f8e54 --- /dev/null +++ b/tests/mem_reader/test_embed_budget_split.py @@ -0,0 +1,88 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from memos.embedders.base import BaseEmbedder +from memos.exceptions import EmbedderError +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.memories.textual.item import ( + SourceMessage, + TextualMemoryItem, + TreeNodeTextualMemoryMetadata, +) + + +class TruncatingEmbedder(BaseEmbedder): + def __init__(self, limit: int): + self.config = SimpleNamespace(max_tokens=limit) + self.submitted_texts = [] + + def embed(self, texts: list[str]) -> list[list[float]]: + texts = self._truncate_texts(texts) + self.submitted_texts.extend(texts) + return [[0.1] for _ in texts] + + +def make_reader(embedder): + reader = MultiModalStructMemReader.__new__(MultiModalStructMemReader) + reader.embedder = embedder + reader.chat_window_max_tokens = 100 + reader._count_tokens = lambda text: len(text.split()) + reader.chunker = Mock() + reader.chunker.chunk.side_effect = lambda text: [text] + return reader + + +def make_item(text: str) -> TextualMemoryItem: + return TextualMemoryItem( + memory=text, + metadata=TreeNodeTextualMemoryMetadata( + sources=[SourceMessage(content=text, role="user", message_id="sample-1")], + ), + ) + + +def test_windows_cover_the_tail_without_embedder_truncation(): + embedder = TruncatingEmbedder(limit=16) + reader = make_reader(embedder) + text = "alpha beta gamma delta epsilon omega" + + windows = reader._concat_multi_modal_memories([make_item(text)]) + + assert len(windows) > 1 + assert "".join(window.memory for window in windows) == text + assert embedder.submitted_texts == [window.memory for window in windows] + assert all(window.metadata.embedding for window in windows) + assert all(window.metadata.sources[0].message_id == "sample-1" for window in windows) + + +def test_failed_single_retry_cannot_leave_a_writeable_partial_batch(): + embedder = Mock() + embedder.embed.side_effect = [RuntimeError("batch"), [[0.1]], RuntimeError("single")] + reader = make_reader(embedder) + items = [make_item("first"), make_item("second")] + + with pytest.raises(EmbedderError): + reader._embed_memory_items(items) + + assert [item.metadata.embedding for item in items] == [None, None] + + +def test_empty_vector_cannot_pass_the_public_reader_boundary(monkeypatch): + embedder = Mock() + embedder.embed.return_value = [[]] + reader = make_reader(embedder) + monkeypatch.setattr( + reader, + "_process_multi_modal_data", + lambda scenes, info, **kwargs: reader._concat_multi_modal_memories([make_item("sample")]), + ) + + with pytest.raises(EmbedderError): + reader.get_memory( + [[{"role": "user", "content": "sample"}]], + "chat", + {"user_id": "sample-user", "session_id": "sample-session"}, + mode="fast", + ) diff --git a/tests/mem_reader/test_project_id_propagation.py b/tests/mem_reader/test_project_id_propagation.py index bf55aca46..aa9033dfe 100644 --- a/tests/mem_reader/test_project_id_propagation.py +++ b/tests/mem_reader/test_project_id_propagation.py @@ -260,8 +260,7 @@ def test_build_window_picks_first_nonempty(self): _assert_fields(self, result) def test_split_large_memory_item_assigns_shared_ingest_batch_id(self): - self.reader._count_tokens = MagicMock(return_value=999) - self.reader.chunker.chunk.return_value = ["chunk one", "chunk two"] + self.reader._count_tokens = len def fake_make_memory_item( *,