From fe4b76a4af52c5099f95b0dc8005462e55bb64bd Mon Sep 17 00:00:00 2001 From: simpleqt <89645338+simpleqt@users.noreply.github.com> Date: Tue, 29 Sep 2026 01:05:09 +0800 Subject: [PATCH] Preserve Chunk objects in SentenceChunker.chunk chunk() built a Chunk and then rebound it to the bare restored-URL string, so the method always returned list[str] while discarding token_count and sentences. Restore the URLs into the Chunk's text and return the object. Fixes #2433 --- src/memos/chunkers/sentence_chunker.py | 10 ++- .../test_sentence_chunker_chunk_objects.py | 79 +++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/chunkers/test_sentence_chunker_chunk_objects.py diff --git a/src/memos/chunkers/sentence_chunker.py b/src/memos/chunkers/sentence_chunker.py index 37e943151..54858f1cd 100644 --- a/src/memos/chunkers/sentence_chunker.py +++ b/src/memos/chunkers/sentence_chunker.py @@ -48,9 +48,13 @@ def chunk(self, text: str) -> list[str] | list[Chunk]: chunks = [] for c in chonkie_chunks: - chunk = Chunk(text=c.text, token_count=c.token_count, sentences=c.sentences) - chunk = self.restore_urls(chunk.text, url_map) - chunks.append(chunk) + chunks.append( + Chunk( + text=self.restore_urls(c.text, url_map), + token_count=c.token_count, + sentences=c.sentences, + ) + ) logger.debug(f"Generated {len(chunks)} chunks from input text") diff --git a/tests/chunkers/test_sentence_chunker_chunk_objects.py b/tests/chunkers/test_sentence_chunker_chunk_objects.py new file mode 100644 index 000000000..240fabcd8 --- /dev/null +++ b/tests/chunkers/test_sentence_chunker_chunk_objects.py @@ -0,0 +1,79 @@ +import logging +import sys +import types + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + + +SRC_DIR = Path(__file__).resolve().parents[2] / "src" / "memos" + + +def _install_memos_package_stub() -> None: + """Only fill in the pieces that are missing so the real package, when + importable, is never shadowed for the rest of the test session.""" + if "memos" not in sys.modules: + memos_pkg = types.ModuleType("memos") + memos_pkg.__path__ = [str(SRC_DIR)] + sys.modules["memos"] = memos_pkg + + if "memos.log" not in sys.modules: + log_stub = types.ModuleType("memos.log") + log_stub.get_logger = logging.getLogger + sys.modules["memos.log"] = log_stub + sys.modules["memos"].log = log_stub + + if "memos.chunkers" not in sys.modules: + chunkers_pkg = types.ModuleType("memos.chunkers") + chunkers_pkg.__path__ = [str(SRC_DIR / "chunkers")] + sys.modules["memos.chunkers"] = chunkers_pkg + sys.modules["memos"].chunkers = chunkers_pkg + + +@pytest.fixture +def sentence_chunker() -> Any: + _install_memos_package_stub() + + with patch("chonkie.SentenceChunker"): + from memos.chunkers.factory import ChunkerFactory + from memos.configs.chunker import ChunkerConfigFactory + + config = ChunkerConfigFactory.model_validate( + { + "backend": "sentence", + "config": { + "tokenizer_or_token_counter": "gpt2", + "chunk_size": 10, + "chunk_overlap": 2, + }, + } + ) + return ChunkerFactory.from_config(config) + + +def test_chunk_returns_chunk_objects_with_restored_urls(sentence_chunker): + from memos.chunkers.base import Chunk + + # chunk() protects URLs first, so the placeholder form is what the + # backend sees and what the returned Chunk must have restored + backend_chunks = [ + MagicMock( + text="Read __URL_0__ for details.", + token_count=6, + sentences=["Read __URL_0__ for details."], + ) + ] + sentence_chunker.chunker = MagicMock() + sentence_chunker.chunker.chunk.return_value = backend_chunks + + chunks = sentence_chunker.chunk("Read https://example.com/a-very-long-url for details.") + + assert len(chunks) == 1 + # the method must return real Chunk objects, not the bare restored string + assert isinstance(chunks[0], Chunk) + assert chunks[0].text == "Read https://example.com/a-very-long-url for details." + assert chunks[0].token_count == 6 + assert chunks[0].sentences == ["Read __URL_0__ for details."]