diff --git a/src/memos/chunkers/sentence_chunker.py b/src/memos/chunkers/sentence_chunker.py index 37e943151..d5485a7ec 100644 --- a/src/memos/chunkers/sentence_chunker.py +++ b/src/memos/chunkers/sentence_chunker.py @@ -41,15 +41,22 @@ def __init__(self, config: SentenceChunkerConfig): logger.info(f"Initialized SentenceChunker with config: {config}") - def chunk(self, text: str) -> list[str] | list[Chunk]: + def chunk(self, text: str) -> list[Chunk]: """Chunk the given text into smaller chunks based on sentences.""" protected_text, url_map = self.protect_urls(text) chonkie_chunks = self.chunker.chunk(protected_text) chunks = [] for c in chonkie_chunks: - chunk = Chunk(text=c.text, token_count=c.token_count, sentences=c.sentences) - chunk = self.restore_urls(chunk.text, url_map) + restored_text = self.restore_urls(c.text, url_map) + # Chonkie 1.3+ returns base chunks without sentence metadata. + restored_sentences = [ + self.restore_urls(sentence if isinstance(sentence, str) else sentence.text, url_map) + for sentence in getattr(c, "sentences", ()) + ] + chunk = Chunk( + text=restored_text, token_count=c.token_count, sentences=restored_sentences + ) chunks.append(chunk) logger.debug(f"Generated {len(chunks)} chunks from input text") diff --git a/tests/chunkers/test_sentence_chunker.py b/tests/chunkers/test_sentence_chunker.py index 7ff6b2ccd..41fe25632 100644 --- a/tests/chunkers/test_sentence_chunker.py +++ b/tests/chunkers/test_sentence_chunker.py @@ -2,6 +2,9 @@ from unittest.mock import MagicMock, patch +import pytest + +from memos.chunkers.base import Chunk from memos.chunkers.factory import ChunkerFactory from memos.configs.chunker import ChunkerConfigFactory @@ -48,16 +51,35 @@ def test_sentence_chunker(self): # Validate the properties of the first chunk mock_chunker.chunk.assert_called_once_with(text) - # Handle both return types: list[str] | list[Chunk] - if isinstance(chunks[0], str): - # If returns list[str], check the string value - self.assertEqual(chunks[0], "This is the first sentence.") - self.assertEqual(chunks[1], "This is the second sentence.") - else: - # If returns list[Chunk], check the Chunk properties - from memos.chunkers.base import Chunk - - self.assertIsInstance(chunks[0], Chunk) - self.assertEqual(chunks[0].text, "This is the first sentence.") - self.assertEqual(chunks[0].token_count, 6) - self.assertEqual(chunks[0].sentences, ["This is the first sentence."]) + self.assertIsInstance(chunks[0], Chunk) + self.assertEqual(chunks[0].text, "This is the first sentence.") + self.assertEqual(chunks[0].token_count, 6) + self.assertEqual(chunks[0].sentences, ["This is the first sentence."]) + + +def test_sentence_chunker_preserves_urls_with_real_chonkie(): + pytest.importorskip("chonkie") + config = ChunkerConfigFactory.model_validate( + { + "backend": "sentence", + "config": { + "tokenizer_or_token_counter": "character", + "chunk_size": 128, + "chunk_overlap": 0, + }, + } + ) + chunker = ChunkerFactory.from_config(config) + text = "See https://example.com/install for installation details." + protected_text, _ = chunker.protect_urls(text) + [raw_chunk] = chunker.chunker.chunk(protected_text) + + [chunk] = chunker.chunk(text) + + assert isinstance(chunk, Chunk) + assert chunk.text == text + assert chunk.token_count == raw_chunk.token_count + if hasattr(raw_chunk, "sentences"): + assert chunk.sentences == [text] + else: + assert chunk.sentences == []