From 0d10fbfe8cd4a0df4fc68372ac840973a67fd789 Mon Sep 17 00:00:00 2001 From: AutoDev Agent Date: Tue, 29 Sep 2026 01:15:53 +0800 Subject: [PATCH 1/2] test: add URL restoration verification for sentences field - Add test_sentence_chunker_preserves_urls_in_sentences to verify URLs are restored in both text and sentences fields - Fix SentenceChunker.chunk to restore URLs in sentences list, not just text - Addresses OCR finding: metadata must survive for sentences --- src/memos/chunkers/sentence_chunker.py | 5 ++- tests/chunkers/test_sentence_chunker.py | 59 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/memos/chunkers/sentence_chunker.py b/src/memos/chunkers/sentence_chunker.py index 37e943151..48a45da75 100644 --- a/src/memos/chunkers/sentence_chunker.py +++ b/src/memos/chunkers/sentence_chunker.py @@ -48,8 +48,9 @@ def chunk(self, text: str) -> list[str] | list[Chunk]: chunks = [] for c in chonkie_chunks: - chunk = Chunk(text=c.text, token_count=c.token_count, sentences=c.sentences) - chunk = self.restore_urls(chunk.text, url_map) + restored_text = self.restore_urls(c.text, url_map) + restored_sentences = [self.restore_urls(s, url_map) for s in c.sentences] + chunk = Chunk(text=restored_text, token_count=c.token_count, sentences=restored_sentences) chunks.append(chunk) logger.debug(f"Generated {len(chunks)} chunks from input text") diff --git a/tests/chunkers/test_sentence_chunker.py b/tests/chunkers/test_sentence_chunker.py index 7ff6b2ccd..1feedfce1 100644 --- a/tests/chunkers/test_sentence_chunker.py +++ b/tests/chunkers/test_sentence_chunker.py @@ -61,3 +61,62 @@ def test_sentence_chunker(self): self.assertEqual(chunks[0].text, "This is the first sentence.") self.assertEqual(chunks[0].token_count, 6) self.assertEqual(chunks[0].sentences, ["This is the first sentence."]) + + def test_sentence_chunker_preserves_urls_in_sentences(self): + """Test that URLs are restored in both text and sentences fields.""" + with patch("chonkie.SentenceChunker") as mock_chunker_cls: + # Set up the mock + mock_chunker = MagicMock() + url = "https://example.com/test" + protected_text = f"Check out __URL_0__ for details." + + # Mock returns chunk with protected URL placeholder in sentences + mock_chunk = MagicMock( + text=protected_text, + token_count=6, + sentences=[protected_text], + ) + mock_chunker.chunk.return_value = [mock_chunk] + mock_chunker_cls.return_value = mock_chunker + + # Create chunker + config = ChunkerConfigFactory.model_validate( + { + "backend": "sentence", + "config": { + "tokenizer_or_token_counter": "gpt2", + "chunk_size": 10, + "chunk_overlap": 2, + }, + } + ) + chunker = ChunkerFactory.from_config(config) + + # Test with URL text + text = f"Check out {url} for details." + chunks = chunker.chunk(text) + + # Verify chunk count + self.assertEqual(len(chunks), 1) + + # Handle both return types: list[str] | list[Chunk] + if isinstance(chunks[0], str): + # If returns list[str], URL should be restored in the string + self.assertIn(url, chunks[0]) + self.assertNotIn("__URL_", chunks[0]) + else: + # If returns list[Chunk], metadata must survive: + # - URLs must be restored in text field + # - URLs must be restored in sentences field (not just placeholder) + # - token_count and sentences must be preserved + from memos.chunkers.base import Chunk + + self.assertIsInstance(chunks[0], Chunk) + self.assertIn(url, chunks[0].text) + self.assertNotIn("__URL_", chunks[0].text) + + # The critical assertions from OCR finding: + # sentences metadata must survive and URLs must be restored there too + self.assertEqual(len(chunks[0].sentences), 1) + self.assertIn(url, chunks[0].sentences[0]) + self.assertNotIn("__URL_", chunks[0].sentences[0]) From 3a9e4944946242ff6993d685b9a07a5efa9acd6f Mon Sep 17 00:00:00 2001 From: Wenqiang Wei <46308778+endxxxx@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:36:29 +0800 Subject: [PATCH 2/2] fix: normalize Chonkie sentence chunk metadata --- src/memos/chunkers/sentence_chunker.py | 12 ++- tests/chunkers/test_sentence_chunker.py | 107 ++++++++---------------- 2 files changed, 44 insertions(+), 75 deletions(-) diff --git a/src/memos/chunkers/sentence_chunker.py b/src/memos/chunkers/sentence_chunker.py index 48a45da75..d5485a7ec 100644 --- a/src/memos/chunkers/sentence_chunker.py +++ b/src/memos/chunkers/sentence_chunker.py @@ -41,7 +41,7 @@ def __init__(self, config: SentenceChunkerConfig): logger.info(f"Initialized SentenceChunker with config: {config}") - def chunk(self, text: str) -> list[str] | list[Chunk]: + def chunk(self, text: str) -> list[Chunk]: """Chunk the given text into smaller chunks based on sentences.""" protected_text, url_map = self.protect_urls(text) chonkie_chunks = self.chunker.chunk(protected_text) @@ -49,8 +49,14 @@ def chunk(self, text: str) -> list[str] | list[Chunk]: chunks = [] for c in chonkie_chunks: restored_text = self.restore_urls(c.text, url_map) - restored_sentences = [self.restore_urls(s, url_map) for s in c.sentences] - chunk = Chunk(text=restored_text, token_count=c.token_count, sentences=restored_sentences) + # Chonkie 1.3+ returns base chunks without sentence metadata. + restored_sentences = [ + self.restore_urls(sentence if isinstance(sentence, str) else sentence.text, url_map) + for sentence in getattr(c, "sentences", ()) + ] + chunk = Chunk( + text=restored_text, token_count=c.token_count, sentences=restored_sentences + ) chunks.append(chunk) logger.debug(f"Generated {len(chunks)} chunks from input text") diff --git a/tests/chunkers/test_sentence_chunker.py b/tests/chunkers/test_sentence_chunker.py index 1feedfce1..41fe25632 100644 --- a/tests/chunkers/test_sentence_chunker.py +++ b/tests/chunkers/test_sentence_chunker.py @@ -2,6 +2,9 @@ from unittest.mock import MagicMock, patch +import pytest + +from memos.chunkers.base import Chunk from memos.chunkers.factory import ChunkerFactory from memos.configs.chunker import ChunkerConfigFactory @@ -48,75 +51,35 @@ def test_sentence_chunker(self): # Validate the properties of the first chunk mock_chunker.chunk.assert_called_once_with(text) - # Handle both return types: list[str] | list[Chunk] - if isinstance(chunks[0], str): - # If returns list[str], check the string value - self.assertEqual(chunks[0], "This is the first sentence.") - self.assertEqual(chunks[1], "This is the second sentence.") - else: - # If returns list[Chunk], check the Chunk properties - from memos.chunkers.base import Chunk - - self.assertIsInstance(chunks[0], Chunk) - self.assertEqual(chunks[0].text, "This is the first sentence.") - self.assertEqual(chunks[0].token_count, 6) - self.assertEqual(chunks[0].sentences, ["This is the first sentence."]) - - def test_sentence_chunker_preserves_urls_in_sentences(self): - """Test that URLs are restored in both text and sentences fields.""" - with patch("chonkie.SentenceChunker") as mock_chunker_cls: - # Set up the mock - mock_chunker = MagicMock() - url = "https://example.com/test" - protected_text = f"Check out __URL_0__ for details." - - # Mock returns chunk with protected URL placeholder in sentences - mock_chunk = MagicMock( - text=protected_text, - token_count=6, - sentences=[protected_text], - ) - mock_chunker.chunk.return_value = [mock_chunk] - mock_chunker_cls.return_value = mock_chunker - - # Create chunker - config = ChunkerConfigFactory.model_validate( - { - "backend": "sentence", - "config": { - "tokenizer_or_token_counter": "gpt2", - "chunk_size": 10, - "chunk_overlap": 2, - }, - } - ) - chunker = ChunkerFactory.from_config(config) - - # Test with URL text - text = f"Check out {url} for details." - chunks = chunker.chunk(text) - - # Verify chunk count - self.assertEqual(len(chunks), 1) - - # Handle both return types: list[str] | list[Chunk] - if isinstance(chunks[0], str): - # If returns list[str], URL should be restored in the string - self.assertIn(url, chunks[0]) - self.assertNotIn("__URL_", chunks[0]) - else: - # If returns list[Chunk], metadata must survive: - # - URLs must be restored in text field - # - URLs must be restored in sentences field (not just placeholder) - # - token_count and sentences must be preserved - from memos.chunkers.base import Chunk - - self.assertIsInstance(chunks[0], Chunk) - self.assertIn(url, chunks[0].text) - self.assertNotIn("__URL_", chunks[0].text) - - # The critical assertions from OCR finding: - # sentences metadata must survive and URLs must be restored there too - self.assertEqual(len(chunks[0].sentences), 1) - self.assertIn(url, chunks[0].sentences[0]) - self.assertNotIn("__URL_", chunks[0].sentences[0]) + self.assertIsInstance(chunks[0], Chunk) + self.assertEqual(chunks[0].text, "This is the first sentence.") + self.assertEqual(chunks[0].token_count, 6) + self.assertEqual(chunks[0].sentences, ["This is the first sentence."]) + + +def test_sentence_chunker_preserves_urls_with_real_chonkie(): + pytest.importorskip("chonkie") + config = ChunkerConfigFactory.model_validate( + { + "backend": "sentence", + "config": { + "tokenizer_or_token_counter": "character", + "chunk_size": 128, + "chunk_overlap": 0, + }, + } + ) + chunker = ChunkerFactory.from_config(config) + text = "See https://example.com/install for installation details." + protected_text, _ = chunker.protect_urls(text) + [raw_chunk] = chunker.chunker.chunk(protected_text) + + [chunk] = chunker.chunk(text) + + assert isinstance(chunk, Chunk) + assert chunk.text == text + assert chunk.token_count == raw_chunk.token_count + if hasattr(raw_chunk, "sentences"): + assert chunk.sentences == [text] + else: + assert chunk.sentences == []