From 0d10fbfe8cd4a0df4fc68372ac840973a67fd789 Mon Sep 17 00:00:00 2001 From: AutoDev Agent Date: Tue, 29 Sep 2026 01:15:53 +0800 Subject: [PATCH] test: add URL restoration verification for sentences field - Add test_sentence_chunker_preserves_urls_in_sentences to verify URLs are restored in both text and sentences fields - Fix SentenceChunker.chunk to restore URLs in sentences list, not just text - Addresses OCR finding: metadata must survive for sentences --- src/memos/chunkers/sentence_chunker.py | 5 ++- tests/chunkers/test_sentence_chunker.py | 59 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/memos/chunkers/sentence_chunker.py b/src/memos/chunkers/sentence_chunker.py index 37e943151..48a45da75 100644 --- a/src/memos/chunkers/sentence_chunker.py +++ b/src/memos/chunkers/sentence_chunker.py @@ -48,8 +48,9 @@ def chunk(self, text: str) -> list[str] | list[Chunk]: chunks = [] for c in chonkie_chunks: - chunk = Chunk(text=c.text, token_count=c.token_count, sentences=c.sentences) - chunk = self.restore_urls(chunk.text, url_map) + restored_text = self.restore_urls(c.text, url_map) + restored_sentences = [self.restore_urls(s, url_map) for s in c.sentences] + chunk = Chunk(text=restored_text, token_count=c.token_count, sentences=restored_sentences) chunks.append(chunk) logger.debug(f"Generated {len(chunks)} chunks from input text") diff --git a/tests/chunkers/test_sentence_chunker.py b/tests/chunkers/test_sentence_chunker.py index 7ff6b2ccd..1feedfce1 100644 --- a/tests/chunkers/test_sentence_chunker.py +++ b/tests/chunkers/test_sentence_chunker.py @@ -61,3 +61,62 @@ def test_sentence_chunker(self): self.assertEqual(chunks[0].text, "This is the first sentence.") self.assertEqual(chunks[0].token_count, 6) self.assertEqual(chunks[0].sentences, ["This is the first sentence."]) + + def test_sentence_chunker_preserves_urls_in_sentences(self): + """Test that URLs are restored in both text and sentences fields.""" + with patch("chonkie.SentenceChunker") as mock_chunker_cls: + # Set up the mock + mock_chunker = MagicMock() + url = "https://example.com/test" + protected_text = f"Check out __URL_0__ for details." + + # Mock returns chunk with protected URL placeholder in sentences + mock_chunk = MagicMock( + text=protected_text, + token_count=6, + sentences=[protected_text], + ) + mock_chunker.chunk.return_value = [mock_chunk] + mock_chunker_cls.return_value = mock_chunker + + # Create chunker + config = ChunkerConfigFactory.model_validate( + { + "backend": "sentence", + "config": { + "tokenizer_or_token_counter": "gpt2", + "chunk_size": 10, + "chunk_overlap": 2, + }, + } + ) + chunker = ChunkerFactory.from_config(config) + + # Test with URL text + text = f"Check out {url} for details." + chunks = chunker.chunk(text) + + # Verify chunk count + self.assertEqual(len(chunks), 1) + + # Handle both return types: list[str] | list[Chunk] + if isinstance(chunks[0], str): + # If returns list[str], URL should be restored in the string + self.assertIn(url, chunks[0]) + self.assertNotIn("__URL_", chunks[0]) + else: + # If returns list[Chunk], metadata must survive: + # - URLs must be restored in text field + # - URLs must be restored in sentences field (not just placeholder) + # - token_count and sentences must be preserved + from memos.chunkers.base import Chunk + + self.assertIsInstance(chunks[0], Chunk) + self.assertIn(url, chunks[0].text) + self.assertNotIn("__URL_", chunks[0].text) + + # The critical assertions from OCR finding: + # sentences metadata must survive and URLs must be restored there too + self.assertEqual(len(chunks[0].sentences), 1) + self.assertIn(url, chunks[0].sentences[0]) + self.assertNotIn("__URL_", chunks[0].sentences[0])