Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/memos/chunkers/sentence_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@ def chunk(self, text: str) -> list[str] | list[Chunk]:

chunks = []
for c in chonkie_chunks:
chunk = Chunk(text=c.text, token_count=c.token_count, sentences=c.sentences)
chunk = self.restore_urls(chunk.text, url_map)
restored_text = self.restore_urls(c.text, url_map)
restored_sentences = [self.restore_urls(s, url_map) for s in c.sentences]
chunk = Chunk(text=restored_text, token_count=c.token_count, sentences=restored_sentences)
chunks.append(chunk)

logger.debug(f"Generated {len(chunks)} chunks from input text")
Expand Down
59 changes: 59 additions & 0 deletions tests/chunkers/test_sentence_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,62 @@ def test_sentence_chunker(self):
self.assertEqual(chunks[0].text, "This is the first sentence.")
self.assertEqual(chunks[0].token_count, 6)
self.assertEqual(chunks[0].sentences, ["This is the first sentence."])

def test_sentence_chunker_preserves_urls_in_sentences(self):
"""Test that URLs are restored in both text and sentences fields."""
with patch("chonkie.SentenceChunker") as mock_chunker_cls:
# Set up the mock
mock_chunker = MagicMock()
url = "https://example.com/test"
protected_text = f"Check out __URL_0__ for details."

# Mock returns chunk with protected URL placeholder in sentences
mock_chunk = MagicMock(
text=protected_text,
token_count=6,
sentences=[protected_text],
)
mock_chunker.chunk.return_value = [mock_chunk]
mock_chunker_cls.return_value = mock_chunker

# Create chunker
config = ChunkerConfigFactory.model_validate(
{
"backend": "sentence",
"config": {
"tokenizer_or_token_counter": "gpt2",
"chunk_size": 10,
"chunk_overlap": 2,
},
}
)
chunker = ChunkerFactory.from_config(config)

# Test with URL text
text = f"Check out {url} for details."
chunks = chunker.chunk(text)

# Verify chunk count
self.assertEqual(len(chunks), 1)

# Handle both return types: list[str] | list[Chunk]
if isinstance(chunks[0], str):
# If returns list[str], URL should be restored in the string
self.assertIn(url, chunks[0])
self.assertNotIn("__URL_", chunks[0])
else:
# If returns list[Chunk], metadata must survive:
# - URLs must be restored in text field
# - URLs must be restored in sentences field (not just placeholder)
# - token_count and sentences must be preserved
from memos.chunkers.base import Chunk

self.assertIsInstance(chunks[0], Chunk)
self.assertIn(url, chunks[0].text)
self.assertNotIn("__URL_", chunks[0].text)

# The critical assertions from OCR finding:
# sentences metadata must survive and URLs must be restored there too
self.assertEqual(len(chunks[0].sentences), 1)
self.assertIn(url, chunks[0].sentences[0])
self.assertNotIn("__URL_", chunks[0].sentences[0])
Loading