Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions src/memos/chunkers/sentence_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,22 @@ def __init__(self, config: SentenceChunkerConfig):

logger.info(f"Initialized SentenceChunker with config: {config}")

def chunk(self, text: str) -> list[str] | list[Chunk]:
def chunk(self, text: str) -> list[Chunk]:
"""Chunk the given text into smaller chunks based on sentences."""
protected_text, url_map = self.protect_urls(text)
chonkie_chunks = self.chunker.chunk(protected_text)

chunks = []
for c in chonkie_chunks:
chunk = Chunk(text=c.text, token_count=c.token_count, sentences=c.sentences)
chunk = self.restore_urls(chunk.text, url_map)
restored_text = self.restore_urls(c.text, url_map)
# Chonkie 1.3+ returns base chunks without sentence metadata.
restored_sentences = [
self.restore_urls(sentence if isinstance(sentence, str) else sentence.text, url_map)
for sentence in getattr(c, "sentences", ())
]
chunk = Chunk(
text=restored_text, token_count=c.token_count, sentences=restored_sentences
)
chunks.append(chunk)

logger.debug(f"Generated {len(chunks)} chunks from input text")
Expand Down
48 changes: 35 additions & 13 deletions tests/chunkers/test_sentence_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

from unittest.mock import MagicMock, patch

import pytest

from memos.chunkers.base import Chunk
from memos.chunkers.factory import ChunkerFactory
from memos.configs.chunker import ChunkerConfigFactory

Expand Down Expand Up @@ -48,16 +51,35 @@ def test_sentence_chunker(self):
# Validate the properties of the first chunk
mock_chunker.chunk.assert_called_once_with(text)

# Handle both return types: list[str] | list[Chunk]
if isinstance(chunks[0], str):
# If returns list[str], check the string value
self.assertEqual(chunks[0], "This is the first sentence.")
self.assertEqual(chunks[1], "This is the second sentence.")
else:
# If returns list[Chunk], check the Chunk properties
from memos.chunkers.base import Chunk

self.assertIsInstance(chunks[0], Chunk)
self.assertEqual(chunks[0].text, "This is the first sentence.")
self.assertEqual(chunks[0].token_count, 6)
self.assertEqual(chunks[0].sentences, ["This is the first sentence."])
self.assertIsInstance(chunks[0], Chunk)
self.assertEqual(chunks[0].text, "This is the first sentence.")
self.assertEqual(chunks[0].token_count, 6)
self.assertEqual(chunks[0].sentences, ["This is the first sentence."])


def test_sentence_chunker_preserves_urls_with_real_chonkie():
pytest.importorskip("chonkie")
config = ChunkerConfigFactory.model_validate(
{
"backend": "sentence",
"config": {
"tokenizer_or_token_counter": "character",
"chunk_size": 128,
"chunk_overlap": 0,
},
}
)
chunker = ChunkerFactory.from_config(config)
text = "See https://example.com/install for installation details."
protected_text, _ = chunker.protect_urls(text)
[raw_chunk] = chunker.chunker.chunk(protected_text)

[chunk] = chunker.chunk(text)

assert isinstance(chunk, Chunk)
assert chunk.text == text
assert chunk.token_count == raw_chunk.token_count
if hasattr(raw_chunk, "sentences"):
assert chunk.sentences == [text]
else:
assert chunk.sentences == []
Loading