diff --git a/src/memos/chunkers/charactertext_chunker.py b/src/memos/chunkers/charactertext_chunker.py index 25739d96f..02ca053ee 100644 --- a/src/memos/chunkers/charactertext_chunker.py +++ b/src/memos/chunkers/charactertext_chunker.py @@ -3,6 +3,7 @@ from memos.log import get_logger from .base import BaseChunker, Chunk +from .simple_chunker import SimpleTextSplitter logger = get_logger(__name__) @@ -27,9 +28,11 @@ def __init__( ) self.config = config + self.chunk_size = config.chunk_size if config else chunk_size + self.chunk_overlap = config.chunk_overlap if config else chunk_overlap self.chunker = RecursiveCharacterTextSplitter( - chunk_size=config.chunk_size if config else chunk_size, - chunk_overlap=config.chunk_overlap if config else chunk_overlap, + chunk_size=self.chunk_size, + chunk_overlap=self.chunk_overlap, length_function=len, separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""], ) @@ -37,6 +40,17 @@ def __init__( def chunk(self, text: str, **kwargs) -> list[str] | list[Chunk]: """Chunk the given text into smaller chunks based on sentences.""" protected_text, url_map = self.protect_urls(text) + if url_map: + # Issue #2510: RecursiveCharacterTextSplitter can cut through the + # __URL___ placeholders, so the URLs can no longer be restored + # and placeholder fragments leak into the chunks. Reuse + # SimpleTextSplitter for URL-containing inputs — it keeps + # placeholders indivisible (see #2228). + chunks = SimpleTextSplitter( + chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap + ).chunk(text, **kwargs) + logger.debug(f"Generated {len(chunks)} chunks from input text") + return chunks chunks = self.chunker.split_text(protected_text) chunks = [self.restore_urls(chunk, url_map) for chunk in chunks] logger.debug(f"Generated {len(chunks)} chunks from input text") diff --git a/tests/chunkers/test_charactertext_chunker.py b/tests/chunkers/test_charactertext_chunker.py new file mode 100644 index 000000000..734fad88c --- /dev/null +++ b/tests/chunkers/test_charactertext_chunker.py @@ -0,0 +1,91 @@ +"""Regression tests for `CharacterTextChunker` URL handling (issue #2510). + +`CharacterTextChunker` replaces URLs with `__URL___` placeholders before +splitting. `RecursiveCharacterTextSplitter` can cut through a placeholder +(e.g. ``__URL`` / ``_0__`` in different chunks), after which the URL can no +longer be restored and placeholder fragments leak into the output. + +The fix reuses `SimpleTextSplitter` — which keeps placeholders indivisible +(see #2228) — for inputs that contain URLs, keeping the recursive path for +plain text. +""" + +import pytest + +from memos.chunkers.base import BaseChunker +from memos.chunkers.charactertext_chunker import CharacterTextChunker + + +pytest.importorskip("langchain_text_splitters") + + +URL = "https://example.com/install" +TEXT_WITH_URL = f"安装文档:{URL} 请查看。" + + +def _assert_url_intact(chunks: list[str], url: str = URL) -> None: + """The URL must survive chunking whole; no placeholder fragments may leak.""" + assert any(url in chunk for chunk in chunks), ( + f"URL was fully lost after splitting; chunks={chunks}" + ) + for chunk in chunks: + assert BaseChunker._URL_PLACEHOLDER_PREFIX not in chunk, ( + f"unresolved URL placeholder leaked into chunk: {chunk!r}" + ) + + +def test_url_preserved_with_small_chunk_size(): + """Exact reproduction from issue #2510.""" + splitter = CharacterTextChunker(chunk_size=10, chunk_overlap=0) + + chunks = splitter.chunk(TEXT_WITH_URL) + + _assert_url_intact(chunks) + + +def test_url_preserved_with_overlap(): + """Overlap must not reintroduce split placeholders either.""" + splitter = CharacterTextChunker(chunk_size=12, chunk_overlap=4) + + chunks = splitter.chunk(TEXT_WITH_URL) + + _assert_url_intact(chunks) + + +def test_url_longer_than_chunk_size_preserved_whole(): + """A URL longer than chunk_size is kept whole (may oversize one chunk).""" + long_url = "https://example.com/" + "a" * 120 + splitter = CharacterTextChunker(chunk_size=20, chunk_overlap=0) + + chunks = splitter.chunk(f"see {long_url} now") + + _assert_url_intact(chunks, url=long_url) + + +def test_multiple_urls_all_preserved(): + """Every URL in the input must survive, not just the first.""" + urls = ["https://example.com/one", "https://example.org/two"] + splitter = CharacterTextChunker(chunk_size=10, chunk_overlap=0) + + chunks = splitter.chunk(f"first {urls[0]} then {urls[1]} end") + + for url in urls: + _assert_url_intact(chunks, url=url) + + +def test_text_without_url_keeps_recursive_splitting(): + """Inputs without URLs must keep the previous recursive-splitter behavior.""" + from langchain_text_splitters import RecursiveCharacterTextSplitter + + text = "第一段内容在这里。\n\n第二段内容在这里,稍微长一点。" + splitter = CharacterTextChunker(chunk_size=10, chunk_overlap=0) + + chunks = splitter.chunk(text) + expected = RecursiveCharacterTextSplitter( + chunk_size=10, + chunk_overlap=0, + length_function=len, + separators=["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " ", ""], + ).split_text(text) + + assert chunks == expected