Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
##############################################################################

name = "MemoryOS"
version = "2.0.27"
version = "2.0.29"
description = "Intelligence Begins with Memory"
license = {text = "Apache-2.0"}
readme = "README.md"
Expand Down
2 changes: 1 addition & 1 deletion src/memos/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = "2.0.27"
__version__ = "2.0.29"

from memos.configs.mem_cube import GeneralMemCubeConfig
from memos.configs.mem_os import MOSConfig
Expand Down
34 changes: 25 additions & 9 deletions src/memos/chunkers/base.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import re
import uuid

from abc import ABC, abstractmethod

Expand All @@ -17,6 +18,12 @@ def __init__(self, text: str, token_count: int, sentences: list[str]):
class BaseChunker(ABC):
"""Base class for all text chunkers."""

_URL_PATTERN = r'https?://[^\s<>"{}|\\^`\[\]]+'
_URL_PLACEHOLDER_PREFIX = "__URL_"
_URL_PLACEHOLDER_PATTERN = re.compile(
rf"{re.escape(_URL_PLACEHOLDER_PREFIX)}(?:[0-9a-f]+_)?\d+__"
)

@abstractmethod
def __init__(self, config: BaseChunkerConfig):
"""Initialize the chunker with the given configuration."""
Expand All @@ -25,6 +32,17 @@ def __init__(self, config: BaseChunkerConfig):
def chunk(self, text: str) -> list[Chunk]:
"""Chunk the given text into smaller chunks."""

@classmethod
def _new_url_placeholder_prefix(cls, text: str) -> str:
"""Return a placeholder namespace that cannot collide with the input."""
if cls._URL_PLACEHOLDER_PREFIX not in text:
return cls._URL_PLACEHOLDER_PREFIX

while True:
placeholder_prefix = f"{cls._URL_PLACEHOLDER_PREFIX}{uuid.uuid4().hex[:8]}_"
if placeholder_prefix not in text:
return placeholder_prefix

def protect_urls(self, text: str) -> tuple[str, dict[str, str]]:
"""
Protect URLs in text from being split during chunking.
Expand All @@ -35,16 +53,16 @@ def protect_urls(self, text: str) -> tuple[str, dict[str, str]]:
Returns:
tuple: (Text with URLs replaced by placeholders, URL mapping dictionary)
"""
url_pattern = r'https?://[^\s<>"{}|\\^`\[\]]+'
url_map = {}
url_map: dict[str, str] = {}
placeholder_prefix = self._new_url_placeholder_prefix(text)

def replace_url(match):
url = match.group(0)
placeholder = f"__URL_{len(url_map)}__"
placeholder = f"{placeholder_prefix}{len(url_map)}__"
url_map[placeholder] = url
return placeholder

protected_text = re.sub(url_pattern, replace_url, text)
protected_text = re.sub(self._URL_PATTERN, replace_url, text)
return protected_text, url_map

def restore_urls(self, text: str, url_map: dict[str, str]) -> str:
Expand All @@ -58,8 +76,6 @@ def restore_urls(self, text: str, url_map: dict[str, str]) -> str:
Returns:
str: Text with URLs restored
"""
restored_text = text
for placeholder, url in url_map.items():
restored_text = restored_text.replace(placeholder, url)

return restored_text
return self._URL_PLACEHOLDER_PATTERN.sub(
lambda match: url_map.get(match.group(0), match.group(0)), text
)
68 changes: 64 additions & 4 deletions src/memos/chunkers/simple_chunker.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,63 @@
class SimpleTextSplitter:
from .base import BaseChunker


class SimpleTextSplitter(BaseChunker):
"""Simple text splitter wrapper."""

def __init__(self, chunk_size: int, chunk_overlap: int):
if chunk_size <= 0:
raise ValueError("chunk_size must be greater than 0")
if chunk_overlap < 0:
raise ValueError("chunk_overlap must be non-negative")
if chunk_overlap >= chunk_size:
raise ValueError("chunk_overlap must be smaller than chunk_size")

self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap

def chunk(self, text: str, **kwargs) -> list[str]:
return self._simple_split_text(text, self.chunk_size, self.chunk_overlap)

@classmethod
def _placeholder_spans(
cls, protected_text: str, url_map: dict[str, str]
) -> list[tuple[int, int]]:
"""Return the protected-text ranges occupied by URL placeholders."""
return [
(match.start(), match.end())
for match in cls._URL_PLACEHOLDER_PATTERN.finditer(protected_text)
if match.group(0) in url_map
]

@staticmethod
def _align_end_to_placeholder(
end: int,
start: int,
chunk_overlap: int,
placeholder_spans: list[tuple[int, int]],
) -> int:
"""Move a chunk end away from the middle of a URL placeholder."""
for placeholder_start, placeholder_end in placeholder_spans:
if placeholder_start < end < placeholder_end:
if placeholder_start - start > chunk_overlap:
return placeholder_start
return placeholder_end
return end

@staticmethod
def _align_start_to_placeholder(
next_start: int,
previous_start: int,
placeholder_spans: list[tuple[int, int]],
) -> int:
"""Keep overlap starts from landing in the middle of a URL placeholder."""
for placeholder_start, placeholder_end in placeholder_spans:
if placeholder_start < next_start < placeholder_end:
if placeholder_start > previous_start:
return placeholder_start
return placeholder_end
return next_start

def _simple_split_text(self, text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
"""
Simple text splitter as fallback when langchain is not available.
Expand All @@ -29,6 +79,7 @@ def _simple_split_text(self, text: str, chunk_size: int, chunk_overlap: int) ->
chunks = []
start = 0
text_len = len(protected_text)
placeholder_spans = self._placeholder_spans(protected_text, url_map)

while start < text_len:
# Calculate end position
Expand All @@ -39,15 +90,24 @@ def _simple_split_text(self, text: str, chunk_size: int, chunk_overlap: int) ->
# Try to break at newline, sentence end, or space
for separator in ["\n\n", "\n", "。", "!", "?", ". ", "! ", "? ", " "]:
last_sep = protected_text.rfind(separator, start, end)
if last_sep != -1:
end = last_sep + len(separator)
if last_sep == -1:
continue
split_end = last_sep + len(separator)
if split_end - start > chunk_overlap:
end = split_end
break

end = self._align_end_to_placeholder(end, start, chunk_overlap, placeholder_spans)

chunk = protected_text[start:end].strip()
if chunk:
chunks.append(chunk)

if end >= text_len:
break

# Move start position with overlap
start = max(start + 1, end - chunk_overlap)
next_start = max(start + 1, end - chunk_overlap)
start = self._align_start_to_placeholder(next_start, start, placeholder_spans)

return [self.restore_urls(chunk, url_map) for chunk in chunks]
20 changes: 16 additions & 4 deletions src/memos/embedders/cache.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,23 @@
from __future__ import annotations

import os
import threading

from collections import Counter
from concurrent.futures import Future
from typing import Any

from cachetools import LRUCache, TTLCache
from typing import TYPE_CHECKING, Any

from memos.context.context import get_current_trace_id
from memos.dependency import require_python_package
from memos.embedders.base import BaseEmbedder
from memos.exceptions import EmbedderError
from memos.log import get_logger


if TYPE_CHECKING:
from cachetools import LRUCache, TTLCache


logger = get_logger(__name__)


Expand Down Expand Up @@ -68,9 +73,16 @@ def _env_int(name: str, default: int, minimum: int = 1) -> int:
class CachingEmbedder(BaseEmbedder):
"""Add exact caches and singleflight coordination to an embedder."""

@require_python_package(
import_name="cachetools",
install_command="pip install 'cachetools>=6.0.0'",
)
def __init__(self, backend: BaseEmbedder):
from cachetools import LRUCache, TTLCache

self._backend = backend
self.config = backend.config
self._lru_cache_cls: type[LRUCache] = LRUCache
self._lock = threading.RLock()
self._inflight: dict[str, Future[CachedVector]] = {}
self._stats: Counter[str] = Counter()
Expand Down Expand Up @@ -196,7 +208,7 @@ def _get_request_cache(self, request_id: str | None) -> LRUCache[str, CachedVect
return None
request_cache = self._request_caches.get(request_id)
if request_cache is None:
request_cache = LRUCache(maxsize=self._request_cache_text_limit)
request_cache = self._lru_cache_cls(maxsize=self._request_cache_text_limit)
self._request_caches[request_id] = request_cache
return request_cache

Expand Down
Loading
Loading