Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions src/memos/mem_reader/read_multi_modal/image_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,18 @@
logger = get_logger(__name__)


def is_sendable_image_url(url: str) -> bool:
"""Return whether a vision API can accept this image address.

Accepted forms are http(s) URLs and ``data:image/...;base64,...`` payloads.
Relative paths such as ``images/<hash>.jpg`` are rejected.
"""
candidate = url.strip().lower()
if candidate.startswith(("http://", "https://")):
return True
return candidate.startswith("data:image/") and "base64," in candidate


class ImageParser(BaseMessageParser):
"""Parser for image_url content parts."""

Expand Down Expand Up @@ -177,6 +189,13 @@ def parse_fine(
logger.warning("[ImageParser] No image URL found in message")
return []

if not is_sendable_image_url(url):
logger.warning(
"[ImageParser] Skip image URL that cannot be sent to the vision model: %s",
url[:200],
)
return []

# Create source for this image
source = self.create_source(message, info)

Expand Down
76 changes: 76 additions & 0 deletions tests/mem_reader/test_image_parser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
"""Tests for vision image URL validation in ImageParser.parse_fine."""

import json

from unittest.mock import MagicMock

import pytest

from memos.mem_reader.read_multi_modal.image_parser import ImageParser


def _parser() -> tuple[ImageParser, MagicMock]:
llm = MagicMock()
llm.generate.return_value = json.dumps(
{
"memory list": [
{
"key": "visible label",
"memory_type": "LongTermMemory",
"value": "A product label is visible.",
"tags": ["image"],
}
],
"summary": "A labeled product.",
}
)
embedder = MagicMock()
embedder.embed.return_value = [[0.1, 0.2]]
return ImageParser(embedder, llm), llm


def _image_message(url: str) -> dict:
return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}}


INFO = {"user_id": "user-1", "session_id": "session-1"}


@pytest.mark.parametrize(
"url",
[
"images/c502e59b213998cd695d7f57a6e33de73f5493f2d3fbab6c4e9635341aa7e7b7.jpg",
"image/photo.png",
"/tmp/photo.jpg",
"file:///tmp/photo.jpg",
"data:images/jpeg;base64,aaaa",
"data:image/jpeg,not-base64",
],
)
def test_parse_fine_skips_non_sendable_image_url(url: str) -> None:
parser, llm = _parser()

result = parser.parse_fine(_image_message(url), INFO)

assert result == []
llm.generate.assert_not_called()


@pytest.mark.parametrize(
"url",
[
"https://example.com/device.jpg",
"http://example.com/images/device.jpg",
"data:image/jpeg;base64,aaaa",
"DATA:IMAGE/PNG;base64,bbbb",
],
)
def test_parse_fine_sends_http_and_base64_image_urls(url: str) -> None:
parser, llm = _parser()

result = parser.parse_fine(_image_message(url), INFO)

assert len(result) == 1
llm.generate.assert_called_once()
sent_url = llm.generate.call_args.args[0][0]["content"][1]["image_url"]["url"]
assert sent_url == url
Loading