From aba1391324f5ed1db2da8fe9b74e22be0002c0dc Mon Sep 17 00:00:00 2001 From: Daoji Wang <627665797@qq.com> Date: Thu, 8 Oct 2026 16:30:32 +0800 Subject: [PATCH] fix: skip vision calls for non-sendable image URLs Relative markdown paths such as images/.jpg were forwarded to the vision API and rejected as invalid base64. Co-authored-by: Cursor --- .../read_multi_modal/image_parser.py | 19 +++++ tests/mem_reader/test_image_parser.py | 76 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 tests/mem_reader/test_image_parser.py diff --git a/src/memos/mem_reader/read_multi_modal/image_parser.py b/src/memos/mem_reader/read_multi_modal/image_parser.py index ec19ba304..40561c430 100644 --- a/src/memos/mem_reader/read_multi_modal/image_parser.py +++ b/src/memos/mem_reader/read_multi_modal/image_parser.py @@ -27,6 +27,18 @@ logger = get_logger(__name__) +def is_sendable_image_url(url: str) -> bool: + """Return whether a vision API can accept this image address. + + Accepted forms are http(s) URLs and ``data:image/...;base64,...`` payloads. + Relative paths such as ``images/.jpg`` are rejected. + """ + candidate = url.strip().lower() + if candidate.startswith(("http://", "https://")): + return True + return candidate.startswith("data:image/") and "base64," in candidate + + class ImageParser(BaseMessageParser): """Parser for image_url content parts.""" @@ -177,6 +189,13 @@ def parse_fine( logger.warning("[ImageParser] No image URL found in message") return [] + if not is_sendable_image_url(url): + logger.warning( + "[ImageParser] Skip image URL that cannot be sent to the vision model: %s", + url[:200], + ) + return [] + # Create source for this image source = self.create_source(message, info) diff --git a/tests/mem_reader/test_image_parser.py b/tests/mem_reader/test_image_parser.py new file mode 100644 index 000000000..374ac17e9 --- /dev/null +++ b/tests/mem_reader/test_image_parser.py @@ -0,0 +1,76 @@ +"""Tests for vision image URL validation in ImageParser.parse_fine.""" + +import json + +from unittest.mock import MagicMock + +import pytest + +from memos.mem_reader.read_multi_modal.image_parser import ImageParser + + +def _parser() -> tuple[ImageParser, MagicMock]: + llm = MagicMock() + llm.generate.return_value = json.dumps( + { + "memory list": [ + { + "key": "visible label", + "memory_type": "LongTermMemory", + "value": "A product label is visible.", + "tags": ["image"], + } + ], + "summary": "A labeled product.", + } + ) + embedder = MagicMock() + embedder.embed.return_value = [[0.1, 0.2]] + return ImageParser(embedder, llm), llm + + +def _image_message(url: str) -> dict: + return {"type": "image_url", "image_url": {"url": url, "detail": "auto"}} + + +INFO = {"user_id": "user-1", "session_id": "session-1"} + + +@pytest.mark.parametrize( + "url", + [ + "images/c502e59b213998cd695d7f57a6e33de73f5493f2d3fbab6c4e9635341aa7e7b7.jpg", + "image/photo.png", + "/tmp/photo.jpg", + "file:///tmp/photo.jpg", + "data:images/jpeg;base64,aaaa", + "data:image/jpeg,not-base64", + ], +) +def test_parse_fine_skips_non_sendable_image_url(url: str) -> None: + parser, llm = _parser() + + result = parser.parse_fine(_image_message(url), INFO) + + assert result == [] + llm.generate.assert_not_called() + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/device.jpg", + "http://example.com/images/device.jpg", + "data:image/jpeg;base64,aaaa", + "DATA:IMAGE/PNG;base64,bbbb", + ], +) +def test_parse_fine_sends_http_and_base64_image_urls(url: str) -> None: + parser, llm = _parser() + + result = parser.parse_fine(_image_message(url), INFO) + + assert len(result) == 1 + llm.generate.assert_called_once() + sent_url = llm.generate.call_args.args[0][0]["content"][1]["image_url"]["url"] + assert sent_url == url