From 0b84090456d4be8223ae48d170883e5c7c7ce6c4 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:38:04 +0800 Subject: [PATCH] fix: preserve video input through multimodal ingestion --- src/memos/mem_reader/multi_modal_struct.py | 3 + .../read_multi_modal/user_parser.py | 12 +++ .../openai_chat_completion_types/__init__.py | 1 + .../chat_completion_content_part_param.py | 2 + ...hat_completion_content_part_video_param.py | 20 +++++ tests/mem_reader/test_video_input.py | 78 +++++++++++++++++++ 6 files changed, 116 insertions(+) create mode 100644 src/memos/types/openai_chat_completion_types/chat_completion_content_part_video_param.py create mode 100644 tests/mem_reader/test_video_input.py diff --git a/src/memos/mem_reader/multi_modal_struct.py b/src/memos/mem_reader/multi_modal_struct.py index c46331f1d..cb987a3ae 100644 --- a/src/memos/mem_reader/multi_modal_struct.py +++ b/src/memos/mem_reader/multi_modal_struct.py @@ -1224,6 +1224,9 @@ def _expand_multimodal_messages(messages: list) -> list: elif part_type in ("file", "image", "image_url"): # Extract as a standalone message for its specialised parser expanded.append(part) + elif part_type == "video_url": + # Keep the chat envelope so the user parser retains video provenance. + expanded.append({**msg, "content": [part]}) else: text_parts.append(f"[{part_type}]") diff --git a/src/memos/mem_reader/read_multi_modal/user_parser.py b/src/memos/mem_reader/read_multi_modal/user_parser.py index a5a248e9d..b4cbfeb59 100644 --- a/src/memos/mem_reader/read_multi_modal/user_parser.py +++ b/src/memos/mem_reader/read_multi_modal/user_parser.py @@ -118,6 +118,18 @@ def create_source( ) source.lang = overall_lang sources.append(source) + elif part_type == "video_url": + video_info = part.get("video_url", {}) + source = SourceMessage( + type="video", + role=role, + chat_time=chat_time, + message_id=message_id, + content=video_info.get("url"), + video_info=video_info, + ) + source.lang = overall_lang + sources.append(source) else: # input_audio, etc. source = SourceMessage( diff --git a/src/memos/types/openai_chat_completion_types/__init__.py b/src/memos/types/openai_chat_completion_types/__init__.py index 025e75360..488d7c02e 100644 --- a/src/memos/types/openai_chat_completion_types/__init__.py +++ b/src/memos/types/openai_chat_completion_types/__init__.py @@ -6,6 +6,7 @@ from .chat_completion_content_part_param import * from .chat_completion_content_part_refusal_param import * from .chat_completion_content_part_text_param import * +from .chat_completion_content_part_video_param import * from .chat_completion_message_custom_tool_call_param import * from .chat_completion_message_function_tool_call_param import * from .chat_completion_message_param import * diff --git a/src/memos/types/openai_chat_completion_types/chat_completion_content_part_param.py b/src/memos/types/openai_chat_completion_types/chat_completion_content_part_param.py index 99b232943..f9f0d6f39 100644 --- a/src/memos/types/openai_chat_completion_types/chat_completion_content_part_param.py +++ b/src/memos/types/openai_chat_completion_types/chat_completion_content_part_param.py @@ -7,6 +7,7 @@ from .chat_completion_content_part_image_param import ChatCompletionContentPartImageParam from .chat_completion_content_part_input_audio_param import ChatCompletionContentPartInputAudioParam from .chat_completion_content_part_text_param import ChatCompletionContentPartTextParam +from .chat_completion_content_part_video_param import ChatCompletionContentPartVideoParam __all__ = ["ChatCompletionContentPartParam", "File", "FileFile"] @@ -39,5 +40,6 @@ class File(TypedDict, total=False): ChatCompletionContentPartTextParam | ChatCompletionContentPartImageParam | ChatCompletionContentPartInputAudioParam + | ChatCompletionContentPartVideoParam | File ) diff --git a/src/memos/types/openai_chat_completion_types/chat_completion_content_part_video_param.py b/src/memos/types/openai_chat_completion_types/chat_completion_content_part_video_param.py new file mode 100644 index 000000000..56d62a9d0 --- /dev/null +++ b/src/memos/types/openai_chat_completion_types/chat_completion_content_part_video_param.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Literal + +from typing_extensions import Required, TypedDict + + +__all__ = ["ChatCompletionContentPartVideoParam", "VideoURL"] + + +class VideoURL(TypedDict, total=False): + url: Required[str] + """A video URL or a data URL containing the encoded video.""" + + +class ChatCompletionContentPartVideoParam(TypedDict, total=False): + video_url: Required[VideoURL] + + type: Required[Literal["video_url"]] + """The type of the content part.""" diff --git a/tests/mem_reader/test_video_input.py b/tests/mem_reader/test_video_input.py new file mode 100644 index 000000000..cd590b16a --- /dev/null +++ b/tests/mem_reader/test_video_input.py @@ -0,0 +1,78 @@ +"""Regression tests for video content validation and memory provenance.""" + +from copy import deepcopy +from unittest.mock import MagicMock + +import pytest + +from pydantic import TypeAdapter, ValidationError + +from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader +from memos.mem_reader.read_multi_modal.user_parser import UserParser +from memos.memories.textual.item import SourceMessage +from memos.types.openai_chat_completion_types import ChatCompletionUserMessageParam + + +@pytest.mark.parametrize("url", ["https://example.com/clip.mp4", "data:video/mp4;base64,AAAA"]) +def test_video_content_survives_chat_validation(url): + message = { + "role": "user", + "content": [{"type": "video_url", "video_url": {"url": url}}], + } + + assert TypeAdapter(ChatCompletionUserMessageParam).validate_python(message) == message + + +def test_video_content_requires_a_url(): + message = {"role": "user", "content": [{"type": "video_url", "video_url": {}}]} + + with pytest.raises(ValidationError): + TypeAdapter(ChatCompletionUserMessageParam).validate_python(message) + + +def test_video_source_survives_message_expansion_and_serialization(): + video = {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}} + message = { + "role": "user", + "content": [{"type": "text", "text": "Remember this clip"}, video], + "chat_time": "2026-10-01T00:00:00Z", + "message_id": "video-message", + } + original = deepcopy(message) + expanded = MultiModalStructMemReader._expand_multimodal_messages([message]) + video_messages = [item for item in expanded if isinstance(item.get("content"), list)] + + assert len(video_messages) == 1 + assert video_messages[0] == {**message, "content": [video]} + assert message == original + assert any(item.get("content") == "Remember this clip" for item in expanded) + + parser = UserParser(embedder=MagicMock()) + memories = parser.parse_fast(video_messages[0], {}, need_emb=False) + assert len(memories) == 1 + source = SourceMessage.model_validate(memories[0].metadata.sources[0].model_dump()) + assert source.type == "video" + assert source.video_info == video["video_url"] + assert source.content == video["video_url"]["url"] + assert source.message_id == message["message_id"] + assert source.chat_time == message["chat_time"] + assert source.role == "user" + + +def test_video_only_message_is_not_replaced_by_a_placeholder(): + message = { + "role": "user", + "content": [{"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}], + } + + assert MultiModalStructMemReader._expand_multimodal_messages([message]) == [message] + + +def test_existing_text_and_image_expansion_is_unchanged(): + image = {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}} + message = {"role": "user", "content": [{"type": "text", "text": "A picture"}, image]} + + assert MultiModalStructMemReader._expand_multimodal_messages([message]) == [ + image, + {"role": "user", "content": "A picture"}, + ]