Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/memos/mem_reader/multi_modal_struct.py
Original file line number Diff line number Diff line change
Expand Up @@ -1224,6 +1224,9 @@ def _expand_multimodal_messages(messages: list) -> list:
elif part_type in ("file", "image", "image_url"):
# Extract as a standalone message for its specialised parser
expanded.append(part)
elif part_type == "video_url":
# Keep the chat envelope so the user parser retains video provenance.
expanded.append({**msg, "content": [part]})
else:
text_parts.append(f"[{part_type}]")

Expand Down
12 changes: 12 additions & 0 deletions src/memos/mem_reader/read_multi_modal/user_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,18 @@ def create_source(
)
source.lang = overall_lang
sources.append(source)
elif part_type == "video_url":
video_info = part.get("video_url", {})
source = SourceMessage(
type="video",
role=role,
chat_time=chat_time,
message_id=message_id,
content=video_info.get("url"),
video_info=video_info,
)
source.lang = overall_lang
sources.append(source)
else:
# input_audio, etc.
source = SourceMessage(
Expand Down
1 change: 1 addition & 0 deletions src/memos/types/openai_chat_completion_types/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from .chat_completion_content_part_param import *
from .chat_completion_content_part_refusal_param import *
from .chat_completion_content_part_text_param import *
from .chat_completion_content_part_video_param import *
from .chat_completion_message_custom_tool_call_param import *
from .chat_completion_message_function_tool_call_param import *
from .chat_completion_message_param import *
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from .chat_completion_content_part_image_param import ChatCompletionContentPartImageParam
from .chat_completion_content_part_input_audio_param import ChatCompletionContentPartInputAudioParam
from .chat_completion_content_part_text_param import ChatCompletionContentPartTextParam
from .chat_completion_content_part_video_param import ChatCompletionContentPartVideoParam


__all__ = ["ChatCompletionContentPartParam", "File", "FileFile"]
Expand Down Expand Up @@ -39,5 +40,6 @@ class File(TypedDict, total=False):
ChatCompletionContentPartTextParam
| ChatCompletionContentPartImageParam
| ChatCompletionContentPartInputAudioParam
| ChatCompletionContentPartVideoParam
| File
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from __future__ import annotations

from typing import Literal

from typing_extensions import Required, TypedDict


__all__ = ["ChatCompletionContentPartVideoParam", "VideoURL"]


class VideoURL(TypedDict, total=False):
url: Required[str]
"""A video URL or a data URL containing the encoded video."""


class ChatCompletionContentPartVideoParam(TypedDict, total=False):
video_url: Required[VideoURL]

type: Required[Literal["video_url"]]
"""The type of the content part."""
78 changes: 78 additions & 0 deletions tests/mem_reader/test_video_input.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""Regression tests for video content validation and memory provenance."""

from copy import deepcopy
from unittest.mock import MagicMock

import pytest

from pydantic import TypeAdapter, ValidationError

from memos.mem_reader.multi_modal_struct import MultiModalStructMemReader
from memos.mem_reader.read_multi_modal.user_parser import UserParser
from memos.memories.textual.item import SourceMessage
from memos.types.openai_chat_completion_types import ChatCompletionUserMessageParam


@pytest.mark.parametrize("url", ["https://example.com/clip.mp4", "data:video/mp4;base64,AAAA"])
def test_video_content_survives_chat_validation(url):
message = {
"role": "user",
"content": [{"type": "video_url", "video_url": {"url": url}}],
}

assert TypeAdapter(ChatCompletionUserMessageParam).validate_python(message) == message


def test_video_content_requires_a_url():
message = {"role": "user", "content": [{"type": "video_url", "video_url": {}}]}

with pytest.raises(ValidationError):
TypeAdapter(ChatCompletionUserMessageParam).validate_python(message)


def test_video_source_survives_message_expansion_and_serialization():
video = {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}
message = {
"role": "user",
"content": [{"type": "text", "text": "Remember this clip"}, video],
"chat_time": "2026-10-01T00:00:00Z",
"message_id": "video-message",
}
original = deepcopy(message)
expanded = MultiModalStructMemReader._expand_multimodal_messages([message])
video_messages = [item for item in expanded if isinstance(item.get("content"), list)]

assert len(video_messages) == 1
assert video_messages[0] == {**message, "content": [video]}
assert message == original
assert any(item.get("content") == "Remember this clip" for item in expanded)

parser = UserParser(embedder=MagicMock())
memories = parser.parse_fast(video_messages[0], {}, need_emb=False)
assert len(memories) == 1
source = SourceMessage.model_validate(memories[0].metadata.sources[0].model_dump())
assert source.type == "video"
assert source.video_info == video["video_url"]
assert source.content == video["video_url"]["url"]
assert source.message_id == message["message_id"]
assert source.chat_time == message["chat_time"]
assert source.role == "user"


def test_video_only_message_is_not_replaced_by_a_placeholder():
message = {
"role": "user",
"content": [{"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}}],
}

assert MultiModalStructMemReader._expand_multimodal_messages([message]) == [message]


def test_existing_text_and_image_expansion_is_unchanged():
image = {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}
message = {"role": "user", "content": [{"type": "text", "text": "A picture"}, image]}

assert MultiModalStructMemReader._expand_multimodal_messages([message]) == [
image,
{"role": "user", "content": "A picture"},
]
Loading