Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/docs/en/user-guide/local-tools/multimodal-tools.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ Multimodal tools analyze text files, images, videos, and audio with model suppor
## ⚙️ Prerequisites

- Configure storage access (e.g., MinIO/S3) and data processing service to fetch files.
- Provide an LLM for `analyze_text_file`, a VLM for `analyze_image`, and a video understanding model for `analyze_video` and `analyze_audio` (must support audio/video input, e.g., Qwen3-Omni series).
- Provide an LLM for `analyze_text_file`, a VLM for `analyze_image`, a video understanding model for `analyze_video`, and an audio understanding model for `analyze_audio` (all must support audio/video input, e.g., Qwen3-Omni series).

## 🛠️ How to Use

Expand Down
2 changes: 1 addition & 1 deletion doc/docs/zh/user-guide/local-tools/multimodal-tools.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ title: 多模态工具
## ⚙️ 前置配置

- 确保已在平台配置可用的存储客户端(如 MinIO/S3)及数据处理服务地址,保证能下载文件。
- 为 `analyze_text_file` 配置可用的 LLM;为 `analyze_image` 配置可用的视觉语言模型;为 `analyze_video` `analyze_audio` 配置可用的视频理解模型(需支持音视频输入,如 Qwen3-Omni 系列模型)。
- 为 `analyze_text_file` 配置可用的 LLM;为 `analyze_image` 配置可用的视觉语言模型;为 `analyze_video` 配置可用的视频理解模型,为 `analyze_audio` 配置可用的音频理解模型(均需支持音视频输入,如 Qwen3-Omni 系列模型)。

## 🛠️ 操作指引

Expand Down
20 changes: 10 additions & 10 deletions sdk/nexent/core/tools/analyze_audio_tool.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""
Analyze Audio Tool

Analyze audio using the configured video understanding model.
Analyze audio using the configured audio understanding model.
Supports audio from S3, HTTP, and HTTPS URLs.
"""

Expand All @@ -25,18 +25,18 @@


class AnalyzeAudioTool(Tool):
"""Tool for understanding and analyzing audio using the video understanding model."""
"""Tool for understanding and analyzing audio using the audio understanding model."""

name = "analyze_audio"
skip_forward_signature_validation = True
description = (
"This tool uses the configured video understanding model to understand audio based on your query and then returns an audio analysis result.\n"
"This tool uses the configured audio understanding model to understand audio based on your query and then returns an audio analysis result.\n"
"It is used to understand and analyze one audio file, with sources supporting S3 URLs (s3://bucket/key or /bucket/key), "
"HTTP, and HTTPS URLs.\n"
"Use this tool when you want to retrieve information contained in audio and provide the audio URL and your query."
)
description_zh = (
"使用视频理解模型,根据你的问题理解音频,并返回音频分析结果。"
"使用音频理解模型,根据你的问题理解音频,并返回音频分析结果。"
"可用于理解和分析一个音频文件,支持 S3 URL(s3://bucket/key 或 /bucket/key)、HTTP 和 HTTPS URL。"
)

Expand All @@ -55,9 +55,9 @@ class AnalyzeAudioTool(Tool):

init_param_descriptions = {
"observer": {"description": "Message observer"},
"vlm_model": {"description": "The video understanding model to use"},
"vlm_model": {"description": "The audio understanding model to use"},
"selected_model_id": {
"description": "Optional Nexent video understanding model ID to use for audio analysis. If omitted, the default video understanding model is used."
"description": "Optional Nexent audio understanding model ID to use for audio analysis. If omitted, the default audio understanding model is used."
},
"storage_client": {"description": "Storage client for downloading files"},
"validate_url_access": {
Expand All @@ -75,11 +75,11 @@ def __init__(
default=None,
exclude=True),
vlm_model: Any = Field(
description="The video understanding model to use",
description="The audio understanding model to use",
default=None,
exclude=True),
selected_model_id: int = Field(
description="Optional Nexent video understanding model ID to use for audio analysis. If omitted, the default video understanding model is used.",
description="Optional Nexent audio understanding model ID to use for audio analysis. If omitted, the default audio understanding model is used.",
default=None),
storage_client: MinIOStorageClient = Field(
description="Storage client for downloading files from S3 URLs, HTTP URLs, and HTTPS URLs.",
Expand Down Expand Up @@ -178,8 +178,8 @@ def _forward_impl(
)
)
except Exception as e:
error_msg_zh = f"音频{index}分析失败: {str(e)}。请检查视频理解模型配置是否正确。"
error_msg_en = f"Failed to analyze audio {index}: {str(e)}. Please check if the video understanding model is configured correctly."
error_msg_zh = f"音频{index}分析失败: {str(e)}。请检查音频理解模型配置是否正确。"
error_msg_en = f"Failed to analyze audio {index}: {str(e)}. Please check if the audio understanding model is configured correctly."
error_msg = error_msg_zh if self._is_chinese else error_msg_en
raise Exception(error_msg)

Expand Down
19 changes: 19 additions & 0 deletions test/sdk/core/tools/test_analyze_audio_video_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,25 @@
assert hasattr(request.media_input, "read")
assert request.kwargs["content_type"].startswith("audio/")

def test_analyze_audio_raises_audio_model_error(observer_en, mock_vlm_model, mock_storage_client, monkeypatch):
monkeypatch.setattr(
analyze_audio_tool,
"get_prompt_template",
lambda template_type, language=None, **_: {"system_prompt": "Analyze audio for {{ query }}"},
)
mock_vlm_model.invoke_sync.side_effect = RuntimeError("audio model unavailable")
tool = AnalyzeAudioTool(
observer=observer_en,
vlm_model=mock_vlm_model,
storage_client=mock_storage_client,
)

with pytest.raises(Exception) as exc_info:

Check warning on line 69 in test/sdk/core/tools/test_analyze_audio_video_tool.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

This assertion is too broad; use a more specific exception type or check the exception message.

See more on https://sonarcloud.io/project/issues?id=ModelEngine-Group_nexent&issues=AaA9k0zEY2O8i2hy89u9&open=AaA9k0zEY2O8i2hy89u9&pullRequest=3787
tool._forward_impl(audio_url=b"ID3audio-bytes", query="what happened?")

assert "Failed to analyze audio 1" in str(exc_info.value)
assert "audio understanding model is configured correctly" in str(exc_info.value)

def test_analyze_audio_schema_uses_single_url():
assert "audio_url" in AnalyzeAudioTool.inputs
assert "audio_urls_list" not in AnalyzeAudioTool.inputs
Expand Down
Loading