diff --git a/doc/docs/en/user-guide/local-tools/multimodal-tools.md b/doc/docs/en/user-guide/local-tools/multimodal-tools.md index 986682c40..6c31b03cc 100644 --- a/doc/docs/en/user-guide/local-tools/multimodal-tools.md +++ b/doc/docs/en/user-guide/local-tools/multimodal-tools.md @@ -46,7 +46,7 @@ Multimodal tools analyze text files, images, videos, and audio with model suppor ## ⚙️ Prerequisites - Configure storage access (e.g., MinIO/S3) and data processing service to fetch files. -- Provide an LLM for `analyze_text_file`, a VLM for `analyze_image`, and a video understanding model for `analyze_video` and `analyze_audio` (must support audio/video input, e.g., Qwen3-Omni series). +- Provide an LLM for `analyze_text_file`, a VLM for `analyze_image`, a video understanding model for `analyze_video`, and an audio understanding model for `analyze_audio` (all must support audio/video input, e.g., Qwen3-Omni series). ## 🛠️ How to Use diff --git a/doc/docs/zh/user-guide/local-tools/multimodal-tools.md b/doc/docs/zh/user-guide/local-tools/multimodal-tools.md index 3470a2c1c..0d6b02edd 100644 --- a/doc/docs/zh/user-guide/local-tools/multimodal-tools.md +++ b/doc/docs/zh/user-guide/local-tools/multimodal-tools.md @@ -46,7 +46,7 @@ title: 多模态工具 ## ⚙️ 前置配置 - 确保已在平台配置可用的存储客户端(如 MinIO/S3)及数据处理服务地址,保证能下载文件。 -- 为 `analyze_text_file` 配置可用的 LLM;为 `analyze_image` 配置可用的视觉语言模型;为 `analyze_video` 和 `analyze_audio` 配置可用的视频理解模型(需支持音视频输入,如 Qwen3-Omni 系列模型)。 +- 为 `analyze_text_file` 配置可用的 LLM;为 `analyze_image` 配置可用的视觉语言模型;为 `analyze_video` 配置可用的视频理解模型,为 `analyze_audio` 配置可用的音频理解模型(均需支持音视频输入,如 Qwen3-Omni 系列模型)。 ## 🛠️ 操作指引 diff --git a/sdk/nexent/core/tools/analyze_audio_tool.py b/sdk/nexent/core/tools/analyze_audio_tool.py index cd4feb30a..b2c05ac44 100644 --- a/sdk/nexent/core/tools/analyze_audio_tool.py +++ b/sdk/nexent/core/tools/analyze_audio_tool.py @@ -1,7 +1,7 @@ """ Analyze Audio Tool -Analyze audio using the configured video understanding model. +Analyze audio using the configured audio understanding model. Supports audio from S3, HTTP, and HTTPS URLs. """ @@ -25,18 +25,18 @@ class AnalyzeAudioTool(Tool): - """Tool for understanding and analyzing audio using the video understanding model.""" + """Tool for understanding and analyzing audio using the audio understanding model.""" name = "analyze_audio" skip_forward_signature_validation = True description = ( - "This tool uses the configured video understanding model to understand audio based on your query and then returns an audio analysis result.\n" + "This tool uses the configured audio understanding model to understand audio based on your query and then returns an audio analysis result.\n" "It is used to understand and analyze one audio file, with sources supporting S3 URLs (s3://bucket/key or /bucket/key), " "HTTP, and HTTPS URLs.\n" "Use this tool when you want to retrieve information contained in audio and provide the audio URL and your query." ) description_zh = ( - "使用视频理解模型,根据你的问题理解音频,并返回音频分析结果。" + "使用音频理解模型,根据你的问题理解音频,并返回音频分析结果。" "可用于理解和分析一个音频文件,支持 S3 URL(s3://bucket/key 或 /bucket/key)、HTTP 和 HTTPS URL。" ) @@ -55,9 +55,9 @@ class AnalyzeAudioTool(Tool): init_param_descriptions = { "observer": {"description": "Message observer"}, - "vlm_model": {"description": "The video understanding model to use"}, + "vlm_model": {"description": "The audio understanding model to use"}, "selected_model_id": { - "description": "Optional Nexent video understanding model ID to use for audio analysis. If omitted, the default video understanding model is used." + "description": "Optional Nexent audio understanding model ID to use for audio analysis. If omitted, the default audio understanding model is used." }, "storage_client": {"description": "Storage client for downloading files"}, "validate_url_access": { @@ -75,11 +75,11 @@ def __init__( default=None, exclude=True), vlm_model: Any = Field( - description="The video understanding model to use", + description="The audio understanding model to use", default=None, exclude=True), selected_model_id: int = Field( - description="Optional Nexent video understanding model ID to use for audio analysis. If omitted, the default video understanding model is used.", + description="Optional Nexent audio understanding model ID to use for audio analysis. If omitted, the default audio understanding model is used.", default=None), storage_client: MinIOStorageClient = Field( description="Storage client for downloading files from S3 URLs, HTTP URLs, and HTTPS URLs.", @@ -178,8 +178,8 @@ def _forward_impl( ) ) except Exception as e: - error_msg_zh = f"音频{index}分析失败: {str(e)}。请检查视频理解模型配置是否正确。" - error_msg_en = f"Failed to analyze audio {index}: {str(e)}. Please check if the video understanding model is configured correctly." + error_msg_zh = f"音频{index}分析失败: {str(e)}。请检查音频理解模型配置是否正确。" + error_msg_en = f"Failed to analyze audio {index}: {str(e)}. Please check if the audio understanding model is configured correctly." error_msg = error_msg_zh if self._is_chinese else error_msg_en raise Exception(error_msg) diff --git a/test/sdk/core/tools/test_analyze_audio_video_tool.py b/test/sdk/core/tools/test_analyze_audio_video_tool.py index 0380a7696..ff169b36e 100644 --- a/test/sdk/core/tools/test_analyze_audio_video_tool.py +++ b/test/sdk/core/tools/test_analyze_audio_video_tool.py @@ -53,6 +53,25 @@ def _fake_get_prompt(template_type, language=None, **_): assert hasattr(request.media_input, "read") assert request.kwargs["content_type"].startswith("audio/") +def test_analyze_audio_raises_audio_model_error(observer_en, mock_vlm_model, mock_storage_client, monkeypatch): + monkeypatch.setattr( + analyze_audio_tool, + "get_prompt_template", + lambda template_type, language=None, **_: {"system_prompt": "Analyze audio for {{ query }}"}, + ) + mock_vlm_model.invoke_sync.side_effect = RuntimeError("audio model unavailable") + tool = AnalyzeAudioTool( + observer=observer_en, + vlm_model=mock_vlm_model, + storage_client=mock_storage_client, + ) + + with pytest.raises(Exception) as exc_info: + tool._forward_impl(audio_url=b"ID3audio-bytes", query="what happened?") + + assert "Failed to analyze audio 1" in str(exc_info.value) + assert "audio understanding model is configured correctly" in str(exc_info.value) + def test_analyze_audio_schema_uses_single_url(): assert "audio_url" in AnalyzeAudioTool.inputs assert "audio_urls_list" not in AnalyzeAudioTool.inputs