From a99630d4f34081b1b713eeabe08efa237d200185 Mon Sep 17 00:00:00 2001 From: jeffwu Date: Wed, 26 Aug 2026 17:26:55 +0800 Subject: [PATCH 1/2] fix(tools): correct analyze_audio description to reference audio model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit analyze_audio 工具的描述、docstring、字段说明和报错文案此前写成 "视频理解模型/video understanding model"，与工具用途不符，也与同文件 内已有的"音频理解模型"措辞不一致。统一改为"音频理解模型/audio understanding model"，并将文档前置配置中 analyze_video/analyze_audio 拆分为各自对应的视频/音频理解模型。 vlm_model 参数名保留不改，避免牵连 tool_configuration_service 等调用方 的接口级改动。 Co-Authored-By: Claude --- .../local-tools/multimodal-tools.md | 2 +- .../local-tools/multimodal-tools.md | 2 +- sdk/nexent/core/tools/analyze_audio_tool.py | 20 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/docs/en/user-guide/local-tools/multimodal-tools.md b/doc/docs/en/user-guide/local-tools/multimodal-tools.md index 986682c403..6c31b03cc7 100644 --- a/doc/docs/en/user-guide/local-tools/multimodal-tools.md +++ b/doc/docs/en/user-guide/local-tools/multimodal-tools.md @@ -46,7 +46,7 @@ Multimodal tools analyze text files, images, videos, and audio with model suppor ## ⚙️ Prerequisites - Configure storage access (e.g., MinIO/S3) and data processing service to fetch files. -- Provide an LLM for `analyze_text_file`, a VLM for `analyze_image`, and a video understanding model for `analyze_video` and `analyze_audio` (must support audio/video input, e.g., Qwen3-Omni series). +- Provide an LLM for `analyze_text_file`, a VLM for `analyze_image`, a video understanding model for `analyze_video`, and an audio understanding model for `analyze_audio` (all must support audio/video input, e.g., Qwen3-Omni series). ## 🛠️ How to Use diff --git a/doc/docs/zh/user-guide/local-tools/multimodal-tools.md b/doc/docs/zh/user-guide/local-tools/multimodal-tools.md index 3470a2c1cc..0d6b02edd0 100644 --- a/doc/docs/zh/user-guide/local-tools/multimodal-tools.md +++ b/doc/docs/zh/user-guide/local-tools/multimodal-tools.md @@ -46,7 +46,7 @@ title: 多模态工具 ## ⚙️ 前置配置 - 确保已在平台配置可用的存储客户端（如 MinIO/S3）及数据处理服务地址，保证能下载文件。 -- 为 `analyze_text_file` 配置可用的 LLM；为 `analyze_image` 配置可用的视觉语言模型；为 `analyze_video` 和 `analyze_audio` 配置可用的视频理解模型（需支持音视频输入，如 Qwen3-Omni 系列模型）。 +- 为 `analyze_text_file` 配置可用的 LLM；为 `analyze_image` 配置可用的视觉语言模型；为 `analyze_video` 配置可用的视频理解模型，为 `analyze_audio` 配置可用的音频理解模型（均需支持音视频输入，如 Qwen3-Omni 系列模型）。 ## 🛠️ 操作指引 diff --git a/sdk/nexent/core/tools/analyze_audio_tool.py b/sdk/nexent/core/tools/analyze_audio_tool.py index cd4feb30ab..b2c05ac448 100644 --- a/sdk/nexent/core/tools/analyze_audio_tool.py +++ b/sdk/nexent/core/tools/analyze_audio_tool.py @@ -1,7 +1,7 @@ """ Analyze Audio Tool -Analyze audio using the configured video understanding model. +Analyze audio using the configured audio understanding model. Supports audio from S3, HTTP, and HTTPS URLs. """ @@ -25,18 +25,18 @@ class AnalyzeAudioTool(Tool): - """Tool for understanding and analyzing audio using the video understanding model.""" + """Tool for understanding and analyzing audio using the audio understanding model.""" name = "analyze_audio" skip_forward_signature_validation = True description = ( - "This tool uses the configured video understanding model to understand audio based on your query and then returns an audio analysis result.\n" + "This tool uses the configured audio understanding model to understand audio based on your query and then returns an audio analysis result.\n" "It is used to understand and analyze one audio file, with sources supporting S3 URLs (s3://bucket/key or /bucket/key), " "HTTP, and HTTPS URLs.\n" "Use this tool when you want to retrieve information contained in audio and provide the audio URL and your query." ) description_zh = ( - "使用视频理解模型，根据你的问题理解音频，并返回音频分析结果。" + "使用音频理解模型，根据你的问题理解音频，并返回音频分析结果。" "可用于理解和分析一个音频文件，支持 S3 URL（s3://bucket/key 或 /bucket/key）、HTTP 和 HTTPS URL。" ) @@ -55,9 +55,9 @@ class AnalyzeAudioTool(Tool): init_param_descriptions = { "observer": {"description": "Message observer"}, - "vlm_model": {"description": "The video understanding model to use"}, + "vlm_model": {"description": "The audio understanding model to use"}, "selected_model_id": { - "description": "Optional Nexent video understanding model ID to use for audio analysis. If omitted, the default video understanding model is used." + "description": "Optional Nexent audio understanding model ID to use for audio analysis. If omitted, the default audio understanding model is used." }, "storage_client": {"description": "Storage client for downloading files"}, "validate_url_access": { @@ -75,11 +75,11 @@ def __init__( default=None, exclude=True), vlm_model: Any = Field( - description="The video understanding model to use", + description="The audio understanding model to use", default=None, exclude=True), selected_model_id: int = Field( - description="Optional Nexent video understanding model ID to use for audio analysis. If omitted, the default video understanding model is used.", + description="Optional Nexent audio understanding model ID to use for audio analysis. If omitted, the default audio understanding model is used.", default=None), storage_client: MinIOStorageClient = Field( description="Storage client for downloading files from S3 URLs, HTTP URLs, and HTTPS URLs.", @@ -178,8 +178,8 @@ def _forward_impl( ) ) except Exception as e: - error_msg_zh = f"音频{index}分析失败: {str(e)}。请检查视频理解模型配置是否正确。" - error_msg_en = f"Failed to analyze audio {index}: {str(e)}. Please check if the video understanding model is configured correctly." + error_msg_zh = f"音频{index}分析失败: {str(e)}。请检查音频理解模型配置是否正确。" + error_msg_en = f"Failed to analyze audio {index}: {str(e)}. Please check if the audio understanding model is configured correctly." error_msg = error_msg_zh if self._is_chinese else error_msg_en raise Exception(error_msg) From abe41251b2eac20501dc701e25c3aa8867f2d9bb Mon Sep 17 00:00:00 2001 From: jeffwu Date: Wed, 26 Aug 2026 18:16:21 +0800 Subject: [PATCH 2/2] test(tools): cover analyze_audio model error branch --- .../tools/test_analyze_audio_video_tool.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/sdk/core/tools/test_analyze_audio_video_tool.py b/test/sdk/core/tools/test_analyze_audio_video_tool.py index 0380a7696d..ff169b36ef 100644 --- a/test/sdk/core/tools/test_analyze_audio_video_tool.py +++ b/test/sdk/core/tools/test_analyze_audio_video_tool.py @@ -53,6 +53,25 @@ def _fake_get_prompt(template_type, language=None, **_): assert hasattr(request.media_input, "read") assert request.kwargs["content_type"].startswith("audio/") +def test_analyze_audio_raises_audio_model_error(observer_en, mock_vlm_model, mock_storage_client, monkeypatch): + monkeypatch.setattr( + analyze_audio_tool, + "get_prompt_template", + lambda template_type, language=None, **_: {"system_prompt": "Analyze audio for {{ query }}"}, + ) + mock_vlm_model.invoke_sync.side_effect = RuntimeError("audio model unavailable") + tool = AnalyzeAudioTool( + observer=observer_en, + vlm_model=mock_vlm_model, + storage_client=mock_storage_client, + ) + + with pytest.raises(Exception) as exc_info: + tool._forward_impl(audio_url=b"ID3audio-bytes", query="what happened?") + + assert "Failed to analyze audio 1" in str(exc_info.value) + assert "audio understanding model is configured correctly" in str(exc_info.value) + def test_analyze_audio_schema_uses_single_url(): assert "audio_url" in AnalyzeAudioTool.inputs assert "audio_urls_list" not in AnalyzeAudioTool.inputs