From ca22a9d3851d5409e9c2f1aa4593c7aaea2ff8df Mon Sep 17 00:00:00 2001 From: cyurekli Date: Thu, 15 Oct 2026 12:30:42 +0200 Subject: [PATCH 1/2] Honor the selected audio transcription client without cross-provider fallback --- README.md | 14 +++ .../markitdown/converters/_audio_converter.py | 5 +- .../converters/_transcribe_audio.py | 20 +++- packages/markitdown/tests/test_module_misc.py | 103 ++++++++++++++++++ 4 files changed, 139 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fa33f6b343..d1c8272782 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,20 @@ print(result.markdown) If any attempt succeeds, image conversion continues normally. If the client raises an error after exhausting its retries, or encounters a non-retryable error, MarkItDown tries other applicable converters and raises `FileConversionException` only if none succeeds. +### Audio transcription provider + +For the built-in audio converter, supplying `llm_client` selects that client's +OpenAI-compatible `audio.transcriptions.create` endpoint with model `whisper-1`. +The client must support audio transcription. Original encoded audio is uploaded +with its matching filename and media type; this path does not require local +decoding or the legacy Google dependencies. Provider errors propagate; a selected +client's recording is never retried through Google. Transcript headings stay generic. + +Without a client, the legacy SpeechRecognition Google service remains the default +and receives the recording using its generic API key. This behavior is preserved +for compatibility; callers requiring a particular provider should supply its client. +The `language` hint continues to apply to the legacy Google path. + ### Docker ```sh diff --git a/packages/markitdown/src/markitdown/converters/_audio_converter.py b/packages/markitdown/src/markitdown/converters/_audio_converter.py index c4393e92dd..263132b2d0 100644 --- a/packages/markitdown/src/markitdown/converters/_audio_converter.py +++ b/packages/markitdown/src/markitdown/converters/_audio_converter.py @@ -95,7 +95,10 @@ def convert( if audio_format: try: transcript = transcribe_audio( - file_stream, audio_format=audio_format, language=language + file_stream, + audio_format=audio_format, + language=language, + client=kwargs.get("llm_client"), ) if transcript: md_content += "\n\n### Audio Transcript:\n" + transcript diff --git a/packages/markitdown/src/markitdown/converters/_transcribe_audio.py b/packages/markitdown/src/markitdown/converters/_transcribe_audio.py index 6caa2c656b..2af18687f2 100644 --- a/packages/markitdown/src/markitdown/converters/_transcribe_audio.py +++ b/packages/markitdown/src/markitdown/converters/_transcribe_audio.py @@ -1,6 +1,6 @@ import io import sys -from typing import BinaryIO +from typing import Any, BinaryIO from .._exceptions import MissingDependencyException # Try loading optional (but in this case, required) dependencies @@ -27,8 +27,24 @@ def transcribe_audio( - file_stream: BinaryIO, *, audio_format: str = "wav", language: str = "en-US" + file_stream: BinaryIO, + *, + audio_format: str = "wav", + language: str = "en-US", + client: Any = None, ) -> str: + if client is not None: + # An explicit provider stays selected, including on authentication/network + # errors. Upload the original encoding with its filename and media type; + # this path does not need the legacy Google or local decoder dependencies. + media_type = {"mp3": "audio/mpeg"}.get(audio_format, f"audio/{audio_format}") + transcription = client.audio.transcriptions.create( + model="whisper-1", + file=(f"audio.{audio_format}", file_stream, media_type), + ) + transcript = transcription.text.strip() + return "[No speech detected]" if not transcript else transcript + # Check for installed dependencies if _dependency_exc_info is not None: raise MissingDependencyException( diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 4d6c9be59b..ced47bdf16 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -878,3 +878,106 @@ def test_a_declared_charset_still_wins() -> None: if __name__ == "__main__": raise SystemExit(pytest.main([__file__])) + + +# Audio provider selection (no recordings leave the test process). +@pytest.fixture +def audio_provider_mocks(monkeypatch): + from contextlib import nullcontext + from markitdown.converters import _audio_converter, _transcribe_audio + + recognizer = MagicMock() + recognizer.recognize_google.return_value = "legacy Google transcript" + monkeypatch.setattr(_transcribe_audio, "_dependency_exc_info", None) + monkeypatch.setattr( + _transcribe_audio, + "sr", + SimpleNamespace( + Recognizer=lambda: recognizer, AudioFile=lambda _: nullcontext(object()) + ), + raising=False, + ) + monkeypatch.setattr(_audio_converter, "exiftool_metadata", lambda *a, **kw: {}) + return recognizer + + +def convert_mock_wav(client=None, **kwargs): + return MarkItDown(llm_client=client).convert_stream( + io.BytesIO((Path(TEST_FILES_DIR) / "test.wav").read_bytes()), + stream_info=StreamInfo(extension=".wav", mimetype="audio/x-wav"), + **kwargs, + ) + + +def test_audio_uses_selected_client_without_google(audio_provider_mocks): + client = MagicMock() + client.audio.transcriptions.create.return_value.text = "selected transcript" + result = convert_mock_wav(client) + assert "selected transcript" in result.markdown + audio_provider_mocks.recognize_google.assert_not_called() + request = client.audio.transcriptions.create.call_args.kwargs + assert request["model"] == "whisper-1" + assert request["file"][0] == "audio.wav" + assert request["file"][2] == "audio/wav" + + +def test_audio_selected_client_failure_never_falls_back(audio_provider_mocks): + client = MagicMock() + client.audio.transcriptions.create.side_effect = RuntimeError( + "selected provider failed" + ) + with pytest.raises(FileConversionException, match="selected provider failed"): + convert_mock_wav(client) + audio_provider_mocks.recognize_google.assert_not_called() + + +def test_audio_selected_client_empty_transcript(audio_provider_mocks): + client = MagicMock() + client.audio.transcriptions.create.return_value.text = " " + assert "[No speech detected]" in convert_mock_wav(client).markdown + audio_provider_mocks.recognize_google.assert_not_called() + + +def test_audio_without_client_preserves_legacy_language(audio_provider_mocks): + assert "legacy Google transcript" in convert_mock_wav(language="fr-FR").markdown + assert audio_provider_mocks.recognize_google.call_args.kwargs == { + "language": "fr-FR" + } + + +def test_selected_audio_client_does_not_require_legacy_dependencies( + audio_provider_mocks, monkeypatch +): + from markitdown.converters import _transcribe_audio + + monkeypatch.setattr( + _transcribe_audio, + "_dependency_exc_info", + (ImportError, ImportError("legacy dependencies absent"), None), + ) + client = MagicMock() + client.audio.transcriptions.create.return_value.text = "selected transcript" + assert "selected transcript" in convert_mock_wav(client).markdown + audio_provider_mocks.recognize_google.assert_not_called() + + +@pytest.mark.parametrize("audio_format", ["mp3", "mp4", "flac", "aiff"]) +def test_selected_audio_client_receives_original_encoded_format( + audio_provider_mocks, audio_format +): + from markitdown.converters._transcribe_audio import transcribe_audio + + stream = io.BytesIO(b"original encoded audio") + client = MagicMock() + client.audio.transcriptions.create.return_value.text = "selected transcript" + assert ( + transcribe_audio(stream, audio_format=audio_format, client=client) + == "selected transcript" + ) + request = client.audio.transcriptions.create.call_args.kwargs + assert request["file"][0] == f"audio.{audio_format}" + assert request["file"][1].read() == b"original encoded audio" + assert request["file"][2] == {"mp3": "audio/mpeg"}.get( + audio_format, f"audio/{audio_format}" + ) + audio_provider_mocks.recognize_google.assert_not_called() From 96f9764ca57aea91bf83104e6eb226ed68e001b3 Mon Sep 17 00:00:00 2001 From: cyurekli Date: Sat, 17 Oct 2026 11:49:59 +0200 Subject: [PATCH 2/2] Preserve chat-only audio compatibility and configure transcription models --- README.md | 33 ++++++---- .../markitdown/src/markitdown/_markitdown.py | 5 ++ .../markitdown/converters/_audio_converter.py | 1 + .../converters/_transcribe_audio.py | 10 ++- packages/markitdown/tests/test_module_misc.py | 66 +++++++++++++++++++ 5 files changed, 101 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d1c8272782..fb1fef7876 100644 --- a/README.md +++ b/README.md @@ -299,17 +299,28 @@ If any attempt succeeds, image conversion continues normally. If the client rais ### Audio transcription provider -For the built-in audio converter, supplying `llm_client` selects that client's -OpenAI-compatible `audio.transcriptions.create` endpoint with model `whisper-1`. -The client must support audio transcription. Original encoded audio is uploaded -with its matching filename and media type; this path does not require local -decoding or the legacy Google dependencies. Provider errors propagate; a selected -client's recording is never retried through Google. Transcript headings stay generic. - -Without a client, the legacy SpeechRecognition Google service remains the default -and receives the recording using its generic API key. This behavior is preserved -for compatibility; callers requiring a particular provider should supply its client. -The `language` hint continues to apply to the legacy Google path. +The built-in audio converter uses `llm_client` for transcription when the client +exposes a callable `audio.transcriptions.create` endpoint. Original encoded audio +is uploaded with its matching filename and media type; this path does not require +local decoding or the legacy Google dependencies. Once that endpoint is selected, +provider errors propagate without retrying the recording through Google. + +Set `audio_model` to the transcription model or Azure deployment name; the default +is `whisper-1`. It is separate from `llm_model`, which configures image descriptions. +A conversion-level `audio_model` overrides the constructor setting: + +```python +md = MarkItDown(llm_client=client, llm_model="image-model", audio_model="speech-deployment") +result = md.convert("recording.wav", audio_model="another-speech-deployment") +``` + +Without a transcription-capable client (including a chat-only client), the legacy +SpeechRecognition Google service remains the default and receives the recording +using its generic API key when the optional audio dependencies are installed. +Without those dependencies, legacy transcription is skipped. This preserves +existing image-only client usage. The `language` hint applies to the legacy Google +path. Callers requiring a particular provider should supply a client with the audio +endpoint and the corresponding transcription model or deployment name. ### Docker diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 8e2fe6a37c..9ecb27e792 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -170,6 +170,7 @@ def __init__( # TODO - remove these (see enable_builtins) self._llm_client: Any = None self._llm_model: Union[str | None] = None + self._audio_model: Union[str | None] = None self._llm_prompt: Union[str | None] = None self._exiftool_path: Union[str | None] = None self._style_map: Union[str | None] = None @@ -195,6 +196,7 @@ def enable_builtins(self, **kwargs) -> None: # TODO: Move these into converter constructors self._llm_client = kwargs.get("llm_client") self._llm_model = kwargs.get("llm_model") + self._audio_model = kwargs.get("audio_model") self._llm_prompt = kwargs.get("llm_prompt") self._exiftool_path = kwargs.get("exiftool_path") self._style_map = kwargs.get("style_map") @@ -632,6 +634,9 @@ def _convert( if "llm_model" not in _kwargs and self._llm_model is not None: _kwargs["llm_model"] = self._llm_model + if "audio_model" not in _kwargs and self._audio_model is not None: + _kwargs["audio_model"] = self._audio_model + if "llm_prompt" not in _kwargs and self._llm_prompt is not None: _kwargs["llm_prompt"] = self._llm_prompt diff --git a/packages/markitdown/src/markitdown/converters/_audio_converter.py b/packages/markitdown/src/markitdown/converters/_audio_converter.py index 263132b2d0..a5313e86f5 100644 --- a/packages/markitdown/src/markitdown/converters/_audio_converter.py +++ b/packages/markitdown/src/markitdown/converters/_audio_converter.py @@ -99,6 +99,7 @@ def convert( audio_format=audio_format, language=language, client=kwargs.get("llm_client"), + model=kwargs.get("audio_model", "whisper-1"), ) if transcript: md_content += "\n\n### Audio Transcript:\n" + transcript diff --git a/packages/markitdown/src/markitdown/converters/_transcribe_audio.py b/packages/markitdown/src/markitdown/converters/_transcribe_audio.py index 2af18687f2..3227dac5d1 100644 --- a/packages/markitdown/src/markitdown/converters/_transcribe_audio.py +++ b/packages/markitdown/src/markitdown/converters/_transcribe_audio.py @@ -32,14 +32,18 @@ def transcribe_audio( audio_format: str = "wav", language: str = "en-US", client: Any = None, + model: str = "whisper-1", ) -> str: - if client is not None: + create_transcription = getattr( + getattr(getattr(client, "audio", None), "transcriptions", None), "create", None + ) + if callable(create_transcription): # An explicit provider stays selected, including on authentication/network # errors. Upload the original encoding with its filename and media type; # this path does not need the legacy Google or local decoder dependencies. media_type = {"mp3": "audio/mpeg"}.get(audio_format, f"audio/{audio_format}") - transcription = client.audio.transcriptions.create( - model="whisper-1", + transcription = create_transcription( + model=model, file=(f"audio.{audio_format}", file_stream, media_type), ) transcript = transcription.text.strip() diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index ced47bdf16..51f2565f08 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -981,3 +981,69 @@ def test_selected_audio_client_receives_original_encoded_format( audio_format, f"audio/{audio_format}" ) audio_provider_mocks.recognize_google.assert_not_called() + + +@pytest.mark.parametrize("legacy_dependencies", [True, False]) +def test_audio_chat_only_client_preserves_legacy_behavior( + audio_provider_mocks, monkeypatch, legacy_dependencies +): + from markitdown.converters import _transcribe_audio + + if not legacy_dependencies: + monkeypatch.setattr( + _transcribe_audio, + "_dependency_exc_info", + (ImportError, ImportError("legacy dependencies absent"), None), + ) + result = convert_mock_wav(SimpleNamespace(chat=object()), language="fr-FR") + if legacy_dependencies: + assert "legacy Google transcript" in result.markdown + audio_provider_mocks.recognize_google.assert_called_once() + assert audio_provider_mocks.recognize_google.call_args.kwargs == { + "language": "fr-FR" + } + else: + assert "Audio Transcript" not in result.markdown + audio_provider_mocks.recognize_google.assert_not_called() + + +def test_audio_noncallable_endpoint_preserves_legacy_behavior(audio_provider_mocks): + client = SimpleNamespace( + audio=SimpleNamespace(transcriptions=SimpleNamespace(create=None)) + ) + assert "legacy Google transcript" in convert_mock_wav(client).markdown + audio_provider_mocks.recognize_google.assert_called_once() + + +@pytest.mark.parametrize( + "constructor_model, conversion_model, expected_model", + [ + ("azure-speech", None, "azure-speech"), + ("azure-speech", "override-speech", "override-speech"), + (None, "override-speech", "override-speech"), + (None, None, "whisper-1"), + ], +) +def test_audio_model_is_separate_from_image_model( + audio_provider_mocks, constructor_model, conversion_model, expected_model +): + client = MagicMock() + client.audio.transcriptions.create.return_value.text = "selected transcript" + constructor_options = ( + {} if constructor_model is None else {"audio_model": constructor_model} + ) + conversion_options = ( + {} if conversion_model is None else {"audio_model": conversion_model} + ) + md = MarkItDown(llm_client=client, llm_model="image-model", **constructor_options) + result = md.convert_stream( + io.BytesIO((Path(TEST_FILES_DIR) / "test.wav").read_bytes()), + stream_info=StreamInfo(extension=".wav", mimetype="audio/x-wav"), + **conversion_options, + ) + assert "selected transcript" in result.markdown + client.audio.transcriptions.create.assert_called_once() + assert ( + client.audio.transcriptions.create.call_args.kwargs["model"] == expected_model + ) + audio_provider_mocks.recognize_google.assert_not_called()