diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 16d2186..1a6c63a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,8 @@ on: pull_request: branches: [ main ] +# ggml is built from source via FetchContent (see CMakeLists), so no system ggml +# is required on any platform. Static build => self-contained binary artifacts. jobs: build: name: build on ${{ matrix.os }} (${{ matrix.arch }}) @@ -16,7 +18,7 @@ jobs: include: - os: ubuntu-latest arch: x86_64 - - os: ubuntu-24.04-arm64 + - os: ubuntu-24.04-arm arch: arm64 - os: macos-latest arch: arm64 @@ -24,45 +26,22 @@ jobs: arch: x86_64 steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install dependencies (Linux) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y libsdl2-dev build-essential cmake - - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install sdl2 cmake - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - vcpkg install sdl2:x64-windows - vcpkg integrate install + - uses: actions/checkout@v4 - - name: Configure CMake - run: > - cmake -S . -B build - -DCMAKE_BUILD_TYPE=Release - ${{ runner.os == 'Windows' && '-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }} - ${{ runner.os == 'macOS' && '-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"' || '' }} + - name: Configure + run: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF - name: Build run: cmake --build build --config Release -j - - name: Upload Artifacts - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v4 with: name: voxtral-${{ runner.os }}-${{ matrix.arch }} path: | - build/voxtral* - build/examples/stream/voxtral-stream* - !build/**/*.dir - !build/**/*.obj - !build/**/*.o + build/voxtral + build/voxtral.exe + build/voxtral-quantize + build/voxtral-quantize.exe + build/Release/voxtral.exe + build/Release/voxtral-quantize.exe + if-no-files-found: ignore diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f68790b..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "ggml"] - path = ggml - url = https://github.com/ggml-org/ggml.git diff --git a/CMakeLists.txt b/CMakeLists.txt index f4ce9f1..5f8847f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,8 +9,10 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(VOXTRAL_WARNINGS_AS_ERRORS "Treat warnings as errors" OFF) option(VOXTRAL_NATIVE_OPT "Enable native CPU tuning flags" ON) -option(VOXTRAL_AUTO_DETECT_CPU "Auto-detect CPU features and set ggml flags" ON) -option(VOXTRAL_AUTO_DETECT_BLAS "Auto-enable ggml BLAS if found" ON) +# Homebrew uses the system ggml formula; everywhere else ggml is built from source +# via FetchContent (no vendored submodule). Backends are discovered at runtime via +# the ggml registry, so no backend flags are needed here. +option(VOXTRAL_USE_SYSTEM_GGML "Link a system ggml instead of building it from source" OFF) if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) @@ -18,75 +20,53 @@ endif() find_package(Threads REQUIRED) -set(GGML_BUILD_TESTS OFF CACHE BOOL "Disable ggml tests" FORCE) -set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "Disable ggml examples" FORCE) - -if(VOXTRAL_AUTO_DETECT_CPU AND NOT CMAKE_CROSSCOMPILING) - if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|i[3-6]86") - include(CheckCXXSourceRuns) - - function(voxtral_check_cpu_supports feature outvar) - set(_src " - #if !(defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) - int main() { return 1; } - #endif - #if defined(__GNUC__) || defined(__clang__) - int main() { return __builtin_cpu_supports(\"${feature}\") ? 0 : 1; } - #else - int main() { return 1; } - #endif - ") - check_cxx_source_runs("${_src}" ${outvar}) - endfunction() - - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") - voxtral_check_cpu_supports("avx" VOXTRAL_CPU_HAS_AVX) - voxtral_check_cpu_supports("avx2" VOXTRAL_CPU_HAS_AVX2) - voxtral_check_cpu_supports("fma" VOXTRAL_CPU_HAS_FMA) - voxtral_check_cpu_supports("f16c" VOXTRAL_CPU_HAS_F16C) - voxtral_check_cpu_supports("bmi2" VOXTRAL_CPU_HAS_BMI2) - voxtral_check_cpu_supports("avx512f" VOXTRAL_CPU_HAS_AVX512) - - set(VOXTRAL_CPU_FEATURES_DETECTED ON) - endif() +if(VOXTRAL_USE_SYSTEM_GGML) + find_package(ggml CONFIG REQUIRED) + set(VOXTRAL_GGML_TARGETS ggml::ggml ggml::ggml-base) + # Imported targets omit the include dir with dynamically-loaded backends. + set(VOXTRAL_GGML_INCLUDES ${GGML_INCLUDE_DIR}) +else() + include(FetchContent) + set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + if(APPLE) + # Embed Metal shaders so the binary needs no external default.metallib. + set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "" FORCE) endif() + FetchContent_Declare(ggml + GIT_REPOSITORY https://github.com/ggml-org/ggml.git + GIT_TAG 5cecdad692d868e28dbd2f7c468504770108f30c) + FetchContent_MakeAvailable(ggml) + set(VOXTRAL_GGML_TARGETS ggml) + set(VOXTRAL_GGML_INCLUDES "") endif() -if(VOXTRAL_CPU_FEATURES_DETECTED) - # Use explicit ISA flags instead of -march=native when auto-detecting. - set(GGML_NATIVE OFF CACHE BOOL "ggml: optimize the build for the current system" FORCE) - set(GGML_AVX ${VOXTRAL_CPU_HAS_AVX} CACHE BOOL "ggml: enable AVX" FORCE) - set(GGML_AVX2 ${VOXTRAL_CPU_HAS_AVX2} CACHE BOOL "ggml: enable AVX2" FORCE) - set(GGML_FMA ${VOXTRAL_CPU_HAS_FMA} CACHE BOOL "ggml: enable FMA" FORCE) - set(GGML_F16C ${VOXTRAL_CPU_HAS_F16C} CACHE BOOL "ggml: enable F16C" FORCE) - set(GGML_BMI2 ${VOXTRAL_CPU_HAS_BMI2} CACHE BOOL "ggml: enable BMI2" FORCE) - set(GGML_AVX512 ${VOXTRAL_CPU_HAS_AVX512} CACHE BOOL "ggml: enable AVX512F" FORCE) -endif() - -if(VOXTRAL_AUTO_DETECT_BLAS) - if(NOT DEFINED CACHE{GGML_BLAS}) - find_package(BLAS QUIET) - if(BLAS_FOUND) - set(GGML_BLAS ON CACHE BOOL "ggml: use BLAS" FORCE) - endif() - endif() -endif() - -add_subdirectory(ggml) - add_library(voxtral_lib src/voxtral.cpp) +# On Windows with BUILD_SHARED_LIBS=ON, export symbols automatically so the +# import library (voxtral_lib.lib) is generated for dependents like `voxtral`. +if(WIN32 AND BUILD_SHARED_LIBS) + set_target_properties(voxtral_lib PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif() + target_include_directories(voxtral_lib PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/include) + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${VOXTRAL_GGML_INCLUDES}) target_link_libraries(voxtral_lib PUBLIC - ggml - Threads::Threads - m - dl) + ${VOXTRAL_GGML_TARGETS} + Threads::Threads) + +if(NOT WIN32) + target_link_libraries(voxtral_lib PUBLIC m) +endif() + +if(CMAKE_DL_LIBS) + target_link_libraries(voxtral_lib PUBLIC ${CMAKE_DL_LIBS}) +endif() if(VOXTRAL_WARNINGS_AS_ERRORS) if (MSVC) @@ -97,14 +77,27 @@ if(VOXTRAL_WARNINGS_AS_ERRORS) endif() if(VOXTRAL_NATIVE_OPT AND NOT MSVC) - target_compile_options(voxtral_lib PRIVATE -march=native -mtune=native) + if(APPLE) + message(STATUS "VOXTRAL_NATIVE_OPT: skipping -march/-mtune=native on Apple toolchains") + else() + target_compile_options(voxtral_lib PRIVATE -march=native -mtune=native) + endif() endif() add_executable(voxtral src/main.cpp) target_link_libraries(voxtral PRIVATE voxtral_lib) add_executable(voxtral-quantize src/voxtral-quantize.cpp) -target_link_libraries(voxtral-quantize PRIVATE ggml Threads::Threads m dl) +target_link_libraries(voxtral-quantize PRIVATE ${VOXTRAL_GGML_TARGETS} Threads::Threads) +target_include_directories(voxtral-quantize PRIVATE ${VOXTRAL_GGML_INCLUDES}) + +if(NOT WIN32) + target_link_libraries(voxtral-quantize PRIVATE m) +endif() + +if(CMAKE_DL_LIBS) + target_link_libraries(voxtral-quantize PRIVATE ${CMAKE_DL_LIBS}) +endif() if(VOXTRAL_WARNINGS_AS_ERRORS) if (MSVC) @@ -115,9 +108,14 @@ if(VOXTRAL_WARNINGS_AS_ERRORS) endif() if(VOXTRAL_NATIVE_OPT AND NOT MSVC) - target_compile_options(voxtral-quantize PRIVATE -march=native -mtune=native) + if(NOT APPLE) + target_compile_options(voxtral-quantize PRIVATE -march=native -mtune=native) + endif() endif() +# Install the CLI tools (used by `cmake --install` and the Homebrew formula). +install(TARGETS voxtral voxtral-quantize RUNTIME DESTINATION bin) + find_package(Python3 COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) add_custom_target(voxtral_convert diff --git a/Formula/voxtral.rb b/Formula/voxtral.rb new file mode 100644 index 0000000..25354c7 --- /dev/null +++ b/Formula/voxtral.rb @@ -0,0 +1,30 @@ +# Homebrew formula for voxtral.cpp. +# +# brew tap andrijdavid/voxtral https://github.com/andrijdavid/voxtral.cpp +# brew install --HEAD voxtral +# +# Builds against the system ggml (no bundled submodule), like the homebrew-core +# whisper-cpp formula. +class Voxtral < Formula + desc "C++ implementation of the Voxtral speech-to-text model (ggml)" + homepage "https://github.com/andrijdavid/voxtral.cpp" + license "MIT" + # Add `url` + `sha256` for a tagged release tarball; HEAD-only for now. + head "https://github.com/andrijdavid/voxtral.cpp.git", branch: "main" + + depends_on "cmake" => :build + depends_on "ggml" + + def install + system "cmake", "-S", ".", "-B", "build", + "-DVOXTRAL_USE_SYSTEM_GGML=ON", + "-DCMAKE_INSTALL_RPATH=#{rpath}", + *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + assert_match "usage", shell_output("#{bin}/voxtral --help") + end +end diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0af4d17 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Andrij David + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 651d4bc..4bda13e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,38 @@ -# voxtral.cpp +# Voxtral.cpp -A ggml-based C++ implementation of Voxtral Realtime 4B. +A ggml-based C++ implementation of Voxtral. + +## Models + +Two model families are supported: + +- [**Voxtral-Mini-4B-Realtime**](https://huggingface.co/andrijdavid/Voxtral-Mini-4B-Realtime-2602-GGUF) (`general.architecture = voxtral_realtime`) - streaming + model that emits one token per 80 ms audio frame. Built for low-latency/longform. +- [**Voxtral-Mini-3B-2507**](https://huggingface.co/andrijdavid/Voxtral-Mini-3B-2507-GGUF) (`general.architecture = voxtral`) - offline audio-LLM that + generates only the text tokens (Whisper-style), much faster for batch transcription of + short clips. Convert from the official `mistralai/Voxtral-Mini-3B-2507` checkpoint: + + ```bash + # needs consolidated.safetensors + params.json + tekken.json in the model dir + python tools/convert_voxtral_to_gguf.py --model-dir models/voxtral-3b \ + --output models/voxtral-3b/voxtral-3b.gguf --out-type q4_k_m + ./build/voxtral --model models/voxtral-3b/voxtral-3b.gguf --audio clip.wav + ``` + +## Voxtral References + +- Official Mistral Voxtral announcement: https://mistral.ai/news/voxtral +- Mistral Audio & Transcription docs: https://docs.mistral.ai/capabilities/audio_transcription +- Mistral Audio Transcriptions API: https://docs.mistral.ai/api/endpoint/audio/transcriptions + +## Model Weights (GGUF) + +Quantized GGUF weights used by this project are hosted on Hugging Face: + +- https://huggingface.co/andrijdavid/Voxtral-Mini-4B-Realtime-2602-GGUF +- https://huggingface.co/andrijdavid/Voxtral-Mini-3B-2507-GGUF + +The `download_model.sh` script downloads from that repository. ## Quickstart @@ -9,10 +41,14 @@ A ggml-based C++ implementation of Voxtral Realtime 4B. Download the pre-converted GGUF model from Hugging Face: ```bash -# Default: Q4_0 quantization -./tools/download_model.sh Q4_0 +# Recommended: Q4_K_M (best quality/size trade-off, ~2.7 GB) +./tools/download_model.sh Q4_K_M ``` +Other quants are available (`Q8_0`, `Q5_K`, `Q4_0`, `Q2_K`, …); `Q4_K_M` is the +recommended default and transcribes with no perceptible quality loss vs full +precision. + ### 2. Build Build the project using CMake: @@ -22,6 +58,13 @@ cmake -B build -DCMAKE_BUILD_TYPE=Release cmake --build build -j ``` +Or via Homebrew (`Formula/voxtral.rb`): + +```bash +brew tap andrijdavid/voxtral https://github.com/andrijdavid/voxtral.cpp +brew install --HEAD voxtral +``` + ### 3. Audio Preparation The model expects **16-bit PCM WAV** files at **16kHz (mono)**. You can use `ffmpeg` to convert your audio files: @@ -34,11 +77,15 @@ ffmpeg -i input.mp3 -ar 16000 -ac 1 -c:a pcm_s16le output.wav ```bash ./build/voxtral \ - --model models/voxtral/Q4_0.gguf \ - --audio path/to/input.wav \ - --threads 8 + --model models/voxtral/Q4_K_M.gguf \ + --audio path/to/input.wav ``` +By default the GPU is auto-detected (`--gpu auto`): **Metal** on Apple Silicon, +**CUDA** on NVIDIA, **Vulkan** otherwise, falling back to CPU. Force a backend +with `--gpu metal|cuda|vulkan|none`. Thread count for the CPU path defaults to +the machine's hardware concurrency; override with `--threads N`. + --- ## Advanced Usage @@ -55,6 +102,34 @@ You can quantize an existing GGUF file using the native quantizer: 8 ``` +### `voxtral-quantize` + +Command format: + +```bash +./build/voxtral-quantize [nthreads] +``` + +Examples: + +```bash +# 1) Quantize to Q4_0 using default thread count +./build/voxtral-quantize models/voxtral/voxtral.gguf models/voxtral/Q4_0.gguf Q4_0 + +# 2) Quantize to Q6_K using 8 threads +./build/voxtral-quantize models/voxtral/voxtral.gguf models/voxtral/Q6_K.gguf Q6_K 8 +``` + +Supported `type` values: + +`Q4_0`, `Q4_1`, `Q5_0`, `Q5_1`, `Q8_0`, `Q2_K`, `Q3_K`, `Q4_K`, `Q5_K`, `Q6_K`, `Q4_K_M` + +Notes: + +- Input must be a Voxtral GGUF (`general.architecture = voxtral_realtime`). +- `Q4_K_M` uses a mixed strategy internally (some tensors kept at higher precision). +- `nthreads` is optional; when omitted, hardware concurrency is used. + ## Testing The test suite runs over `samples/*.wav` files. @@ -72,4 +147,4 @@ python3 tests/test_voxtral_reference.py You can override comparison tolerances via environment variables: - `VOXTRAL_TEST_ATOL` (default: 1e-2) - `VOXTRAL_TEST_RTOL` (default: 1e-2) -- `VOXTRAL_TEST_THREADS` \ No newline at end of file +- `VOXTRAL_TEST_THREADS` diff --git a/docs/HOMEBREW.md b/docs/HOMEBREW.md new file mode 100644 index 0000000..5f243cd --- /dev/null +++ b/docs/HOMEBREW.md @@ -0,0 +1,95 @@ +# Homebrew packaging & homebrew-core plan + +Two tiers, modelled on `whisper-cpp` (which is in homebrew-core and ships `whisper-cli`). + +## Tier 1 — personal tap (works today) +`Formula/voxtral.rb` is a git-based formula that builds the bundled ggml submodule +into a static, self-contained binary. Publish by pointing a tap at this repo: + +```bash +brew tap andrijdavid/voxtral https://github.com/andrijdavid/voxtral.cpp +brew install --HEAD voxtral +``` + +Good enough for users now; **not** acceptable for homebrew-core (see below). + +## Tier 2 — homebrew-core + +homebrew-core has hard rules. Status of each for voxtral.cpp: + +| Requirement | whisper-cpp does | voxtral status | +| --- | --- | --- | +| **No bundled/vendored deps** ("reject PRs that bundle ggml") | `depends_on "ggml"`, `-DWHISPER_USE_SYSTEM_GGML=ON` | ❌ bundles ggml as a submodule — **needs the backend-registry migration below** | +| Stable versioned release, tarball + `sha256` (no git/HEAD, no submodule fetch — the build sandbox has **no network**) | `archive/refs/tags/vX.tar.gz` | ❌ no tagged release yet | +| Builds shared, sets `-DCMAKE_INSTALL_RPATH=#{rpath}` | yes | n/a until system-ggml build exists | +| Meaningful `test do` | downloads a tiny model + transcribes `jfk.wav` | partial (`--help`); want a real run | +| Recognised SPDX license | MIT | ✅ MIT | +| **Notability** (maintained, stable, widely used; not a personal fork) | upstream ggml-org project | ⚠️ **the real blocker** — a new personal repo will likely be rejected | + +### Prerequisite code change: migrate to the ggml backend **registry** API +This is the gating technical item, and it's why voxtral can't yet use system ggml. + +- The Homebrew `ggml` formula (0.15.x) exports only the **registry** API + (`ggml_backend_reg_count`, `ggml_backend_dev_by_type`, `ggml_backend_dev_init`). + It does **not** export the direct `ggml_backend_cpu_init` / `ggml_backend_metal_init` / + `ggml_backend_blas_init` / `ggml_backend_cpu_set_n_threads` symbols voxtral calls — + those live in dynamically-loaded backend modules. +- whisper.cpp/llama.cpp work with system ggml precisely because they select backends + via the registry, not direct init. +- Good news (verified): voxtral's headers/API usage **compile cleanly against ggml + 0.15.1** — only the backend *init* path needs porting. + +Migration (in `src/voxtral.cpp`, `voxtral_init_from_model` + `voxtral_model_load_from_file`): +- GPU: `ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU)` → `ggml_backend_dev_init(dev, NULL)` + (replaces `ggml_backend_metal_init` / `cuda_init` / `vk_init`). +- CPU: `ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU)` → `ggml_backend_dev_init` + (replaces `ggml_backend_cpu_init`). +- Threads: `ggml_backend_set_n_threads(backend, n)` via the generic setter + (replaces `ggml_backend_cpu_set_n_threads` / `ggml_backend_blas_set_n_threads`). +- Drop the `GGML_USE_*` `#ifdef` gating around init (the registry discovers backends at + runtime). Keep it working for the bundled build too — this is a net simplification. + +Then add the CMake switch (was prototyped, reverted until the migration lands): +```cmake +option(VOXTRAL_USE_SYSTEM_GGML "Link a system ggml instead of the bundled submodule" OFF) +# when ON: find_package(ggml CONFIG REQUIRED); link ggml::ggml ggml::ggml-base +``` + +### Target formula (whisper-cpp style) +```ruby +class Voxtral < Formula + desc "C++ implementation of the Voxtral speech-to-text model (ggml)" + homepage "https://github.com/andrijdavid/voxtral.cpp" + url "https://github.com/andrijdavid/voxtral.cpp/archive/refs/tags/v0.1.0.tar.gz" + sha256 "..." + license "MIT" + head "https://github.com/andrijdavid/voxtral.cpp.git", branch: "main" + + depends_on "cmake" => :build + depends_on "ggml" + # depends_on "sdl2" # only once mic streaming (examples/stream) is added + + def install + system "cmake", "-S", ".", "-B", "build", + "-DVOXTRAL_USE_SYSTEM_GGML=ON", + "-DBUILD_SHARED_LIBS=ON", + "-DCMAKE_INSTALL_RPATH=#{rpath}", + *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + assert_match "usage", shell_output("#{bin}/voxtral --help") + end +end +``` + +## Recommended path +1. Land the backend-registry migration + `VOXTRAL_USE_SYSTEM_GGML`; verify a system-ggml + build transcribes a sample on Metal/CPU. +2. Ship Tier-1 tap; let usage/stars accrue (helps the notability case). +3. Tag `v0.1.0`, compute the tarball sha256, and only then open the homebrew-core PR + with `brew audit --new --strict voxtral` passing. +4. Be ready for the notability objection — a tap is the realistic home until the project + is demonstrably used; consider an org-owned repo name. diff --git a/ggml b/ggml deleted file mode 160000 index 5cecdad..0000000 --- a/ggml +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5cecdad692d868e28dbd2f7c468504770108f30c diff --git a/include/voxtral.h b/include/voxtral.h index d4e587a..627f545 100644 --- a/include/voxtral.h +++ b/include/voxtral.h @@ -80,6 +80,14 @@ enum class voxtral_log_level : int { debug = 3, }; +enum class voxtral_gpu_backend : int { + none = 0, + auto_detect, + cuda, + metal, + vulkan, +}; + using voxtral_log_callback = std::function; // ============================================================================ @@ -96,7 +104,7 @@ struct voxtral_context_params { int32_t n_threads = 0; voxtral_log_level log_level = voxtral_log_level::info; voxtral_log_callback logger = nullptr; - bool use_metal = false; + voxtral_gpu_backend gpu = voxtral_gpu_backend::none; }; // ============================================================================ @@ -122,7 +130,7 @@ struct voxtral_context; voxtral_model * voxtral_model_load_from_file( const std::string & path, voxtral_log_callback logger = nullptr, - bool use_metal = false); + voxtral_gpu_backend gpu = voxtral_gpu_backend::none); void voxtral_model_free(voxtral_model * model); diff --git a/src/main.cpp b/src/main.cpp index e17cd6f..0c6a96d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,9 @@ #include "voxtral.h" +#include +#include +#include +#include #include #include #include @@ -20,11 +24,175 @@ struct cli_params { std::string output_text; int32_t threads = 0; uint32_t seed = 0; - int32_t max_tokens = 256; + int32_t max_tokens = 0; // 0 = decode whole file (until end of audio / EOS) voxtral_log_level log_level = voxtral_log_level::info; - bool use_metal = false; + voxtral_gpu_backend gpu = voxtral_gpu_backend::auto_detect; }; +struct backend_reg_info { + std::string name; + size_t devices = 0; +}; + +std::string to_lower_copy(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return s; +} + +bool has_runtime_backend(const std::vector & regs, const std::string & name) { + const std::string name_lc = to_lower_copy(name); + for (const auto & reg : regs) { + if (to_lower_copy(reg.name) == name_lc) { + return reg.devices > 0; + } + } + return false; +} + +std::vector collect_runtime_backends() { + std::vector regs; + const size_t n_reg = ggml_backend_reg_count(); + regs.reserve(n_reg); + + for (size_t i = 0; i < n_reg; ++i) { + ggml_backend_reg_t reg = ggml_backend_reg_get(i); + if (!reg) { + continue; + } + + const char * name = ggml_backend_reg_name(reg); + regs.push_back({ + name ? name : "unknown", + ggml_backend_reg_dev_count(reg), + }); + } + + return regs; +} + +std::string format_build_backend_flags() { + std::ostringstream os; + bool first = true; + auto append_flag = [&](const char * name, bool enabled) { + if (!first) { + os << ", "; + } + os << name << '=' << (enabled ? "ON" : "OFF"); + first = false; + }; + +#ifdef GGML_USE_CPU + append_flag("GGML_USE_CPU", true); +#else + append_flag("GGML_USE_CPU", false); +#endif +#ifdef GGML_USE_BLAS + append_flag("GGML_USE_BLAS", true); +#else + append_flag("GGML_USE_BLAS", false); +#endif +#ifdef GGML_USE_METAL + append_flag("GGML_USE_METAL", true); +#else + append_flag("GGML_USE_METAL", false); +#endif +#ifdef GGML_USE_CUDA + append_flag("GGML_USE_CUDA", true); +#else + append_flag("GGML_USE_CUDA", false); +#endif +#ifdef GGML_USE_VULKAN + append_flag("GGML_USE_VULKAN", true); +#else + append_flag("GGML_USE_VULKAN", false); +#endif +#ifdef GGML_USE_SYCL + append_flag("GGML_USE_SYCL", true); +#else + append_flag("GGML_USE_SYCL", false); +#endif +#ifdef GGML_USE_OPENCL + append_flag("GGML_USE_OPENCL", true); +#else + append_flag("GGML_USE_OPENCL", false); +#endif +#ifdef GGML_USE_CANN + append_flag("GGML_USE_CANN", true); +#else + append_flag("GGML_USE_CANN", false); +#endif +#ifdef GGML_USE_WEBGPU + append_flag("GGML_USE_WEBGPU", true); +#else + append_flag("GGML_USE_WEBGPU", false); +#endif +#ifdef GGML_USE_RPC + append_flag("GGML_USE_RPC", true); +#else + append_flag("GGML_USE_RPC", false); +#endif +#ifdef GGML_USE_ZDNN + append_flag("GGML_USE_ZDNN", true); +#else + append_flag("GGML_USE_ZDNN", false); +#endif +#ifdef GGML_USE_ZENDNN + append_flag("GGML_USE_ZENDNN", true); +#else + append_flag("GGML_USE_ZENDNN", false); +#endif +#ifdef GGML_USE_HEXAGON + append_flag("GGML_USE_HEXAGON", true); +#else + append_flag("GGML_USE_HEXAGON", false); +#endif +#ifdef GGML_USE_VIRTGPU_FRONTEND + append_flag("GGML_USE_VIRTGPU_FRONTEND", true); +#else + append_flag("GGML_USE_VIRTGPU_FRONTEND", false); +#endif + + return os.str(); +} + +std::string format_runtime_backends(const std::vector & regs) { + if (regs.empty()) { + return "none"; + } + + std::ostringstream os; + for (size_t i = 0; i < regs.size(); ++i) { + if (i > 0) { + os << ", "; + } + os << regs[i].name << '(' << regs[i].devices << " dev"; + if (regs[i].devices != 1) { + os << 's'; + } + os << ')'; + } + return os.str(); +} + +std::string format_runtime_availability(const std::vector & regs) { + static const std::array known_backends = { + "CPU", "BLAS", "METAL", "CUDA", "VULKAN", "SYCL", "OPENCL", + "CANN", "WEBGPU", "RPC", "ZDNN", "ZENDNN", "HEXAGON", "VIRTGPU", + }; + + std::ostringstream os; + for (size_t i = 0; i < known_backends.size(); ++i) { + if (i > 0) { + os << ", "; + } + os << known_backends[i] << '=' + << (has_runtime_backend(regs, known_backends[i]) ? "yes" : "no"); + } + return os.str(); +} + void print_usage(const char * argv0) { std::cout << "usage: " << argv0 << " --model path.gguf --audio file.wav [options]\n" @@ -36,14 +204,15 @@ void print_usage(const char * argv0) { << " --seed N rng seed (reserved for sampling)\n" << " --prompt TEXT prompt text (compatibility, currently ignored for realtime mode)\n" << " --n-tokens N max decode tokens (alias of --max-len)\n" - << " --max-len N max decode tokens\n" + << " --max-len N max decode tokens (0 = whole file, default)\n" << " --verbose equivalent to --log-level debug\n" << " --log-level LEVEL error|warn|info|debug\n" << " --dump-logits PATH write step-0 logits (first 32 values) as text\n" << " --dump-logits-bin P write full step-0 logits as float32 raw bytes\n" << " --dump-tokens PATH write generated token ids as a single line\n" << " --output-text PATH write decoded text to file (still prints to stdout)\n" - << " --metal use Metal backend when available\n" + << " --gpu BACKEND gpu backend: auto|cuda|metal|vulkan|none (default: auto)\n" + << " --metal alias for --gpu metal\n" << " -h, --help show this help\n"; } @@ -67,6 +236,16 @@ bool parse_u32(const std::string & s, uint32_t & out) { return true; } +bool parse_gpu(const std::string & s, voxtral_gpu_backend & out) { + const std::string lc = to_lower_copy(s); + if (lc == "none") { out = voxtral_gpu_backend::none; return true; } + if (lc == "auto") { out = voxtral_gpu_backend::auto_detect; return true; } + if (lc == "cuda") { out = voxtral_gpu_backend::cuda; return true; } + if (lc == "metal") { out = voxtral_gpu_backend::metal; return true; } + if (lc == "vulkan") { out = voxtral_gpu_backend::vulkan; return true; } + return false; +} + bool parse_level(const std::string & s, voxtral_log_level & out) { if (s == "error") { out = voxtral_log_level::error; @@ -170,8 +349,14 @@ bool parse_args(int argc, char ** argv, cli_params & p) { return false; } p.output_text = v; + } else if (a == "--gpu") { + const char * v = need_value("--gpu"); + if (!v || !parse_gpu(v, p.gpu)) { + std::cerr << "invalid --gpu (expected: auto|cuda|metal|vulkan|none)\n"; + return false; + } } else if (a == "--metal") { - p.use_metal = true; + p.gpu = voxtral_gpu_backend::metal; } else { std::cerr << "unknown option: " << a << "\n"; return false; @@ -188,10 +373,6 @@ bool parse_args(int argc, char ** argv, cli_params & p) { return false; } - if (p.max_tokens <= 0) { - p.max_tokens = 1; - } - return true; } @@ -204,6 +385,20 @@ int main(int argc, char ** argv) { return 1; } + const auto t_start = std::chrono::steady_clock::now(); + auto finish = [&](int exit_code) { + const auto elapsed_ms = std::chrono::duration( + std::chrono::steady_clock::now() - t_start).count(); + const std::vector runtime_backends = collect_runtime_backends(); + + std::cerr << std::fixed << std::setprecision(2); + std::cerr << "[summary] processing_time_ms=" << elapsed_ms << "\n"; + std::cerr << "[summary] build_backend_flags: " << format_build_backend_flags() << "\n"; + std::cerr << "[summary] runtime_backends: " << format_runtime_backends(runtime_backends) << "\n"; + std::cerr << "[summary] runtime_available: " << format_runtime_availability(runtime_backends) << "\n"; + return exit_code; + }; + voxtral_log_callback logger = [level = p.log_level](voxtral_log_level msg_level, const std::string & msg) { if (static_cast(msg_level) > static_cast(level)) { return; @@ -221,9 +416,9 @@ int main(int argc, char ** argv) { std::cerr << "voxtral_" << tag << ": " << msg << "\n"; }; - voxtral_model * model = voxtral_model_load_from_file(p.model, logger, p.use_metal); + voxtral_model * model = voxtral_model_load_from_file(p.model, logger, p.gpu); if (!model) { - return 2; + return finish(2); } voxtral_context_params ctx_p; @@ -231,19 +426,19 @@ int main(int argc, char ** argv) { // ctx_p.seed = p.seed; ctx_p.log_level = p.log_level; ctx_p.logger = logger; - ctx_p.use_metal = p.use_metal; + ctx_p.gpu = p.gpu; voxtral_context * ctx = voxtral_init_from_model(model, ctx_p); if (!ctx) { voxtral_model_free(model); - return 3; + return finish(3); } voxtral_result result; if (!voxtral_transcribe_file(*ctx, p.audio, p.max_tokens, result)) { voxtral_free(ctx); voxtral_model_free(model); - return 4; + return finish(4); } const std::string printed_text = result.text.empty() ? std::string("[no-transcript]") : result.text; @@ -312,5 +507,5 @@ int main(int argc, char ** argv) { voxtral_free(ctx); voxtral_model_free(model); - return 0; + return finish(0); } diff --git a/src/voxtral-quantize.cpp b/src/voxtral-quantize.cpp index 8ea9e58..8616011 100644 --- a/src/voxtral-quantize.cpp +++ b/src/voxtral-quantize.cpp @@ -345,9 +345,10 @@ int quantize_voxtral_gguf( } const std::string arch = get_kv_str(gguf_in, "general.architecture"); - if (arch != kVoxtralArch) { + if (arch != kVoxtralArch && arch != "voxtral") { throw std::runtime_error( - "unsupported architecture '" + arch + "', expected '" + std::string(kVoxtralArch) + "'"); + "unsupported architecture '" + arch + "', expected '" + + std::string(kVoxtralArch) + "' or 'voxtral'"); } gguf_out = gguf_init_empty(); diff --git a/src/voxtral.cpp b/src/voxtral.cpp index e47f9ce..f8260ca 100644 --- a/src/voxtral.cpp +++ b/src/voxtral.cpp @@ -1,14 +1,10 @@ #include "voxtral.h" #include "gguf.h" -#include "ggml-cpu.h" -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - #include #include #include #include +#include #include #include #include @@ -17,6 +13,7 @@ #include #include #include +#include #include // ============================================================================ @@ -25,10 +22,10 @@ static constexpr float VOXTRAL_PI = 3.14159265358979323846f; static constexpr int32_t VOXTRAL_N_FFT = VOXTRAL_WINDOW_SIZE; // 400 -static constexpr int32_t VOXTRAL_N_FREQ = VOXTRAL_N_FFT / 2 + 1; // 201 -static constexpr int32_t VOXTRAL_MAX_MEL_FRAMES = 4000; -static constexpr int32_t VOXTRAL_MAX_ENC_SEQ = VOXTRAL_MAX_MEL_FRAMES / 2; // after conv stride-2 -static constexpr int32_t VOXTRAL_MAX_DEC_SEQ = VOXTRAL_MAX_ENC_SEQ / VOXTRAL_DOWNSAMPLE_FACTOR; +static constexpr int32_t VOXTRAL_N_FREQ = VOXTRAL_N_FFT / 2 + 1; +static constexpr int32_t VOXTRAL_ENC_CHUNK_MEL = 3000; // mel frames per encoder chunk +static constexpr int32_t VOXTRAL_ENC_CHUNK_OVERLAP = 750; // overlap in encoder-token space (= window) +static constexpr int32_t VOXTRAL_MAX_ENC_CHUNK = 2000; // max enc tokens per single chunk // ============================================================================ // Logging helper @@ -54,6 +51,7 @@ static constexpr int32_t VOXTRAL_MAX_DEC_SEQ = VOXTRAL_MAX_ENC_SEQ / VOXTRAL_DOW struct voxtral_encoder_layer { ggml_tensor * attn_norm_weight; // [enc_dim] + ggml_tensor * attn_norm_bias = nullptr; // [enc_dim] — offline LayerNorm only ggml_tensor * attn_q_weight; // [enc_heads*enc_head_dim, enc_dim] ggml_tensor * attn_q_bias; // [enc_heads*enc_head_dim] ggml_tensor * attn_k_weight; // [enc_kv_heads*enc_head_dim, enc_dim] @@ -62,10 +60,12 @@ struct voxtral_encoder_layer { ggml_tensor * attn_o_weight; // [enc_dim, enc_heads*enc_head_dim] ggml_tensor * attn_o_bias; // [enc_dim] ggml_tensor * ffn_norm_weight; // [enc_dim] + ggml_tensor * ffn_norm_bias = nullptr; // [enc_dim] — offline LayerNorm only ggml_tensor * ffn_w1_weight; // [enc_hidden, enc_dim] + ggml_tensor * ffn_w1_bias = nullptr; // [enc_hidden] - offline GELU MLP only ggml_tensor * ffn_w2_weight; // [enc_dim, enc_hidden] ggml_tensor * ffn_w2_bias; // [enc_dim] - ggml_tensor * ffn_w3_weight; // [enc_hidden, enc_dim] + ggml_tensor * ffn_w3_weight = nullptr; // [enc_hidden, enc_dim] — realtime SwiGLU only }; struct voxtral_decoder_layer { @@ -86,7 +86,50 @@ struct voxtral_decoder_layer { // Model structure // ============================================================================ +// Runtime model hyperparameters from GGUF metadata at load time with the VOXTRAL_* as fallbacks. +struct voxtral_hparams { + bool is_offline = false; // general.architecture == "voxtral" (offline) vs "voxtral_realtime" + + // Encoder + int32_t enc_dim = VOXTRAL_ENC_DIM; + int32_t enc_layers = VOXTRAL_ENC_LAYERS; + int32_t enc_heads = VOXTRAL_ENC_HEADS; + int32_t enc_head_dim = VOXTRAL_ENC_HEAD_DIM; + int32_t enc_hidden = VOXTRAL_ENC_HIDDEN; + int32_t enc_kv_heads = VOXTRAL_ENC_KV_HEADS; + bool enc_causal = true; // realtime: causal+window; offline: full bidirectional + float enc_norm_eps = VOXTRAL_ENC_NORM_EPS; + float enc_rope_theta = VOXTRAL_ENC_ROPE_THETA; + + // Decoder + int32_t dec_dim = VOXTRAL_DEC_DIM; + int32_t dec_layers = VOXTRAL_DEC_LAYERS; + int32_t dec_heads = VOXTRAL_DEC_HEADS; + int32_t dec_head_dim = VOXTRAL_DEC_HEAD_DIM; + int32_t dec_hidden = VOXTRAL_DEC_HIDDEN; + int32_t dec_kv_heads = VOXTRAL_DEC_KV_HEADS; + float dec_norm_eps = VOXTRAL_DEC_NORM_EPS; + float dec_rope_theta = VOXTRAL_DEC_ROPE_THETA; + bool ada_t_cond = true; // realtime adaptive RMS norm time-conditioning + + int32_t vocab_size = VOXTRAL_VOCAB_SIZE; + + // Audio + int32_t downsample_factor = VOXTRAL_DOWNSAMPLE_FACTOR; + + // Special tokens + int32_t tok_bos = VOXTRAL_TOKEN_BOS; + int32_t tok_eos = VOXTRAL_TOKEN_EOS; + int32_t tok_audio = VOXTRAL_TOKEN_AUDIO; + int32_t tok_begin_audio = VOXTRAL_TOKEN_BEGIN_AUDIO; + int32_t tok_transcribe = 34; // [TRANSCRIBE]; offline prompt only + int32_t tok_inst = 3; // [INST] + int32_t tok_inst_end = 4; // [/INST] +}; + struct voxtral_model { + voxtral_hparams hp; + // Encoder conv stem ggml_tensor * enc_conv0_weight; // [enc_dim, num_mel_bins, 3] ggml_tensor * enc_conv0_bias; // [enc_dim] @@ -94,6 +137,9 @@ struct voxtral_model { ggml_tensor * enc_conv1_bias; // [enc_dim] std::vector enc_layers; ggml_tensor * enc_norm_weight; // [enc_dim] + ggml_tensor * enc_norm_bias = nullptr; // [enc_dim] — offline LayerNorm only + ggml_tensor * enc_pos_embedding = nullptr; // [enc_dim, max_pos] — offline Whisper sinusoids + ggml_tensor * output_weight = nullptr; // [vocab, dec_dim] — offline untied output proj // Adapter ggml_tensor * adapter_0_weight; // [dec_dim, enc_dim*downsample] @@ -118,7 +164,8 @@ struct voxtral_model { gguf_context * gguf_ctx = nullptr; ggml_backend_buffer_t buf_weights = nullptr; ggml_backend_t backend_weights = nullptr; - bool weights_on_metal = false; + bool weights_on_gpu = false; + voxtral_gpu_backend gpu_type = voxtral_gpu_backend::none; }; // ============================================================================ @@ -134,20 +181,33 @@ struct voxtral_context { // Backend ggml_backend_t backend = nullptr; ggml_backend_t backend_cpu = nullptr; - bool backend_is_cpu = true; + ggml_backend_t blas_backend = nullptr; + voxtral_gpu_backend gpu_type = voxtral_gpu_backend::none; // Persistent device tensors (allocated once) - ggml_context * ctx_persistent = nullptr; - ggml_backend_buffer_t buf_persistent = nullptr; + ggml_context * ctx_persistent = nullptr; + ggml_backend_buffer_t buf_persistent = nullptr; - ggml_tensor * encoder_output = nullptr; // [enc_dim, max_enc_seq] - ggml_tensor * decoder_memory = nullptr; // [dec_dim, max_dec_seq] + // Per-chunk encoder output (fixed size, reused each chunk) + ggml_tensor * encoder_chunk_output = nullptr; // [enc_dim, MAX_ENC_CHUNK] ggml_tensor * decoder_logits = nullptr; // [vocab_size] + ggml_tensor * decoder_argmax = nullptr; // [1] i32 — greedy token, computed on device // KV cache: [kv_heads*head_dim, dec_window, dec_layers] ggml_tensor * kv_self_k = nullptr; ggml_tensor * kv_self_v = nullptr; + // Full accumulated encoder output (dynamic, allocated per utterance ON DEVICE) + ggml_context * ctx_enc_full = nullptr; + ggml_backend_buffer_t buf_enc_full = nullptr; + ggml_tensor * encoder_output = nullptr; // [enc_dim, total_enc_tokens] + int32_t total_enc_tokens = 0; + + // Dynamic decoder memory (allocated per utterance ON DEVICE) + ggml_context * ctx_dec_mem = nullptr; + ggml_backend_buffer_t buf_dec_mem = nullptr; + ggml_tensor * decoder_memory = nullptr; // [dec_dim, dec_seq] + // Actual sizes (set per utterance) int32_t enc_seq_len = 0; // after conv, before left-trunc int32_t enc_seq_used = 0; // after left-trunc (multiple of downsample_factor) @@ -169,7 +229,7 @@ struct voxtral_context { }; // ============================================================================ -// Mel filterbank computation (Slaney-style, matches Python reference) +// Mel filterbank computation (Slaney-style) // ============================================================================ static float hertz_to_mel(float freq_hz) { @@ -195,20 +255,16 @@ static float mel_to_hertz(float mels) { } static void compute_mel_filters_slaney(std::vector & filters) { - // Output: filters[k * n_mel + m] for k in [0..n_freq), m in [0..n_mel) - // Matches Python compute_mel_filters() exactly - constexpr int32_t n_freq = VOXTRAL_N_FREQ; // 201 - constexpr int32_t n_mel = VOXTRAL_NUM_MEL_BINS; // 128 + constexpr int32_t n_freq = VOXTRAL_N_FREQ; + constexpr int32_t n_mel = VOXTRAL_NUM_MEL_BINS; filters.resize(n_freq * n_mel, 0.0f); - // FFT frequencies: linspace(0, sr/2, n_freq) std::vector fft_freqs(n_freq); for (int32_t i = 0; i < n_freq; i++) { fft_freqs[i] = (float)(VOXTRAL_SAMPLE_RATE / 2) * (float)i / (float)(n_freq - 1); } - // Mel frequencies: linspace(mel(0), mel(8000), n_mel+2) const float mel_min = hertz_to_mel(0.0f); const float mel_max = hertz_to_mel(8000.0f); @@ -222,7 +278,7 @@ static void compute_mel_filters_slaney(std::vector & filters) { filter_freqs[i] = mel_to_hertz(mel_pts[i]); } - // Build triangular filters (matching Python slopes approach) + // Build triangular filters for (int32_t m = 0; m < n_mel; m++) { const float f_left = filter_freqs[m]; const float f_center = filter_freqs[m + 1]; @@ -589,15 +645,106 @@ static ggml_tensor * get_tensor(ggml_context * ctx, const char * name) { return t; } +// Optional tensor: returns nullptr without warning if absent (arch-dependent tensors). +static ggml_tensor * get_tensor_opt(ggml_context * ctx, const char * name) { + return ggml_get_tensor(ctx, name); +} + +// ============================================================================ +// Backend selection via the ggml backend registry // ============================================================================ + +// Case-insensitive substring test (registry names vary by ggml version/build, +// e.g. the Metal backend is "MTL" in the bundled build but "Metal" elsewhere). +static bool name_has(const char * hay, const char * needle_lc) { + if (!hay) return false; + std::string h(hay); + for (char & c : h) if (c >= 'A' && c <= 'Z') c += 32; + return h.find(needle_lc) != std::string::npos; +} + +// Does a registry name belong to the requested GPU family? (auto matches any GPU.) +static bool gpu_reg_matches(const char * rn, voxtral_gpu_backend req) { + switch (req) { + case voxtral_gpu_backend::metal: return name_has(rn, "metal") || name_has(rn, "mtl"); + case voxtral_gpu_backend::cuda: return name_has(rn, "cuda"); + case voxtral_gpu_backend::vulkan: return name_has(rn, "vulkan") || name_has(rn, "vk"); + default: return true; // auto_detect + } +} + +// Map a registry name back to our backend enum. +static voxtral_gpu_backend gpu_from_reg(const char * rn) { + if (name_has(rn, "cuda")) return voxtral_gpu_backend::cuda; + if (name_has(rn, "vulkan") || name_has(rn, "vk")) return voxtral_gpu_backend::vulkan; + return voxtral_gpu_backend::metal; // metal/mtl, or a generic GPU +} + +// First GPU/IGPU device matching the requested family (any GPU for auto_detect). +static ggml_backend_dev_t find_gpu_device(voxtral_gpu_backend req) { + const size_t n = ggml_backend_dev_count(); + for (size_t i = 0; i < n; ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + const enum ggml_backend_dev_type t = ggml_backend_dev_type(dev); + if (t != GGML_BACKEND_DEVICE_TYPE_GPU && t != GGML_BACKEND_DEVICE_TYPE_IGPU) { + continue; + } + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev); + if (gpu_reg_matches(reg ? ggml_backend_reg_name(reg) : "", req)) { + return dev; + } + } + return nullptr; +} + +// Find an accelerator device (e.g. BLAS/Accelerate) used alongside the CPU. +static ggml_backend_dev_t find_accel_device() { + const size_t n = ggml_backend_dev_count(); + for (size_t i = 0; i < n; ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) { + return dev; + } + } + return nullptr; +} + +// Set the thread count on a backend via the registry proc-address (no-op if the +// backend does not support it, e.g. GPU backends). +static void backend_set_threads(ggml_backend_t backend, int n_threads) { + if (!backend) return; + ggml_backend_dev_t dev = ggml_backend_get_device(backend); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (!reg) return; + auto set_threads = (ggml_backend_set_n_threads_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + if (set_threads) set_threads(backend, n_threads); +} + +// Initialize the CPU backend via the registry. +static ggml_backend_t init_cpu_backend() { + ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + return dev ? ggml_backend_dev_init(dev, nullptr) : nullptr; +} + +// Load all dynamically-linked ggml backend modules into the registry, exactly +// once. Required when linking a system ggml (its backends are separate modules); +// a no-op for the bundled static build whose backends self-register. +static void ensure_backends_loaded() { + static const bool loaded = [] { ggml_backend_load_all(); return true; }(); + (void) loaded; +} + +// ========================================================================== // Model loading // ============================================================================ voxtral_model * voxtral_model_load_from_file( const std::string & path, voxtral_log_callback logger, - bool use_metal) + voxtral_gpu_backend gpu) { + ensure_backends_loaded(); auto log_info = [&](const std::string & msg) { if (logger) logger(voxtral_log_level::info, msg); }; @@ -621,27 +768,86 @@ voxtral_model * voxtral_model_load_from_file( model->gguf_ctx = gguf_ctx; model->ctx_gguf = ctx_meta; - // Allocate a backend buffer for all the weights + // ---- Populate runtime hyperparameters from GGUF metadata (fallback to defaults) ---- + { + voxtral_hparams & hp = model->hp; + auto gi32 = [&](const char * k, int32_t & dst) { + const int64_t kid = gguf_find_key(gguf_ctx, k); + if (kid >= 0) dst = gguf_get_val_i32(gguf_ctx, kid); + }; + auto gf32 = [&](const char * k, float & dst) { + const int64_t kid = gguf_find_key(gguf_ctx, k); + if (kid >= 0) dst = gguf_get_val_f32(gguf_ctx, kid); + }; + auto gbool = [&](const char * k, bool & dst) { + const int64_t kid = gguf_find_key(gguf_ctx, k); + if (kid >= 0) dst = gguf_get_val_bool(gguf_ctx, kid); + }; + + const int64_t arch_kid = gguf_find_key(gguf_ctx, "general.architecture"); + if (arch_kid >= 0) { + const std::string arch = gguf_get_val_str(gguf_ctx, arch_kid); + hp.is_offline = (arch == "voxtral"); // offline arch string + } + + gi32("voxtral.encoder.dim", hp.enc_dim); + gi32("voxtral.encoder.n_layers", hp.enc_layers); + gi32("voxtral.encoder.n_heads", hp.enc_heads); + gi32("voxtral.encoder.head_dim", hp.enc_head_dim); + gi32("voxtral.encoder.hidden_dim", hp.enc_hidden); + gi32("voxtral.encoder.n_kv_heads", hp.enc_kv_heads); + gf32("voxtral.encoder.norm_eps", hp.enc_norm_eps); + gf32("voxtral.encoder.rope_theta", hp.enc_rope_theta); + // encoder causality: explicit key if present, else from params.*, else infer from window + hp.enc_causal = !hp.is_offline; + gbool("voxtral.encoder.causal", hp.enc_causal); + gbool("voxtral.params.multimodal.whisper_model_args.encoder_args.causal", hp.enc_causal); + + gi32("voxtral.decoder.dim", hp.dec_dim); + gi32("voxtral.decoder.n_layers", hp.dec_layers); + gi32("voxtral.decoder.n_heads", hp.dec_heads); + gi32("voxtral.decoder.head_dim", hp.dec_head_dim); + gi32("voxtral.decoder.hidden_dim", hp.dec_hidden); + gi32("voxtral.decoder.n_kv_heads", hp.dec_kv_heads); + gf32("voxtral.decoder.norm_eps", hp.dec_norm_eps); + gf32("voxtral.decoder.rope_theta", hp.dec_rope_theta); + + hp.ada_t_cond = !hp.is_offline; + gbool("voxtral.ada_rms_norm_t_cond", hp.ada_t_cond); + + gi32("voxtral.vocab_size", hp.vocab_size); + gi32("voxtral.audio.downsample_factor", hp.downsample_factor); + gi32("voxtral.token.bos", hp.tok_bos); + gi32("voxtral.token.eos", hp.tok_eos); + gi32("voxtral.token.audio", hp.tok_audio); + gi32("voxtral.token.begin_audio", hp.tok_begin_audio); + } + + // Allocate a backend buffer for all the weights. The GPU backend (if any) is + // selected from the ggml backend registry; weights are allocated on it. ggml_backend_t weights_backend = nullptr; -#ifdef GGML_USE_METAL - if (use_metal) { - weights_backend = ggml_backend_metal_init(); - if (!weights_backend) { - fprintf(stderr, "voxtral: ggml_backend_metal_init() failed, falling back to CPU\n"); + voxtral_gpu_backend resolved_gpu = voxtral_gpu_backend::none; + + if (gpu != voxtral_gpu_backend::none) { + ggml_backend_dev_t gdev = find_gpu_device(gpu); + if (gdev) { + weights_backend = ggml_backend_dev_init(gdev, nullptr); + } + if (weights_backend) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(gdev); + resolved_gpu = gpu_from_reg(reg ? ggml_backend_reg_name(reg) : ""); + } else { + log_info("no GPU backend available, using CPU"); } } -#else - if (use_metal) { - fprintf(stderr, "voxtral: Metal backend not available in this build, using CPU\n"); - } -#endif + if (!weights_backend) { - weights_backend = ggml_backend_cpu_init(); - use_metal = false; + weights_backend = init_cpu_backend(); } model->backend_weights = weights_backend; - model->weights_on_metal = use_metal; + model->weights_on_gpu = (resolved_gpu != voxtral_gpu_backend::none); + model->gpu_type = resolved_gpu; model->buf_weights = ggml_backend_alloc_ctx_tensors(ctx_meta, weights_backend); if (!model->buf_weights) { @@ -693,9 +899,12 @@ voxtral_model * voxtral_model_load_from_file( model->enc_conv1_weight = get_tensor(ctx_meta, "enc.conv1.weight"); model->enc_conv1_bias = get_tensor(ctx_meta, "enc.conv1.bias"); model->enc_norm_weight = get_tensor(ctx_meta, "enc.norm.weight"); + model->enc_norm_bias = get_tensor_opt(ctx_meta, "enc.norm.bias"); // offline LayerNorm + model->enc_pos_embedding = get_tensor_opt(ctx_meta, "enc.pos_embedding"); // offline Whisper sinusoids + model->output_weight = get_tensor_opt(ctx_meta, "output.weight"); // offline untied output - model->enc_layers.resize(VOXTRAL_ENC_LAYERS); - for (int32_t i = 0; i < VOXTRAL_ENC_LAYERS; i++) { + model->enc_layers.resize(model->hp.enc_layers); + for (int32_t i = 0; i < model->hp.enc_layers; i++) { char nm[256]; auto & L = model->enc_layers[i]; snprintf(nm,sizeof(nm),"enc.blk.%d.attn_norm.weight",i); L.attn_norm_weight = get_tensor(ctx_meta,nm); @@ -710,7 +919,14 @@ voxtral_model * voxtral_model_load_from_file( snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_w1.weight",i); L.ffn_w1_weight = get_tensor(ctx_meta,nm); snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_w2.weight",i); L.ffn_w2_weight = get_tensor(ctx_meta,nm); snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_w2.bias",i); L.ffn_w2_bias = get_tensor(ctx_meta,nm); - snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_w3.weight",i); L.ffn_w3_weight = get_tensor(ctx_meta,nm); + if (model->hp.is_offline) { + // Offline Whisper encoder: LayerNorm biases + GELU-MLP w1 bias (no SwiGLU w3). + snprintf(nm,sizeof(nm),"enc.blk.%d.attn_norm.bias",i); L.attn_norm_bias = get_tensor_opt(ctx_meta,nm); + snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_norm.bias",i); L.ffn_norm_bias = get_tensor_opt(ctx_meta,nm); + snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_w1.bias",i); L.ffn_w1_bias = get_tensor_opt(ctx_meta,nm); + } else { + snprintf(nm,sizeof(nm),"enc.blk.%d.ffn_w3.weight",i); L.ffn_w3_weight = get_tensor(ctx_meta,nm); + } } model->adapter_0_weight = get_tensor(ctx_meta, "adapter.0.weight"); @@ -719,8 +935,8 @@ voxtral_model * voxtral_model_load_from_file( model->tok_embeddings_weight = get_tensor(ctx_meta, "tok_embeddings.weight"); model->dec_norm_weight = get_tensor(ctx_meta, "norm.weight"); - model->dec_layers.resize(VOXTRAL_DEC_LAYERS); - for (int32_t i = 0; i < VOXTRAL_DEC_LAYERS; i++) { + model->dec_layers.resize(model->hp.dec_layers); + for (int32_t i = 0; i < model->hp.dec_layers; i++) { char nm[256]; auto & L = model->dec_layers[i]; snprintf(nm,sizeof(nm),"dec.blk.%d.attn_norm.weight",i); L.attn_norm_weight = get_tensor(ctx_meta,nm); @@ -732,8 +948,13 @@ voxtral_model * voxtral_model_load_from_file( snprintf(nm,sizeof(nm),"dec.blk.%d.ffn_w1.weight",i); L.ffn_w1_weight = get_tensor(ctx_meta,nm); snprintf(nm,sizeof(nm),"dec.blk.%d.ffn_w2.weight",i); L.ffn_w2_weight = get_tensor(ctx_meta,nm); snprintf(nm,sizeof(nm),"dec.blk.%d.ffn_w3.weight",i); L.ffn_w3_weight = get_tensor(ctx_meta,nm); - snprintf(nm,sizeof(nm),"dec.blk.%d.ada0.weight",i); L.ada0_weight = get_tensor(ctx_meta,nm); - snprintf(nm,sizeof(nm),"dec.blk.%d.ada2.weight",i); L.ada2_weight = get_tensor(ctx_meta,nm); + // Adaptive RMS-norm time-conditioning weights are realtime-only. + L.ada0_weight = nullptr; + L.ada2_weight = nullptr; + if (model->hp.ada_t_cond) { + snprintf(nm,sizeof(nm),"dec.blk.%d.ada0.weight",i); L.ada0_weight = get_tensor(ctx_meta,nm); + snprintf(nm,sizeof(nm),"dec.blk.%d.ada2.weight",i); L.ada2_weight = get_tensor(ctx_meta,nm); + } } model->mel_filters = get_tensor(ctx_meta, "audio.mel_filters"); @@ -771,23 +992,26 @@ voxtral_model * voxtral_model_load_from_file( } } - log_info("model loaded: enc_layers=" + std::to_string(VOXTRAL_ENC_LAYERS) + - " dec_layers=" + std::to_string(VOXTRAL_DEC_LAYERS) + - " vocab=" + std::to_string(VOXTRAL_VOCAB_SIZE)); + log_info(std::string("model arch: ") + (model->hp.is_offline ? "voxtral (offline)" : "voxtral_realtime") + + " enc_causal=" + (model->hp.enc_causal ? "1" : "0") + + " ada_t_cond=" + (model->hp.ada_t_cond ? "1" : "0")); + log_info("model loaded: enc_layers=" + std::to_string(model->hp.enc_layers) + + " dec_layers=" + std::to_string(model->hp.dec_layers) + + " vocab=" + std::to_string(model->hp.vocab_size)); if (model->buf_weights) { const double sz_mb = (double) ggml_backend_buffer_get_size(model->buf_weights) / 1e6; log_info("model weights: " + std::to_string(sz_mb) + " MB"); } - log_info("encoder: dim=" + std::to_string(VOXTRAL_ENC_DIM) + - " heads=" + std::to_string(VOXTRAL_ENC_HEADS) + - " head_dim=" + std::to_string(VOXTRAL_ENC_HEAD_DIM) + - " hidden=" + std::to_string(VOXTRAL_ENC_HIDDEN)); - log_info("decoder: dim=" + std::to_string(VOXTRAL_DEC_DIM) + - " heads=" + std::to_string(VOXTRAL_DEC_HEADS) + - " head_dim=" + std::to_string(VOXTRAL_DEC_HEAD_DIM) + - " hidden=" + std::to_string(VOXTRAL_DEC_HIDDEN) + - " kv_heads=" + std::to_string(VOXTRAL_DEC_KV_HEADS)); + log_info("encoder: dim=" + std::to_string(model->hp.enc_dim) + + " heads=" + std::to_string(model->hp.enc_heads) + + " head_dim=" + std::to_string(model->hp.enc_head_dim) + + " hidden=" + std::to_string(model->hp.enc_hidden)); + log_info("decoder: dim=" + std::to_string(model->hp.dec_dim) + + " heads=" + std::to_string(model->hp.dec_heads) + + " head_dim=" + std::to_string(model->hp.dec_head_dim) + + " hidden=" + std::to_string(model->hp.dec_hidden) + + " kv_heads=" + std::to_string(model->hp.dec_kv_heads)); { char buf[128]; @@ -819,36 +1043,62 @@ voxtral_context * voxtral_init_from_model( ctx->model = model; ctx->log_level = params.log_level; ctx->logger = params.logger; - ctx->n_threads = params.n_threads > 0 ? params.n_threads : 4; + if (params.n_threads > 0) { + ctx->n_threads = params.n_threads; + } else { + const unsigned hw = std::thread::hardware_concurrency(); + ctx->n_threads = hw > 0 ? (int32_t) std::min(hw, 16u) : 4; + } - // Select backend - bool want_metal = params.use_metal || (model && model->weights_on_metal); + voxtral_gpu_backend gpu = params.gpu; + if (gpu == voxtral_gpu_backend::none && model && model->weights_on_gpu) { + gpu = model->gpu_type; + } + ctx->gpu_type = voxtral_gpu_backend::none; -#ifdef GGML_USE_METAL - if (want_metal) { - ctx->backend = ggml_backend_metal_init(); - if (!ctx->backend) { - LOG_WARN(ctx, "failed to initialize Metal backend, falling back to CPU"); + // GPU compute backend (registry): match the requested backend, or any GPU. + if (gpu != voxtral_gpu_backend::none) { + ggml_backend_dev_t gdev = find_gpu_device(gpu); + if (gdev) { + ctx->backend = ggml_backend_dev_init(gdev, nullptr); + } + if (ctx->backend) { + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(gdev); + ctx->gpu_type = gpu_from_reg(reg ? ggml_backend_reg_name(reg) : ""); + } else { + LOG_INFO(ctx, "no GPU backend available, using CPU"); } } -#endif - if (!ctx->backend) { - ctx->backend = ggml_backend_cpu_init(); - ctx->backend_is_cpu = true; - } else { - ctx->backend_is_cpu = false; - } - if (ctx->backend_is_cpu) { - ggml_backend_cpu_set_n_threads(ctx->backend, ctx->n_threads); + bool has_gpu = (ctx->gpu_type != voxtral_gpu_backend::none); + + if (!ctx->backend) { + // CPU-only: the CPU backend is the compute backend. + ctx->backend = init_cpu_backend(); + backend_set_threads(ctx->backend, ctx->n_threads); LOG_INFO(ctx, "backend: CPU with %d threads", ctx->n_threads); } else { - ctx->backend_cpu = ggml_backend_cpu_init(); - ggml_backend_cpu_set_n_threads(ctx->backend_cpu, ctx->n_threads); - LOG_INFO(ctx, "backend: METAL (CPU fallback %d threads)", ctx->n_threads); + // GPU compute + a CPU backend for fallback ops. + ctx->backend_cpu = init_cpu_backend(); + backend_set_threads(ctx->backend_cpu, ctx->n_threads); + const char * gpu_name = "GPU"; + if (ctx->gpu_type == voxtral_gpu_backend::cuda) gpu_name = "CUDA"; + if (ctx->gpu_type == voxtral_gpu_backend::metal) gpu_name = "METAL"; + if (ctx->gpu_type == voxtral_gpu_backend::vulkan) gpu_name = "VULKAN"; + LOG_INFO(ctx, "backend: %s (CPU fallback %d threads)", gpu_name, ctx->n_threads); + } + + // Accelerator (BLAS/Accelerate) device for faster CPU matmuls, added to the + // scheduler below — preserves CPU-path performance. + if (ggml_backend_dev_t adev = find_accel_device()) { + ctx->blas_backend = ggml_backend_dev_init(adev, nullptr); + if (ctx->blas_backend) { + backend_set_threads(ctx->blas_backend, ctx->n_threads); + LOG_INFO(ctx, "BLAS backend enabled with %d threads", ctx->n_threads); + } } - // Allocate persistent tensors for encoder output, decoder memory, KV cache, logits + // Allocate persistent tensors: encoder chunk output, decoder logits, KV cache { constexpr size_t n_tensors = 5; ggml_init_params p = { @@ -858,29 +1108,31 @@ voxtral_context * voxtral_init_from_model( }; ctx->ctx_persistent = ggml_init(p); - // encoder_output: [enc_dim, max_enc_seq] (transposed: ne[0]=enc_dim rows) - ctx->encoder_output = ggml_new_tensor_2d(ctx->ctx_persistent, GGML_TYPE_F32, - VOXTRAL_ENC_DIM, VOXTRAL_MAX_ENC_SEQ); - ggml_set_name(ctx->encoder_output, "encoder_output"); - - // decoder_memory: [dec_dim, max_dec_seq] - ctx->decoder_memory = ggml_new_tensor_2d(ctx->ctx_persistent, GGML_TYPE_F32, - VOXTRAL_DEC_DIM, VOXTRAL_MAX_DEC_SEQ); - ggml_set_name(ctx->decoder_memory, "decoder_memory"); + // encoder_chunk_output: [enc_dim, MAX_ENC_CHUNK] + ctx->encoder_chunk_output = ggml_new_tensor_2d(ctx->ctx_persistent, GGML_TYPE_F32, + VOXTRAL_ENC_DIM, VOXTRAL_MAX_ENC_CHUNK); + ggml_set_name(ctx->encoder_chunk_output, "encoder_chunk_output"); // decoder_logits: [vocab_size] ctx->decoder_logits = ggml_new_tensor_1d(ctx->ctx_persistent, GGML_TYPE_F32, VOXTRAL_VOCAB_SIZE); ggml_set_name(ctx->decoder_logits, "decoder_logits"); - // KV cache: [kv_dim, dec_window, dec_layers] - const int32_t kv_dim = VOXTRAL_DEC_KV_HEADS * VOXTRAL_DEC_HEAD_DIM; // 1024 + // decoder_argmax: [1] i32 — greedy token id computed on device, so the + // hot decode loop reads back 4 bytes instead of the full vocab logits. + ctx->decoder_argmax = ggml_new_tensor_1d(ctx->ctx_persistent, GGML_TYPE_I32, 1); + ggml_set_name(ctx->decoder_argmax, "decoder_argmax"); + + // KV cache: [kv_dim, dec_window, dec_layers] — layer count is model-dependent + // (realtime 26, offline 30); window is the physical cache capacity for both. + const int32_t kv_dim = ctx->model->hp.dec_kv_heads * ctx->model->hp.dec_head_dim; // 1024 + const int32_t kv_layers = ctx->model->hp.dec_layers; ctx->kv_self_k = ggml_new_tensor_3d(ctx->ctx_persistent, GGML_TYPE_F32, - kv_dim, VOXTRAL_DEC_WINDOW, VOXTRAL_DEC_LAYERS); + kv_dim, VOXTRAL_DEC_WINDOW, kv_layers); ggml_set_name(ctx->kv_self_k, "kv_self_k"); ctx->kv_self_v = ggml_new_tensor_3d(ctx->ctx_persistent, GGML_TYPE_F32, - kv_dim, VOXTRAL_DEC_WINDOW, VOXTRAL_DEC_LAYERS); + kv_dim, VOXTRAL_DEC_WINDOW, kv_layers); ggml_set_name(ctx->kv_self_v, "kv_self_v"); ctx->buf_persistent = ggml_backend_alloc_ctx_tensors(ctx->ctx_persistent, ctx->backend); @@ -890,26 +1142,32 @@ voxtral_context * voxtral_init_from_model( return nullptr; } - // Zero KV cache + // Zero persistent buffer (KV cache etc.) ggml_backend_buffer_clear(ctx->buf_persistent, 0); } { - const double enc_mb = (double) ggml_nbytes(ctx->encoder_output) / 1e6; - const double dec_mb = (double) ggml_nbytes(ctx->decoder_memory) / 1e6; + const double chunk_mb = (double) ggml_nbytes(ctx->encoder_chunk_output) / 1e6; const double kv_mb = (double) (ggml_nbytes(ctx->kv_self_k) + ggml_nbytes(ctx->kv_self_v)) / 1e6; - LOG_INFO(ctx, "buffers: encoder_output=%.2f MB decoder_memory=%.2f MB kv_cache=%.2f MB", - enc_mb, dec_mb, kv_mb); + LOG_INFO(ctx, "buffers: encoder_chunk=%.2f MB kv_cache=%.2f MB", + chunk_mb, kv_mb); } - // Schedulers (Metal + CPU fallback, or CPU only) - ggml_backend_t backends[2]; + // Schedulers — ggml requires the last backend to be CPU. + // With GPU: [GPU, BLAS?, CPU] + // Without GPU: [BLAS?, CPU] + ggml_backend_t backends[4]; int n_backends = 0; - backends[n_backends++] = ctx->backend; - if (!ctx->backend_is_cpu && ctx->backend_cpu) { - backends[n_backends++] = ctx->backend_cpu; + if (has_gpu) { + backends[n_backends++] = ctx->backend; // GPU first + } + if (ctx->blas_backend) { + backends[n_backends++] = ctx->blas_backend; // BLAS before CPU } - const bool op_offload = !ctx->backend_is_cpu; + // CPU must be last + ggml_backend_t cpu_be = has_gpu ? ctx->backend_cpu : ctx->backend; + backends[n_backends++] = cpu_be; + const bool op_offload = has_gpu; ctx->sched_encoder = ggml_backend_sched_new(backends, nullptr, n_backends, GGML_DEFAULT_GRAPH_SIZE, false, op_offload); ctx->sched_adapter = ggml_backend_sched_new(backends, nullptr, n_backends, GGML_DEFAULT_GRAPH_SIZE, false, op_offload); @@ -945,8 +1203,13 @@ void voxtral_free(voxtral_context * ctx) { if (ctx->sched_adapter) ggml_backend_sched_free(ctx->sched_adapter); if (ctx->sched_dec_pre) ggml_backend_sched_free(ctx->sched_dec_pre); if (ctx->sched_dec_step) ggml_backend_sched_free(ctx->sched_dec_step); + if (ctx->buf_enc_full) ggml_backend_buffer_free(ctx->buf_enc_full); + if (ctx->ctx_enc_full) ggml_free(ctx->ctx_enc_full); + if (ctx->buf_dec_mem) ggml_backend_buffer_free(ctx->buf_dec_mem); + if (ctx->ctx_dec_mem) ggml_free(ctx->ctx_dec_mem); if (ctx->buf_persistent) ggml_backend_buffer_free(ctx->buf_persistent); if (ctx->ctx_persistent) ggml_free(ctx->ctx_persistent); + if (ctx->blas_backend) ggml_backend_free(ctx->blas_backend); if (ctx->backend_cpu) ggml_backend_free(ctx->backend_cpu); if (ctx->backend) ggml_backend_free(ctx->backend); delete ctx; @@ -990,7 +1253,7 @@ static void kv_cache_shift_left(voxtral_context * ctx, int32_t shift) { const size_t row_bytes = ctx->kv_self_k->nb[1]; const size_t layer_stride = ctx->kv_self_k->nb[2]; - for (int32_t l = 0; l < VOXTRAL_DEC_LAYERS; ++l) { + for (int32_t l = 0; l < ctx->model->hp.dec_layers; ++l) { uint8_t * k_base = k_data + (size_t) l * layer_stride; uint8_t * v_base = v_data + (size_t) l * layer_stride; @@ -1033,6 +1296,79 @@ causal_conv1d_dims compute_causal_conv1d_dims(int32_t in_len, int32_t kernel_siz return out; } +// Compute the number of encoder tokens from mel frames (accounting for conv and truncation) +static int32_t mel_frames_to_enc_tokens(int32_t n_frames) { + auto d0 = compute_causal_conv1d_dims(n_frames, 3, 1); // conv0 + auto d1 = compute_causal_conv1d_dims(d0.out_len, 3, 2); // conv1 (stride 2) + int32_t trunc = d1.out_len % VOXTRAL_DOWNSAMPLE_FACTOR; + return d1.out_len - trunc; +} + +// Pre-compute total encoder tokens for a given mel frame count (for buffer allocation) +static int32_t compute_total_enc_tokens(int32_t total_mel_frames) { + const int32_t mel_stride = VOXTRAL_ENC_CHUNK_MEL - VOXTRAL_ENC_CHUNK_OVERLAP * 2; + int32_t total = 0; + int32_t mel_offset = 0; + bool first = true; + + while (mel_offset < total_mel_frames) { + int32_t chunk_mel = std::min(VOXTRAL_ENC_CHUNK_MEL, total_mel_frames - mel_offset); + int32_t chunk_tokens = mel_frames_to_enc_tokens(chunk_mel); + int32_t skip = first ? 0 : VOXTRAL_ENC_CHUNK_OVERLAP; + int32_t stride = chunk_tokens - skip; + if (stride <= 0) break; + total += stride; + mel_offset += mel_stride; + first = false; + } + return total; +} + +// Allocate per-utterance encoder output buffer on device +static bool alloc_encoder_output(voxtral_context * ctx, int32_t n_tokens) { + // Free previous allocation + if (ctx->buf_enc_full) { ggml_backend_buffer_free(ctx->buf_enc_full); ctx->buf_enc_full = nullptr; } + if (ctx->ctx_enc_full) { ggml_free(ctx->ctx_enc_full); ctx->ctx_enc_full = nullptr; } + ctx->encoder_output = nullptr; + + ggml_init_params p = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer=*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx->ctx_enc_full = ggml_init(p); + ctx->encoder_output = ggml_new_tensor_2d(ctx->ctx_enc_full, GGML_TYPE_F32, + VOXTRAL_ENC_DIM, n_tokens); + ggml_set_name(ctx->encoder_output, "encoder_output"); + ctx->buf_enc_full = ggml_backend_alloc_ctx_tensors(ctx->ctx_enc_full, ctx->backend); + if (!ctx->buf_enc_full) return false; + + ctx->total_enc_tokens = n_tokens; + return true; +} + +// Allocate per-utterance decoder memory buffer on device +static bool alloc_decoder_memory(voxtral_context * ctx, int32_t dec_seq) { + if (ctx->buf_dec_mem) { ggml_backend_buffer_free(ctx->buf_dec_mem); ctx->buf_dec_mem = nullptr; } + if (ctx->ctx_dec_mem) { ggml_free(ctx->ctx_dec_mem); ctx->ctx_dec_mem = nullptr; } + ctx->decoder_memory = nullptr; + + ggml_init_params p = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer=*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx->ctx_dec_mem = ggml_init(p); + ctx->decoder_memory = ggml_new_tensor_2d(ctx->ctx_dec_mem, GGML_TYPE_F32, + ctx->model->hp.dec_dim, dec_seq); + ggml_set_name(ctx->decoder_memory, "decoder_memory"); + ctx->buf_dec_mem = ggml_backend_alloc_ctx_tensors(ctx->ctx_dec_mem, ctx->backend); + if (!ctx->buf_dec_mem) return false; + + ctx->dec_seq_len = dec_seq; + return true; +} + ggml_tensor * causal_conv1d_graph( ggml_context * ctx0, ggml_tensor * x, @@ -1042,7 +1378,8 @@ ggml_tensor * causal_conv1d_graph( int32_t out_channels, int32_t kernel_size, int32_t stride, - int32_t & out_len) { + int32_t & out_len, + bool symmetric = false) { out_len = 0; if (ctx0 == nullptr || x == nullptr || weight == nullptr || kernel_size <= 0 || stride <= 0) { return nullptr; @@ -1051,7 +1388,17 @@ ggml_tensor * causal_conv1d_graph( return nullptr; } - const auto dims = compute_causal_conv1d_dims(in_len, kernel_size, stride); + causal_conv1d_dims dims{}; + if (symmetric) { + // PyTorch Conv1d(padding=p) with p=(kernel-1)/2 — used by the offline Whisper encoder. + const int32_t p = (kernel_size - 1) / 2; + dims.pad_left = p; + dims.pad_right = p; + dims.padded_len = in_len + 2 * p; + dims.out_len = (dims.padded_len - kernel_size) / stride + 1; + } else { + dims = compute_causal_conv1d_dims(in_len, kernel_size, stride); + } if (dims.out_len <= 0) { return nullptr; } @@ -1079,8 +1426,8 @@ void print_tensor_info(struct ggml_tensor * tensor) { printf("Tensor name: %s\n", tensor->name); printf("Tensor type: %s\n", ggml_type_name(tensor->type)); printf("Number of dimensions: %d\n", ggml_n_dims(tensor)); - printf("Total elements: %ld \n", ggml_nelements(tensor)); - printf("Shape: [%ld , %ld, %ld, %ld]\n", + printf("Total elements: %" PRId64 "\n", ggml_nelements(tensor)); + printf("Shape: [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]\n", tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); } @@ -1089,11 +1436,11 @@ static void log_tensor_info(voxtral_context * ctx, const char * tag, struct ggml LOG_DBG(ctx, "%s: ", tag); return; } - LOG_DBG(ctx, "%s: type=%s ne=[%ld,%ld,%ld,%ld] nb=[%ld,%ld,%ld,%ld] n_dims=%d nbytes=%zu", + LOG_DBG(ctx, "%s: type=%s ne=[%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 "] nb=[%zu,%zu,%zu,%zu] n_dims=%d nbytes=%zu", tag, ggml_type_name(t->type), t->ne[0], t->ne[1], t->ne[2], t->ne[3], - t->nb[0], t->nb[1], t->nb[2], t->nb[3], + (size_t) t->nb[0], (size_t) t->nb[1], (size_t) t->nb[2], (size_t) t->nb[3], ggml_n_dims(t), (size_t) ggml_nbytes(t)); } @@ -1107,12 +1454,23 @@ static void log_graph_info(voxtral_context * ctx, const char * name, struct ggml LOG_INFO(ctx, "%s graph: size=%d nodes=%d", name, size, nodes); } -// Build encoder graph that writes output into ctx->encoder_output +// Build encoder graph that writes output into ctx->encoder_chunk_output +// Apply LayerNorm (offline Whisper: weight+bias) or RMSNorm (realtime: weight only), +// if a bias tensor is provided. +static ggml_tensor * enc_apply_norm(ggml_context * gctx, ggml_tensor * x, + ggml_tensor * w, ggml_tensor * b, float eps, bool layernorm) { + ggml_tensor * n = layernorm ? ggml_norm(gctx, x, eps) : ggml_rms_norm(gctx, x, eps); + n = ggml_mul(gctx, n, w); + if (b) n = ggml_add(gctx, n, b); + return n; +} + static ggml_cgraph * build_encoder_graph( voxtral_context * ctx, ggml_context * gctx, const float * mel_data, // [n_mel, n_frames] on CPU - int32_t n_frames) + int32_t n_frames, + int32_t * out_seq_len) // output: encoder tokens produced by this chunk { LOG_DBG(ctx, "Building encoder graph"); voxtral_model * model = ctx->model; @@ -1133,11 +1491,12 @@ static ggml_cgraph * build_encoder_graph( log_tensor_info(ctx, "enc.conv1.weight", model->enc_conv1_weight); log_tensor_info(ctx, "mel_input", mel_input); + const bool conv_sym = model->hp.is_offline; // offline Whisper uses symmetric conv padding int32_t conv0_len = 0; ggml_tensor * conv0_out = causal_conv1d_graph( gctx, mel_input, n_frames, model->enc_conv0_weight, model->enc_conv0_bias, - VOXTRAL_ENC_DIM, 3, 1, conv0_len); + model->hp.enc_dim, 3, 1, conv0_len, conv_sym); if (conv0_out == nullptr) { LOG_ERR(ctx, "conv0_out is null"); return gf; @@ -1149,7 +1508,7 @@ static ggml_cgraph * build_encoder_graph( ggml_tensor * conv1_out = causal_conv1d_graph( gctx, conv0_out, conv0_len, model->enc_conv1_weight, model->enc_conv1_bias, - VOXTRAL_ENC_DIM, 3, 2, conv_out_len); + model->hp.enc_dim, 3, 2, conv_out_len, conv_sym); if (conv1_out == nullptr) { LOG_ERR(ctx, "conv1_out is null"); return gf; @@ -1163,13 +1522,13 @@ static ggml_cgraph * build_encoder_graph( // This is what we need for mul_mat: ggml_mul_mat(weight[out,in], x[in,seq]) -> [out,seq] // Left-truncate to multiple of downsample_factor (matching Python) - const int32_t trunc = conv_out_len % VOXTRAL_DOWNSAMPLE_FACTOR; + const int32_t trunc = conv_out_len % model->hp.downsample_factor; ggml_tensor * x_len_first = conv1_out; int32_t seq_len = conv_out_len; if (trunc > 0) { // Skip first 'trunc' frames along length dimension (ne0) x_len_first = ggml_view_3d(gctx, conv1_out, - conv_out_len - trunc, VOXTRAL_ENC_DIM, 1, + conv_out_len - trunc, model->hp.enc_dim, 1, conv1_out->nb[1], conv1_out->nb[2], (size_t) trunc * conv1_out->nb[0]); // [len, enc_dim, 1] seq_len = conv_out_len - trunc; @@ -1180,7 +1539,14 @@ static ggml_cgraph * build_encoder_graph( // Transpose to [enc_dim, seq_len] for transformer blocks ggml_tensor * x = ggml_permute(gctx, x_len_first, 1, 0, 2, 3); // [enc_dim, seq_len, 1] x = ggml_cont(gctx, x); - x = ggml_reshape_2d(gctx, x, VOXTRAL_ENC_DIM, seq_len); + x = ggml_reshape_2d(gctx, x, model->hp.enc_dim, seq_len); + // Offline Whisper encoder: add fixed sinusoidal positional embeddings to the + // conv output (it uses absolute positions, not RoPE). + if (model->hp.is_offline && model->enc_pos_embedding) { + ggml_tensor * pos = ggml_view_2d(gctx, model->enc_pos_embedding, + model->hp.enc_dim, seq_len, model->enc_pos_embedding->nb[1], 0); + x = ggml_add(gctx, x, pos); + } log_tensor_info(ctx, "encoder_x", x); // Position tensor for RoPE: [seq_len] int32 @@ -1188,119 +1554,104 @@ static ggml_cgraph * build_encoder_graph( ggml_set_name(enc_positions, "enc_positions"); ggml_backend_sched_set_tensor_backend(ctx->sched_encoder, enc_positions, ctx->backend); - // Encoder attention mask (sliding causal window) - ggml_tensor * enc_attn_mask = ggml_new_tensor_2d(gctx, GGML_TYPE_F32, seq_len, seq_len); - ggml_set_name(enc_attn_mask, "enc_attn_mask"); - ggml_backend_sched_set_tensor_backend(ctx->sched_encoder, enc_attn_mask, ctx->backend); + // Encoder attention mask (sliding causal window) — realtime only. The offline + // Whisper encoder uses full bidirectional attention, so no mask is created. + ggml_tensor * enc_attn_mask_f16 = nullptr; + if (!model->hp.is_offline) { + ggml_tensor * enc_attn_mask = ggml_new_tensor_2d(gctx, GGML_TYPE_F32, seq_len, seq_len); + ggml_set_name(enc_attn_mask, "enc_attn_mask"); + ggml_backend_sched_set_tensor_backend(ctx->sched_encoder, enc_attn_mask, ctx->backend); + enc_attn_mask_f16 = ggml_cast(gctx, enc_attn_mask, GGML_TYPE_F16); + } // Transformer layers - for (int32_t i = 0; i < VOXTRAL_ENC_LAYERS; i++) { + const auto & hp = model->hp; + const bool ln = hp.is_offline; // offline = LayerNorm; realtime = RMSNorm + const int32_t e_heads = hp.enc_heads; + const int32_t e_kv_heads = hp.enc_kv_heads; + const int32_t e_hd = hp.enc_head_dim; + for (int32_t i = 0; i < hp.enc_layers; i++) { auto & L = model->enc_layers[i]; - // Pre-attention RMS norm + // Pre-attention norm (LayerNorm for offline, RMSNorm for realtime) ggml_tensor * residual = x; // [enc_dim, seq_len] - ggml_tensor * x_norm = ggml_rms_norm(gctx, x, VOXTRAL_ENC_NORM_EPS); // [enc_dim, seq_len] - x_norm = ggml_mul(gctx, x_norm, L.attn_norm_weight); // [enc_dim, seq_len] + ggml_tensor * x_norm = enc_apply_norm(gctx, x, L.attn_norm_weight, L.attn_norm_bias, hp.enc_norm_eps, ln); // Q, K, V projections - ggml_tensor * q = ggml_mul_mat(gctx, L.attn_q_weight, x_norm); // [enc_heads*head_dim, seq_len] - q = ggml_add(gctx, q, L.attn_q_bias); // [enc_heads*head_dim, seq_len] - - ggml_tensor * k = ggml_mul_mat(gctx, L.attn_k_weight, x_norm); // [enc_kv_heads*head_dim, seq_len] - // k has no bias in encoder - - ggml_tensor * v = ggml_mul_mat(gctx, L.attn_v_weight, x_norm); // [enc_kv_heads*head_dim, seq_len] - v = ggml_add(gctx, v, L.attn_v_bias); // [enc_kv_heads*head_dim, seq_len] - - // Reshape for RoPE: [head_dim, n_heads, seq_len] - q = ggml_reshape_3d(gctx, q, VOXTRAL_ENC_HEAD_DIM, VOXTRAL_ENC_HEADS, seq_len); - k = ggml_reshape_3d(gctx, k, VOXTRAL_ENC_HEAD_DIM, VOXTRAL_ENC_KV_HEADS, seq_len); - - // Apply RoPE (interleaved, mode=0) - // ggml_rope_ext expects: a=[head_dim, n_heads, seq], b=[seq] positions - q = ggml_rope_ext(gctx, q, enc_positions, nullptr, - VOXTRAL_ENC_HEAD_DIM, 0, 0, - VOXTRAL_ENC_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); // [head_dim, n_heads, seq_len] - k = ggml_rope_ext(gctx, k, enc_positions, nullptr, - VOXTRAL_ENC_HEAD_DIM, 0, 0, - VOXTRAL_ENC_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); // [head_dim, n_kv_heads, seq_len] - - // Reshape for attention: [head_dim, n_heads, seq_len] -> permute to [head_dim, seq_len, n_heads] - q = ggml_permute(gctx, q, 0, 2, 1, 3); // [head_dim, seq_len, n_heads] - k = ggml_permute(gctx, k, 0, 2, 1, 3); // [head_dim, seq_len, n_kv_heads] - - // v: [enc_kv_heads*head_dim, seq_len] -> [head_dim, n_kv_heads, seq_len] -> permute - v = ggml_reshape_3d(gctx, v, VOXTRAL_ENC_HEAD_DIM, VOXTRAL_ENC_KV_HEADS, seq_len); - v = ggml_permute(gctx, v, 0, 2, 1, 3); // [head_dim, seq_len, n_kv_heads] - // For value in attention, we need [head_dim, seq_len, n_kv_heads] - // Actually ggml attention: let's use ggml_soft_max_ext + mul_mat manually - - // GQA: expand KV heads if needed - // Encoder: ENC_HEADS == ENC_KV_HEADS == 32, so no expansion needed - - // Compute attention scores: Q @ K^T / sqrt(head_dim) - // Q: [head_dim, seq_len, n_heads], K: [head_dim, seq_len, n_kv_heads] - // ggml_mul_mat over 3D: for each head, Q[head_dim, seq_q] @ K[head_dim, seq_k] -> [seq_k, seq_q] - // This is: K^T @ Q -> [seq_k, seq_q] per head - ggml_tensor * scores = ggml_mul_mat(gctx, k, q); // [seq_len, seq_len, n_heads] - - // Apply sliding causal mask + scale + softmax in one op - const float scale = 1.0f / sqrtf((float)VOXTRAL_ENC_HEAD_DIM); - scores = ggml_soft_max_ext(gctx, scores, enc_attn_mask, scale, 0.0f); // [seq_len, seq_len, n_heads] - - // Apply attention: scores @ V - // V: [head_dim, seq_len, n_heads] - // Need V transposed: [seq_len, head_dim, n_heads] - ggml_tensor * v_t = ggml_permute(gctx, v, 1, 0, 2, 3); // [seq_len, head_dim, n_heads] - // But actually for ggml_mul_mat: A[K,N] @ B[K,M] -> [N,M] - // scores: [seq_len(K), seq_len(Q), n_heads] - // v: [head_dim, seq_len(V), n_heads] -> we want result [head_dim, seq_q, n_heads] - // Use: ggml_mul_mat(v, scores) -> v has [head_dim, seq_v], scores has [seq_v, seq_q] - // -> but v's ne[0]=head_dim, scores's ne[0]=seq_len, mismatch! - // Correct: v_t = ggml_cont(permute(v, 1,0,2,3)) -> [seq_len, head_dim, n_heads] - // ggml_mul_mat(v_t, scores): v_t[seq_len, head_dim], scores[seq_len, seq_q] -> [head_dim, seq_q] - v_t = ggml_cont(gctx, v_t); // make contiguous [seq_len, head_dim, n_heads] - ggml_tensor * attn_out = ggml_mul_mat(gctx, v_t, scores); // [head_dim, seq_len, n_heads] - - // Reshape back: [head_dim, seq_len, n_heads] -> permute to [head_dim, n_heads, seq_len] - attn_out = ggml_permute(gctx, attn_out, 0, 2, 1, 3); // [head_dim, n_heads, seq_len] + ggml_tensor * q = ggml_mul_mat(gctx, L.attn_q_weight, x_norm); // [e_heads*hd, seq_len] + q = ggml_add(gctx, q, L.attn_q_bias); + + ggml_tensor * k = ggml_mul_mat(gctx, L.attn_k_weight, x_norm); // [e_kv_heads*hd, seq_len] (no bias) + + ggml_tensor * v = ggml_mul_mat(gctx, L.attn_v_weight, x_norm); // [e_kv_heads*hd, seq_len] + v = ggml_add(gctx, v, L.attn_v_bias); + + q = ggml_reshape_3d(gctx, q, e_hd, e_heads, seq_len); + k = ggml_reshape_3d(gctx, k, e_hd, e_kv_heads, seq_len); + + // Realtime encoder uses RoPE; offline Whisper encoder uses absolute + // sinusoidal positions added above, so no RoPE here. + if (!hp.is_offline) { + q = ggml_rope_ext(gctx, q, enc_positions, nullptr, + e_hd, 0, 0, hp.enc_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(gctx, k, enc_positions, nullptr, + e_hd, 0, 0, hp.enc_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + } + + q = ggml_permute(gctx, q, 0, 2, 1, 3); // [hd, seq_len, e_heads] + k = ggml_permute(gctx, k, 0, 2, 1, 3); // [hd, seq_len, e_kv_heads] + + v = ggml_reshape_3d(gctx, v, e_hd, e_kv_heads, seq_len); + v = ggml_permute(gctx, v, 0, 2, 1, 3); // [hd, seq_len, e_kv_heads] + + const float scale = 1.0f / sqrtf((float)e_hd); + + // Mask: realtime passes the sliding-causal mask; offline passes none (full attention). + ggml_tensor * mask = ln ? nullptr : enc_attn_mask_f16; + ggml_tensor * attn_out = ggml_flash_attn_ext(gctx, q, k, v, mask, scale, 0.0f, 0.0f); attn_out = ggml_cont(gctx, attn_out); - attn_out = ggml_reshape_2d(gctx, attn_out, VOXTRAL_ENC_HEADS * VOXTRAL_ENC_HEAD_DIM, seq_len); // [n_heads*head_dim, seq_len] + attn_out = ggml_reshape_2d(gctx, attn_out, e_heads * e_hd, seq_len); // Output projection + residual ggml_tensor * attn_proj = ggml_mul_mat(gctx, L.attn_o_weight, attn_out); // [enc_dim, seq_len] - attn_proj = ggml_add(gctx, attn_proj, L.attn_o_bias); // [enc_dim, seq_len] - x = ggml_add(gctx, residual, attn_proj); // [enc_dim, seq_len] + attn_proj = ggml_add(gctx, attn_proj, L.attn_o_bias); + x = ggml_add(gctx, residual, attn_proj); // FFN - residual = x; // [enc_dim, seq_len] - x_norm = ggml_rms_norm(gctx, x, VOXTRAL_ENC_NORM_EPS); // [enc_dim, seq_len] - x_norm = ggml_mul(gctx, x_norm, L.ffn_norm_weight); // [enc_dim, seq_len] - - // SwiGLU: silu(w1(x)) * w3(x), then w2 - ggml_tensor * gate = ggml_mul_mat(gctx, L.ffn_w1_weight, x_norm); // [enc_hidden, seq_len] - gate = ggml_silu(gctx, gate); // [enc_hidden, seq_len] - ggml_tensor * up = ggml_mul_mat(gctx, L.ffn_w3_weight, x_norm); // [enc_hidden, seq_len] - ggml_tensor * ffn_out = ggml_mul(gctx, gate, up); // [enc_hidden, seq_len] - ffn_out = ggml_mul_mat(gctx, L.ffn_w2_weight, ffn_out); // [enc_dim, seq_len] - ffn_out = ggml_add(gctx, ffn_out, L.ffn_w2_bias); // [enc_dim, seq_len] + residual = x; + x_norm = enc_apply_norm(gctx, x, L.ffn_norm_weight, L.ffn_norm_bias, hp.enc_norm_eps, ln); + + ggml_tensor * ffn_out; + if (hp.is_offline) { + // Standard Whisper MLP: w2(gelu(w1 x + b1)) + b2 + ggml_tensor * h = ggml_mul_mat(gctx, L.ffn_w1_weight, x_norm); // [enc_hidden, seq_len] + if (L.ffn_w1_bias) h = ggml_add(gctx, h, L.ffn_w1_bias); + h = ggml_gelu_erf(gctx, h); + ffn_out = ggml_mul_mat(gctx, L.ffn_w2_weight, h); // [enc_dim, seq_len] + } else { + // Realtime SwiGLU: silu(w1 x) * w3(x), then w2 + ggml_tensor * gate = ggml_mul_mat(gctx, L.ffn_w1_weight, x_norm); + gate = ggml_silu(gctx, gate); + ggml_tensor * up = ggml_mul_mat(gctx, L.ffn_w3_weight, x_norm); + ffn_out = ggml_mul(gctx, gate, up); + ffn_out = ggml_mul_mat(gctx, L.ffn_w2_weight, ffn_out); // [enc_dim, seq_len] + } + if (L.ffn_w2_bias) ffn_out = ggml_add(gctx, ffn_out, L.ffn_w2_bias); - x = ggml_add(gctx, residual, ffn_out); // [enc_dim, seq_len] + x = ggml_add(gctx, residual, ffn_out); } // Final norm - x = ggml_rms_norm(gctx, x, VOXTRAL_ENC_NORM_EPS); // [enc_dim, seq_len] - x = ggml_mul(gctx, x, model->enc_norm_weight); // [enc_dim, seq_len] + x = enc_apply_norm(gctx, x, model->enc_norm_weight, model->enc_norm_bias, hp.enc_norm_eps, ln); - // Copy result to persistent encoder_output - ggml_tensor * enc_out_view = ggml_view_2d(gctx, ctx->encoder_output, + // Copy result to encoder_chunk_output (per-chunk buffer, reused each chunk) + ggml_tensor * enc_out_view = ggml_view_2d(gctx, ctx->encoder_chunk_output, VOXTRAL_ENC_DIM, seq_len, - ctx->encoder_output->nb[1], 0); // [enc_dim, seq_len] + ctx->encoder_chunk_output->nb[1], 0); // [enc_dim, seq_len] ggml_tensor * cpy = ggml_cpy(gctx, x, enc_out_view); ggml_build_forward_expand(gf, cpy); - ctx->enc_seq_len = conv_out_len; - ctx->enc_seq_used = seq_len; + if (out_seq_len) *out_seq_len = seq_len; return gf; } @@ -1337,7 +1688,7 @@ static ggml_cgraph * build_adapter_graph( // Copy to persistent decoder_memory ggml_tensor * dec_mem_view = ggml_view_2d(gctx, ctx->decoder_memory, - VOXTRAL_DEC_DIM, dec_seq, + ctx->model->hp.dec_dim, dec_seq, ctx->decoder_memory->nb[1], 0); // [dec_dim, dec_seq] ggml_tensor * cpy = ggml_cpy(gctx, x, dec_mem_view); ggml_build_forward_expand(gf, cpy); @@ -1367,13 +1718,19 @@ static ggml_tensor * build_decoder_layer( ggml_tensor * attn_mask) // [n_kv, n_tokens] or nullptr { voxtral_model * model = ctx->model; + const auto & hp = model->hp; auto & L = model->dec_layers[layer_idx]; - const int32_t kv_dim = VOXTRAL_DEC_KV_HEADS * VOXTRAL_DEC_HEAD_DIM; // 1024 + const int32_t head_dim = hp.dec_head_dim; + const int32_t n_heads = hp.dec_heads; + const int32_t n_kv_heads = hp.dec_kv_heads; + const int32_t kv_dim = n_kv_heads * head_dim; + const float norm_eps = hp.dec_norm_eps; + const float dec_rope_theta = hp.dec_rope_theta; // Pre-attention RMS norm ggml_tensor * residual = x; // [dec_dim, n_tokens] - ggml_tensor * x_norm = ggml_rms_norm(gctx, x, VOXTRAL_DEC_NORM_EPS); // [dec_dim, n_tokens] + ggml_tensor * x_norm = ggml_rms_norm(gctx, x, norm_eps); // [dec_dim, n_tokens] x_norm = ggml_mul(gctx, x_norm, L.attn_norm_weight); // [dec_dim, n_tokens] // Q, K, V (no bias in decoder) @@ -1382,19 +1739,17 @@ static ggml_tensor * build_decoder_layer( ggml_tensor * v = ggml_mul_mat(gctx, L.attn_v_weight, x_norm); // [kv_dim, n_tokens] // Reshape for RoPE: [head_dim, n_heads, n_tokens] - q = ggml_reshape_3d(gctx, q, VOXTRAL_DEC_HEAD_DIM, VOXTRAL_DEC_HEADS, n_tokens); - k = ggml_reshape_3d(gctx, k, VOXTRAL_DEC_HEAD_DIM, VOXTRAL_DEC_KV_HEADS, n_tokens); + q = ggml_reshape_3d(gctx, q, head_dim, n_heads, n_tokens); + k = ggml_reshape_3d(gctx, k, head_dim, n_kv_heads, n_tokens); // RoPE (interleaved, mode=0) q = ggml_rope_ext(gctx, q, positions, nullptr, - VOXTRAL_DEC_HEAD_DIM, 0, 0, - VOXTRAL_DEC_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + head_dim, 0, 0, dec_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); k = ggml_rope_ext(gctx, k, positions, nullptr, - VOXTRAL_DEC_HEAD_DIM, 0, 0, - VOXTRAL_DEC_ROPE_THETA, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + head_dim, 0, 0, dec_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); // Flatten Q back: [head_dim, n_heads, n_tokens] -> [n_heads*head_dim, n_tokens] - q = ggml_cont(gctx, ggml_reshape_2d(gctx, q, VOXTRAL_DEC_HEADS * VOXTRAL_DEC_HEAD_DIM, n_tokens)); + q = ggml_cont(gctx, ggml_reshape_2d(gctx, q, n_heads * head_dim, n_tokens)); k = ggml_cont(gctx, ggml_reshape_2d(gctx, k, kv_dim, n_tokens)); // Store K, V in KV cache at positions [kv_offset .. kv_offset+n_tokens-1] @@ -1425,49 +1780,29 @@ static ggml_tensor * build_decoder_layer( ctx->kv_self_v->nb[1], layer_idx * ctx->kv_self_v->nb[2]); // [kv_dim, n_kv] - // Multi-head attention with GQA - // Q: [n_heads*head_dim, n_tokens] -> [head_dim, n_heads, n_tokens] -> permute [head_dim, n_tokens, n_heads] - ggml_tensor * q3 = ggml_reshape_3d(gctx, q, VOXTRAL_DEC_HEAD_DIM, VOXTRAL_DEC_HEADS, n_tokens); + // Flash attention with GQA + // Q: [n_heads*head_dim, n_tokens] -> [head_dim, n_heads, n_tokens] -> [head_dim, n_tokens, n_heads] + ggml_tensor * q3 = ggml_reshape_3d(gctx, q, head_dim, n_heads, n_tokens); q3 = ggml_permute(gctx, q3, 0, 2, 1, 3); // [head_dim, n_tokens, n_heads] - // K: [kv_dim, n_kv] -> [head_dim, n_kv_heads, n_kv] -> permute [head_dim, n_kv, n_kv_heads] - ggml_tensor * k3 = ggml_reshape_3d(gctx, k_full, VOXTRAL_DEC_HEAD_DIM, VOXTRAL_DEC_KV_HEADS, n_kv); + // K: [kv_dim, n_kv] -> [head_dim, n_kv_heads, n_kv] -> [head_dim, n_kv, n_kv_heads] + ggml_tensor * k3 = ggml_reshape_3d(gctx, k_full, head_dim, n_kv_heads, n_kv); k3 = ggml_permute(gctx, k3, 0, 2, 1, 3); // [head_dim, n_kv, n_kv_heads] - // V: [kv_dim, n_kv] -> [head_dim, n_kv_heads, n_kv] -> permute [head_dim, n_kv, n_kv_heads] - ggml_tensor * v3 = ggml_reshape_3d(gctx, v_full, VOXTRAL_DEC_HEAD_DIM, VOXTRAL_DEC_KV_HEADS, n_kv); + // V: [kv_dim, n_kv] -> [head_dim, n_kv_heads, n_kv] -> [head_dim, n_kv, n_kv_heads] + ggml_tensor * v3 = ggml_reshape_3d(gctx, v_full, head_dim, n_kv_heads, n_kv); v3 = ggml_permute(gctx, v3, 0, 2, 1, 3); // [head_dim, n_kv, n_kv_heads] - // GQA: replicate KV heads to match Q heads - // dec_heads=32, dec_kv_heads=8, ratio=4 - // ggml_mul_mat broadcasts: if ne[2] of A < ne[2] of B, A is repeated - // So if k3 has ne[2]=n_kv_heads=8 and q3 has ne[2]=n_heads=32, ggml_mul_mat - // will automatically broadcast k3 across groups of 4 - - // Scores: K^T @ Q -> [n_kv, n_tokens, n_heads] - ggml_tensor * scores = ggml_mul_mat(gctx, k3, q3); // [n_kv, n_tokens, n_heads] + const float scale = 1.0f / sqrtf((float) head_dim); - // Scale + mask + softmax - const float scale = 1.0f / sqrtf((float)VOXTRAL_DEC_HEAD_DIM); - - if (attn_mask) { - // Use ggml_soft_max_ext which combines scale + mask + softmax - scores = ggml_soft_max_ext(gctx, scores, attn_mask, scale, 0.0f); - } else { - // For step graph (n_tokens=1), all KV positions are valid (causal by construction) - scores = ggml_soft_max_ext(gctx, scores, nullptr, scale, 0.0f); - } - - // Attention output: V @ scores^T - // V: [head_dim, n_kv, n_kv_heads], scores: [n_kv, n_tokens, n_heads] - // v_t: [n_kv, head_dim, n_kv_heads] - ggml_tensor * v_t = ggml_cont(gctx, ggml_permute(gctx, v3, 1, 0, 2, 3)); // [n_kv, head_dim, n_kv_heads] - ggml_tensor * attn_out = ggml_mul_mat(gctx, v_t, scores); // [head_dim, n_tokens, n_heads] - - // Reshape: [head_dim, n_tokens, n_heads] -> permute [head_dim, n_heads, n_tokens] -> flatten - attn_out = ggml_permute(gctx, attn_out, 0, 2, 1, 3); // [head_dim, n_heads, n_tokens] + // ggml_flash_attn_ext fuses Q@K^T, scale, mask, softmax, @V in one op + // GQA broadcast is built-in (n_heads % n_kv_heads == 0) + // Mask is cast to F16 inside the graph if provided + ggml_tensor * attn_mask_f16 = attn_mask ? ggml_cast(gctx, attn_mask, GGML_TYPE_F16) : nullptr; + ggml_tensor * attn_out = ggml_flash_attn_ext(gctx, q3, k3, v3, attn_mask_f16, scale, 0.0f, 0.0f); + // Output: [head_dim, n_heads, n_tokens] (already permuted by flash_attn_ext) attn_out = ggml_cont(gctx, attn_out); - attn_out = ggml_reshape_2d(gctx, attn_out, VOXTRAL_DEC_HEADS * VOXTRAL_DEC_HEAD_DIM, n_tokens); + attn_out = ggml_reshape_2d(gctx, attn_out, n_heads * head_dim, n_tokens); // Output projection + residual ggml_tensor * attn_proj = ggml_mul_mat(gctx, L.attn_o_weight, attn_out); // [dec_dim, n_tokens] @@ -1475,13 +1810,12 @@ static ggml_tensor * build_decoder_layer( // Pre-FFN RMS norm residual = x; // [dec_dim, n_tokens] - ggml_tensor * h_norm = ggml_rms_norm(gctx, x, VOXTRAL_DEC_NORM_EPS); // [dec_dim, n_tokens] + ggml_tensor * h_norm = ggml_rms_norm(gctx, x, norm_eps); // [dec_dim, n_tokens] h_norm = ggml_mul(gctx, h_norm, L.ffn_norm_weight); // [dec_dim, n_tokens] - // Ada time conditioning: h_norm = h_norm * (1 + ada_mlp(time_emb)) - // = h_norm + h_norm * ada_scale - // ada_mlp: Linear(3072->32) -> GELU -> Linear(32->3072) - { + // Ada time conditioning: h_norm *= (1 + ada_mlp(time_emb)). + // The offline Ministral decoder has no time conditioning just RMSNorm. + if (model->hp.ada_t_cond && L.ada0_weight && L.ada2_weight && time_emb) { ggml_tensor * ada_hidden = ggml_mul_mat(gctx, L.ada0_weight, time_emb); // [ada_dim] ada_hidden = ggml_gelu_erf(gctx, ada_hidden); // [ada_dim] ggml_tensor * ada_scale = ggml_mul_mat(gctx, L.ada2_weight, ada_hidden); // [dec_dim] @@ -1548,13 +1882,13 @@ static ggml_cgraph * build_decoder_prefill_graph( ggml_backend_sched_set_tensor_backend(ctx->sched_dec_pre, causal_mask, ctx->backend); // Decoder layers - for (int32_t i = 0; i < VOXTRAL_DEC_LAYERS; i++) { + for (int32_t i = 0; i < model->hp.dec_layers; i++) { x = build_decoder_layer(ctx, gctx, gf, x, positions, time_emb, i, n_tokens, /*kv_offset=*/0, causal_mask); } // Final norm - x = ggml_rms_norm(gctx, x, VOXTRAL_DEC_NORM_EPS); // [dec_dim, n_tokens] + x = ggml_rms_norm(gctx, x, model->hp.dec_norm_eps); // [dec_dim, n_tokens] x = ggml_mul(gctx, x, model->dec_norm_weight); // [dec_dim, n_tokens] // Logits for last token only: extract last token -> matmul with embeddings @@ -1569,6 +1903,17 @@ static ggml_cgraph * build_decoder_prefill_graph( return gf; } +// Emit logits = W @ last_hidden into the persistent decoder_logits tensor, plus an +// on-device greedy argmax into decoder_argmax (so the hot loop reads back 4 bytes). +// last_hidden is [dec_dim]; W is the tied (tok_embeddings) or untied output proj. +static void emit_logits_argmax(voxtral_context * ctx, ggml_context * gctx, ggml_cgraph * gf, + ggml_tensor * last_hidden, ggml_tensor * W) { + ggml_tensor * logits = ggml_mul_mat(gctx, W, last_hidden); // [vocab] + ggml_build_forward_expand(gf, ggml_cpy(gctx, logits, ctx->decoder_logits)); + ggml_tensor * amax = ggml_argmax(gctx, ggml_reshape_2d(gctx, logits, ctx->model->hp.vocab_size, 1)); + ggml_build_forward_expand(gf, ggml_cpy(gctx, amax, ctx->decoder_argmax)); +} + // ============================================================================ // Graph Building: Decoder Step (single token) // ============================================================================ @@ -1611,22 +1956,102 @@ static ggml_cgraph * build_decoder_step_graph( ggml_tensor * x = ggml_add(gctx, tok_emb, audio_emb); // [dec_dim, 1] // Decoder layers (no mask needed for single token - all KV positions are valid) - for (int32_t i = 0; i < VOXTRAL_DEC_LAYERS; i++) { + for (int32_t i = 0; i < model->hp.dec_layers; i++) { x = build_decoder_layer(ctx, gctx, gf, x, pos_tensor, time_emb, i, 1, /*kv_offset=*/kv_used, /*attn_mask=*/nullptr); } // Final norm - x = ggml_rms_norm(gctx, x, VOXTRAL_DEC_NORM_EPS); // [dec_dim, 1] + x = ggml_rms_norm(gctx, x, model->hp.dec_norm_eps); // [dec_dim, 1] x = ggml_mul(gctx, x, model->dec_norm_weight); // [dec_dim, 1] - // Logits + // Logits (tied to token embeddings) + on-device argmax. ggml_tensor * x_flat = ggml_reshape_1d(gctx, x, VOXTRAL_DEC_DIM); // [dec_dim] - ggml_tensor * logits = ggml_mul_mat(gctx, model->tok_embeddings_weight, x_flat); // [vocab_size] + emit_logits_argmax(ctx, gctx, gf, x_flat, model->tok_embeddings_weight); - // Copy to persistent - ggml_build_forward_expand(gf, ggml_cpy(gctx, logits, ctx->decoder_logits)); + return gf; +} + +// ============================================================================ +// Graph Building: Offline decode (Voxtral-Mini-3B-2507) +// +// The offline model is a standard audio-LLM: the prompt is a mixed sequence of +// text-token embeddings and audio embeddings (the adapter output substituted at +// the [AUDIO] placeholder positions), prefilled once; then text tokens are +// generated autoregressively until EOS. No per-frame audio indexing, no ada. +// ============================================================================ + +// Final RMS norm + (untied) output projection + argmax for the LAST token only. +static void offline_logits_tail(voxtral_context * ctx, ggml_context * gctx, ggml_cgraph * gf, + ggml_tensor * x /*[dec_dim, n_tokens]*/, int32_t n_tokens) { + voxtral_model * m = ctx->model; + ggml_tensor * last = ggml_view_1d(gctx, x, m->hp.dec_dim, + (size_t)(n_tokens - 1) * x->nb[1]); // [dec_dim] + last = ggml_rms_norm(gctx, last, m->hp.dec_norm_eps); + last = ggml_mul(gctx, last, m->dec_norm_weight); + // Offline model has an untied output projection; fall back to tied embeddings. + ggml_tensor * W = m->output_weight ? m->output_weight : m->tok_embeddings_weight; + emit_logits_argmax(ctx, gctx, gf, last, W); +} + +// Prefill: prefix text tokens + n_audio audio embeddings + suffix text tokens. +static ggml_cgraph * build_offline_prefill_graph( + voxtral_context * ctx, ggml_context * gctx, + int32_t n_prefix, int32_t n_audio, int32_t n_suffix) { + voxtral_model * model = ctx->model; + ggml_cgraph * gf = ggml_new_graph_custom(gctx, GGML_DEFAULT_GRAPH_SIZE * 8, false); + const int32_t n_tokens = n_prefix + n_audio + n_suffix; + + ggml_tensor * prefix_ids = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, n_prefix); + ggml_set_name(prefix_ids, "prefix_ids"); + ggml_backend_sched_set_tensor_backend(ctx->sched_dec_pre, prefix_ids, ctx->backend); + ggml_tensor * suffix_ids = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, n_suffix); + ggml_set_name(suffix_ids, "suffix_ids"); + ggml_backend_sched_set_tensor_backend(ctx->sched_dec_pre, suffix_ids, ctx->backend); + + ggml_tensor * prefix_emb = ggml_get_rows(gctx, model->tok_embeddings_weight, prefix_ids); // [dec_dim, n_prefix] + ggml_tensor * audio_emb = ggml_cont(gctx, ggml_view_2d(gctx, ctx->decoder_memory, + model->hp.dec_dim, n_audio, ctx->decoder_memory->nb[1], 0)); // [dec_dim, n_audio] + ggml_tensor * suffix_emb = ggml_get_rows(gctx, model->tok_embeddings_weight, suffix_ids); // [dec_dim, n_suffix] + ggml_tensor * x = ggml_concat(gctx, prefix_emb, audio_emb, 1); + x = ggml_concat(gctx, x, suffix_emb, 1); // [dec_dim, n_tokens] + + ggml_tensor * positions = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, n_tokens); + ggml_set_name(positions, "positions"); + ggml_backend_sched_set_tensor_backend(ctx->sched_dec_pre, positions, ctx->backend); + + ggml_tensor * causal_mask = ggml_new_tensor_2d(gctx, GGML_TYPE_F32, n_tokens, n_tokens); + ggml_set_name(causal_mask, "causal_mask"); + ggml_backend_sched_set_tensor_backend(ctx->sched_dec_pre, causal_mask, ctx->backend); + + for (int32_t i = 0; i < model->hp.dec_layers; i++) { + x = build_decoder_layer(ctx, gctx, gf, x, positions, /*time_emb=*/nullptr, + i, n_tokens, /*kv_offset=*/0, causal_mask); + } + offline_logits_tail(ctx, gctx, gf, x, n_tokens); + return gf; +} + +// Single-token autoregressive step (no audio embedding). +static ggml_cgraph * build_offline_step_graph(voxtral_context * ctx, ggml_context * gctx) { + voxtral_model * model = ctx->model; + ggml_cgraph * gf = ggml_new_graph_custom(gctx, GGML_DEFAULT_GRAPH_SIZE * 4, false); + const int32_t kv_used = ctx->kv_used; + + ggml_tensor * token_id = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, 1); + ggml_set_name(token_id, "token_id"); + ggml_backend_sched_set_tensor_backend(ctx->sched_dec_step, token_id, ctx->backend); + ggml_tensor * pos_tensor = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, 1); + ggml_set_name(pos_tensor, "position"); + ggml_backend_sched_set_tensor_backend(ctx->sched_dec_step, pos_tensor, ctx->backend); + + ggml_tensor * x = ggml_get_rows(gctx, model->tok_embeddings_weight, token_id); // [dec_dim, 1] + for (int32_t i = 0; i < model->hp.dec_layers; i++) { + x = build_decoder_layer(ctx, gctx, gf, x, pos_tensor, /*time_emb=*/nullptr, + i, 1, /*kv_offset=*/kv_used, /*attn_mask=*/nullptr); + } + offline_logits_tail(ctx, gctx, gf, x, 1); return gf; } @@ -1638,13 +2063,65 @@ static ggml_tensor * find_tensor_in_graph(ggml_cgraph * gf, const char * name) { return ggml_graph_get_tensor(gf, name); } +// Set a named graph input tensor if it exists (no-op if absent). +static void set_graph_input(ggml_cgraph * gf, const char * name, const void * data, size_t bytes) { + if (ggml_tensor * t = find_tensor_in_graph(gf, name)) { + ggml_backend_tensor_set(t, data, 0, bytes); + } +} + +// Allocate a no_alloc graph context backed by `buf` (resized as needed). The +// graph_mult scales the default node budget (step graphs use 4, larger prefill 8). +static ggml_context * init_graph_ctx(std::vector & buf, int32_t graph_mult) { + const size_t meta = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE * graph_mult + + ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE * graph_mult, false); + if (buf.size() < meta) buf.resize(meta); + ggml_init_params p = { meta, buf.data(), /*.no_alloc=*/ true }; + return ggml_init(p); +} + +// Reset + allocate the graph on `sched`, run `set_inputs(gf)`, compute, then reset +// and free `gctx`. Readbacks target persistent tensors, so the caller reads them +// after this returns. Returns false (and frees gctx) on allocation failure. +template +static bool run_graph(voxtral_context * ctx, ggml_backend_sched_t sched, + ggml_context * gctx, ggml_cgraph * gf, + SetInputs && set_inputs, const char * what) { + ggml_backend_sched_reset(sched); + if (!ggml_backend_sched_alloc_graph(sched, gf)) { + LOG_ERR(ctx, "%s: failed to allocate graph", what); + ggml_free(gctx); + return false; + } + set_inputs(gf); + ggml_backend_sched_graph_compute(sched, gf); + ggml_backend_sched_reset(sched); + ggml_free(gctx); + return true; +} + +// Fill an [n, n] lower-triangular causal mask (0 = attend, -inf = masked). +static void fill_causal_mask(std::vector & mask, int32_t n) { + mask.assign((size_t) n * n, 0.0f); + for (int32_t i = 0; i < n; ++i) { + for (int32_t j = i + 1; j < n; ++j) { + mask[(size_t) i * n + j] = -INFINITY; + } + } +} + // ============================================================================ // Run Encoder // ============================================================================ -static bool run_encoder(voxtral_context * ctx, const float * mel_data, int32_t n_frames) { - LOG_INFO(ctx, "running encoder: %d mel frames", n_frames); - +// Run a single encoder chunk: build graph, set inputs, compute, return seq_len +static bool run_encoder_chunk( + voxtral_context * ctx, + const float * chunk_mel_data, // [n_mel, chunk_mel_frames] + int32_t chunk_mel_frames, + int32_t rope_pos_offset, + int32_t * out_seq_len) +{ const size_t meta_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE * 4 + ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE * 4, false); std::vector meta_buf(meta_size); @@ -1656,65 +2133,57 @@ static bool run_encoder(voxtral_context * ctx, const float * mel_data, int32_t n }; ggml_context * gctx = ggml_init(p); - ggml_cgraph * gf = build_encoder_graph(ctx, gctx, mel_data, n_frames); - log_graph_info(ctx, "encoder", gf); + int32_t chunk_seq_len = 0; + ggml_cgraph * gf = build_encoder_graph(ctx, gctx, chunk_mel_data, chunk_mel_frames, &chunk_seq_len); - // Allocate - LOG_DBG(ctx, "Allocating scheduler for the encoder"); ggml_backend_sched_reset(ctx->sched_encoder); if (!ggml_backend_sched_alloc_graph(ctx->sched_encoder, gf)) { - LOG_ERR(ctx, "encoder: failed to allocate graph"); + LOG_ERR(ctx, "encoder chunk: failed to allocate graph"); ggml_free(gctx); return false; } - // Set input data: mel_input + // Set mel input ggml_tensor * mel_t = find_tensor_in_graph(gf, "mel_input"); if (mel_t) { - log_tensor_info(ctx, "mel_input(runtime)", mel_t); - const int64_t expected_ne0 = n_frames; // length - const int64_t expected_ne1 = VOXTRAL_NUM_MEL_BINS; // channels + const int64_t expected_ne0 = chunk_mel_frames; + const int64_t expected_ne1 = VOXTRAL_NUM_MEL_BINS; if (mel_t->ne[0] == expected_ne0 && mel_t->ne[1] == expected_ne1) { - // mel_data layout [n_mel, n_frames] matches ggml [length, channels] memory order. - ggml_backend_tensor_set(mel_t, mel_data, 0, - (size_t) VOXTRAL_NUM_MEL_BINS * n_frames * sizeof(float)); + ggml_backend_tensor_set(mel_t, chunk_mel_data, 0, + (size_t) VOXTRAL_NUM_MEL_BINS * chunk_mel_frames * sizeof(float)); } else if (mel_t->ne[0] == expected_ne1 && mel_t->ne[1] == expected_ne0) { - // If tensor expects [n_mel, n_frames], transpose from mel_data. - std::vector mel_tbuf((size_t) n_frames * VOXTRAL_NUM_MEL_BINS); + std::vector mel_tbuf((size_t) chunk_mel_frames * VOXTRAL_NUM_MEL_BINS); for (int32_t m = 0; m < VOXTRAL_NUM_MEL_BINS; ++m) { - const float * src = mel_data + (size_t) m * n_frames; - for (int32_t f = 0; f < n_frames; ++f) { + const float * src = chunk_mel_data + (size_t) m * chunk_mel_frames; + for (int32_t f = 0; f < chunk_mel_frames; ++f) { mel_tbuf[(size_t) m + (size_t) VOXTRAL_NUM_MEL_BINS * f] = src[f]; } } ggml_backend_tensor_set(mel_t, mel_tbuf.data(), 0, - (size_t) VOXTRAL_NUM_MEL_BINS * n_frames * sizeof(float)); + (size_t) VOXTRAL_NUM_MEL_BINS * chunk_mel_frames * sizeof(float)); } else { - // Fallback: assume mel_data layout matches tensor shape and size - ggml_backend_tensor_set(mel_t, mel_data, 0, - (size_t) VOXTRAL_NUM_MEL_BINS * n_frames * sizeof(float)); + ggml_backend_tensor_set(mel_t, chunk_mel_data, 0, + (size_t) VOXTRAL_NUM_MEL_BINS * chunk_mel_frames * sizeof(float)); } } - // Set positions: 0, 1, 2, ..., seq_len-1 + // Set positions with RoPE offset for absolute positions across chunks ggml_tensor * pos_t = find_tensor_in_graph(gf, "enc_positions"); if (pos_t) { - const int32_t seq_len = ctx->enc_seq_used; - std::vector pos(seq_len); - std::iota(pos.begin(), pos.end(), 0); - ggml_backend_tensor_set(pos_t, pos.data(), 0, seq_len * sizeof(int32_t)); + std::vector pos(chunk_seq_len); + std::iota(pos.begin(), pos.end(), rope_pos_offset); + ggml_backend_tensor_set(pos_t, pos.data(), 0, chunk_seq_len * sizeof(int32_t)); } - // Set encoder sliding causal mask + // Set encoder sliding causal mask (local to chunk) ggml_tensor * mask_t = find_tensor_in_graph(gf, "enc_attn_mask"); if (mask_t) { - const int32_t seq_len = ctx->enc_seq_used; - std::vector mask((size_t) seq_len * seq_len); - for (int32_t q = 0; q < seq_len; ++q) { + std::vector mask((size_t) chunk_seq_len * chunk_seq_len); + for (int32_t q = 0; q < chunk_seq_len; ++q) { const int32_t min_kv = std::max(0, q - (VOXTRAL_ENC_WINDOW - 1)); - for (int32_t kv = 0; kv < seq_len; ++kv) { + for (int32_t kv = 0; kv < chunk_seq_len; ++kv) { const bool allow = (kv <= q) && (kv >= min_kv); - mask[(size_t) q * seq_len + kv] = allow ? 0.0f : -INFINITY; + mask[(size_t) q * chunk_seq_len + kv] = allow ? 0.0f : -INFINITY; } } ggml_backend_tensor_set(mask_t, mask.data(), 0, mask.size() * sizeof(float)); @@ -1725,7 +2194,150 @@ static bool run_encoder(voxtral_context * ctx, const float * mel_data, int32_t n ggml_backend_sched_reset(ctx->sched_encoder); ggml_free(gctx); - LOG_INFO(ctx, "encoder done: enc_seq_used=%d", ctx->enc_seq_used); + if (out_seq_len) *out_seq_len = chunk_seq_len; + return true; +} + +// Process mel spectrogram in overlapping chunks, accumulating encoder output on device +// Encoder token count for the offline (symmetric-conv) Whisper encoder. +static int32_t mel_frames_to_enc_tokens_sym(int32_t n_frames) { + const int32_t c0 = n_frames; // conv0 k3 s1 pad1 -> same length + const int32_t c1 = (c0 - 1) / 2 + 1; // conv1 k3 s2 pad1 -> (n+2-3)/2 + 1 + return c1 - (c1 % VOXTRAL_DOWNSAMPLE_FACTOR); +} + +// Offline encoder: single full-window pass (<=30s), non-causal, no chunk overlap. +static bool run_encoder_offline(voxtral_context * ctx, const float * mel_data, int32_t total_mel_frames) { + int32_t est = mel_frames_to_enc_tokens_sym(total_mel_frames); + if (est <= 0) { LOG_ERR(ctx, "encoder offline: audio too short"); return false; } + if (est > VOXTRAL_MAX_ENC_CHUNK) est = VOXTRAL_MAX_ENC_CHUNK; + if (!alloc_encoder_output(ctx, est)) { + LOG_ERR(ctx, "encoder offline: failed to allocate output (%d tokens)", est); + return false; + } + int32_t seq_len = 0; + if (!run_encoder_chunk(ctx, mel_data, total_mel_frames, /*rope_pos_offset=*/0, &seq_len)) { + return false; + } + if (seq_len > est) seq_len = est; + const size_t bytes = (size_t) seq_len * VOXTRAL_ENC_DIM * sizeof(float); + std::vector tmp(bytes); + ggml_backend_tensor_get(ctx->encoder_chunk_output, tmp.data(), 0, bytes); + ggml_backend_tensor_set(ctx->encoder_output, tmp.data(), 0, bytes); + // Trim to a multiple of the downsample factor for the adapter. + ctx->enc_seq_used = (seq_len / VOXTRAL_DOWNSAMPLE_FACTOR) * VOXTRAL_DOWNSAMPLE_FACTOR; + ctx->total_enc_tokens = ctx->enc_seq_used; + LOG_INFO(ctx, "encoder offline: %d mel frames -> %d enc tokens", total_mel_frames, ctx->enc_seq_used); + return true; +} + +static bool run_encoder_chunked(voxtral_context * ctx, const float * mel_data, int32_t total_mel_frames) { + const int32_t mel_overlap = VOXTRAL_ENC_CHUNK_OVERLAP * 2; // mel frames of overlap (1500) + const int32_t mel_stride = VOXTRAL_ENC_CHUNK_MEL - mel_overlap; // 1500 + + // Pre-compute total encoder tokens for allocation + int32_t alloc_total = compute_total_enc_tokens(total_mel_frames); + if (alloc_total <= 0) { + LOG_ERR(ctx, "encoder: audio too short to produce encoder tokens"); + return false; + } + + // Allocate encoder_output on device + if (!alloc_encoder_output(ctx, alloc_total)) { + LOG_ERR(ctx, "encoder: failed to allocate encoder output (%d tokens, %.2f MB)", + alloc_total, (double) alloc_total * VOXTRAL_ENC_DIM * sizeof(float) / 1e6); + return false; + } + + LOG_INFO(ctx, "encoder chunked: %d mel frames, %d alloc enc tokens, mel_stride=%d", + total_mel_frames, alloc_total, mel_stride); + + int32_t mel_offset = 0; + int32_t enc_write_offset = 0; + int32_t chunk_idx = 0; + + while (mel_offset < total_mel_frames) { + int32_t chunk_mel_frames = std::min(VOXTRAL_ENC_CHUNK_MEL, total_mel_frames - mel_offset); + + // Pre-check: will this chunk contribute any new tokens? + // This avoids building and running the full encoder graph for nothing. + int32_t skip = (chunk_idx > 0) ? VOXTRAL_ENC_CHUNK_OVERLAP : 0; + { + int32_t expected_tokens = mel_frames_to_enc_tokens(chunk_mel_frames); + if (expected_tokens - skip <= 0) { + LOG_DBG(ctx, "encoder chunk %d: skipped (expected %d tokens, skip=%d)", + chunk_idx, expected_tokens, skip); + break; + } + } + + // For single-chunk case (entire mel fits), use mel_data directly to avoid copy + const float * chunk_mel_ptr = nullptr; + std::vector chunk_mel_buf; + if (mel_offset == 0 && chunk_mel_frames == total_mel_frames) { + // Single chunk — mel_data is already in [n_mel, total_frames] layout + chunk_mel_ptr = mel_data; + } else { + // Multi-chunk — extract sub-range of frames for this chunk + chunk_mel_buf.resize((size_t) VOXTRAL_NUM_MEL_BINS * chunk_mel_frames); + for (int32_t m = 0; m < VOXTRAL_NUM_MEL_BINS; m++) { + memcpy(chunk_mel_buf.data() + (size_t) m * chunk_mel_frames, + mel_data + (size_t) m * total_mel_frames + mel_offset, + chunk_mel_frames * sizeof(float)); + } + chunk_mel_ptr = chunk_mel_buf.data(); + } + + int32_t rope_offset = enc_write_offset - skip; + + // Run encoder for this chunk + int32_t chunk_seq_len = 0; + if (!run_encoder_chunk(ctx, chunk_mel_ptr, chunk_mel_frames, rope_offset, &chunk_seq_len)) { + LOG_ERR(ctx, "encoder chunk %d: failed", chunk_idx); + return false; + } + + int32_t stride = chunk_seq_len - skip; + if (stride <= 0) { + LOG_DBG(ctx, "encoder chunk %d: no new tokens (seq_len=%d, skip=%d), stopping", + chunk_idx, chunk_seq_len, skip); + break; + } + + // Clamp stride to not overflow pre-allocated buffer + if (enc_write_offset + stride > alloc_total) { + stride = alloc_total - enc_write_offset; + if (stride <= 0) break; + } + + LOG_INFO(ctx, "encoder chunk %d: mel[%d..%d) enc_tokens=%d skip=%d stride=%d rope_offset=%d", + chunk_idx, mel_offset, mel_offset + chunk_mel_frames, + chunk_seq_len, skip, stride, rope_offset); + + // Copy stride portion from encoder_chunk_output to encoder_output + // Goes through CPU (device->CPU->device) + { + const size_t elem_bytes = VOXTRAL_ENC_DIM * sizeof(float); + const size_t src_offset = (size_t) skip * elem_bytes; + const size_t dst_offset = (size_t) enc_write_offset * elem_bytes; + const size_t copy_bytes = (size_t) stride * elem_bytes; + + std::vector tmp(copy_bytes); + ggml_backend_tensor_get(ctx->encoder_chunk_output, tmp.data(), src_offset, copy_bytes); + ggml_backend_tensor_set(ctx->encoder_output, tmp.data(), dst_offset, copy_bytes); + } + + enc_write_offset += stride; + mel_offset += mel_stride; + chunk_idx++; + } + + // Trim to multiple of downsample factor for adapter compatibility + ctx->enc_seq_used = (enc_write_offset / VOXTRAL_DOWNSAMPLE_FACTOR) * VOXTRAL_DOWNSAMPLE_FACTOR; + ctx->total_enc_tokens = ctx->enc_seq_used; + + LOG_INFO(ctx, "encoder done: %d chunks, enc_seq_used=%d (raw=%d)", + chunk_idx, ctx->enc_seq_used, enc_write_offset); return true; } @@ -1734,7 +2346,17 @@ static bool run_encoder(voxtral_context * ctx, const float * mel_data, int32_t n // ============================================================================ static bool run_adapter(voxtral_context * ctx) { - LOG_INFO(ctx, "running adapter"); + const int32_t enc_seq = ctx->enc_seq_used; + const int32_t dec_seq = enc_seq / VOXTRAL_DOWNSAMPLE_FACTOR; + + LOG_INFO(ctx, "running adapter: enc_seq=%d -> dec_seq=%d", enc_seq, dec_seq); + + // Allocate decoder_memory for this utterance + if (!alloc_decoder_memory(ctx, dec_seq)) { + LOG_ERR(ctx, "adapter: failed to allocate decoder memory (%d tokens, %.2f MB)", + dec_seq, (double) dec_seq * VOXTRAL_DEC_DIM * sizeof(float) / 1e6); + return false; + } const size_t meta_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, false); @@ -1761,7 +2383,9 @@ static bool run_adapter(voxtral_context * ctx) { ggml_backend_sched_reset(ctx->sched_adapter); ggml_free(gctx); - LOG_INFO(ctx, "adapter done: dec_seq_len=%d", ctx->dec_seq_len); + LOG_INFO(ctx, "adapter done: dec_seq_len=%d (%.2f MB on device)", + ctx->dec_seq_len, + (double) ggml_nbytes(ctx->decoder_memory) / 1e6); return true; } @@ -1782,68 +2406,27 @@ static bool run_decoder_prefill( return false; } - const size_t meta_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE * 4 + - ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE * 4, false); - std::vector meta_buf(meta_size); - - ggml_init_params p = { - /*.mem_size =*/ meta_size, - /*.mem_buffer=*/ meta_buf.data(), - /*.no_alloc =*/ true, - }; - ggml_context * gctx = ggml_init(p); - + static thread_local std::vector meta_buf; + ggml_context * gctx = init_graph_ctx(meta_buf, 4); ggml_cgraph * gf = build_decoder_prefill_graph(ctx, gctx, n_tokens); log_graph_info(ctx, "decoder prefill", gf); - ggml_backend_sched_reset(ctx->sched_dec_pre); - if (!ggml_backend_sched_alloc_graph(ctx->sched_dec_pre, gf)) { - LOG_ERR(ctx, "decoder prefill: failed to allocate graph"); - ggml_free(gctx); - return false; - } - - // Set inputs - ggml_tensor * tok_t = find_tensor_in_graph(gf, "token_ids"); - if (tok_t) { - ggml_backend_tensor_set(tok_t, token_ids, 0, n_tokens * sizeof(int32_t)); - } - - ggml_tensor * pos_t = find_tensor_in_graph(gf, "positions"); - if (pos_t) { + if (!run_graph(ctx, ctx->sched_dec_pre, gctx, gf, [&](ggml_cgraph * g) { + set_graph_input(g, "token_ids", token_ids, n_tokens * sizeof(int32_t)); std::vector pos(n_tokens); std::iota(pos.begin(), pos.end(), 0); - ggml_backend_tensor_set(pos_t, pos.data(), 0, n_tokens * sizeof(int32_t)); - } - - ggml_tensor * time_t = find_tensor_in_graph(gf, "time_emb"); - if (time_t) { - ggml_backend_tensor_set(time_t, ctx->time_emb_cpu.data(), 0, VOXTRAL_DEC_DIM * sizeof(float)); - } - - // Set causal mask: lower-triangular (0 for allowed, -inf for masked) - ggml_tensor * mask_t = find_tensor_in_graph(gf, "causal_mask"); - if (mask_t) { - std::vector mask(n_tokens * n_tokens); - for (int32_t i = 0; i < n_tokens; i++) { - for (int32_t j = 0; j < n_tokens; j++) { - mask[i * n_tokens + j] = (j <= i) ? 0.0f : -INFINITY; - } - } - ggml_backend_tensor_set(mask_t, mask.data(), 0, mask.size() * sizeof(float)); + set_graph_input(g, "positions", pos.data(), n_tokens * sizeof(int32_t)); + set_graph_input(g, "time_emb", ctx->time_emb_cpu.data(), VOXTRAL_DEC_DIM * sizeof(float)); + std::vector mask; + fill_causal_mask(mask, n_tokens); + set_graph_input(g, "causal_mask", mask.data(), mask.size() * sizeof(float)); + }, "decoder prefill")) { + return false; } - // Compute - ggml_backend_sched_graph_compute(ctx->sched_dec_pre, gf); - - // Read logits ggml_backend_tensor_get(ctx->decoder_logits, logits_out, 0, VOXTRAL_VOCAB_SIZE * sizeof(float)); - ctx->kv_used = std::min(n_tokens, VOXTRAL_DEC_WINDOW); - ggml_backend_sched_reset(ctx->sched_dec_pre); - ggml_free(gctx); - LOG_INFO(ctx, "decoder prefill done"); return true; } @@ -1857,61 +2440,204 @@ static bool run_decoder_step( int32_t token_id, int32_t position, // absolute position in decoder sequence int32_t audio_pos, // position in adapter output for audio embedding - float * logits_out) // [vocab_size] + float * logits_out, // [vocab_size] — optional, full logits readback + int32_t * token_out) // optional — greedy argmax token (cheap readback) { if (ctx->kv_used >= VOXTRAL_DEC_WINDOW) { kv_cache_shift_left(ctx, 1); ctx->kv_used = VOXTRAL_DEC_WINDOW - 1; } - const size_t meta_size = ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE * 4 + - ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE * 4, false); - std::vector meta_buf(meta_size); - - ggml_init_params p = { - /*.mem_size =*/ meta_size, - /*.mem_buffer=*/ meta_buf.data(), - /*.no_alloc =*/ true, - }; - ggml_context * gctx = ggml_init(p); - + static thread_local std::vector step_meta_buf; + ggml_context * gctx = init_graph_ctx(step_meta_buf, 4); ggml_cgraph * gf = build_decoder_step_graph(ctx, gctx, position, audio_pos); - log_graph_info(ctx, "decoder step", gf); - ggml_backend_sched_reset(ctx->sched_dec_step); - if (!ggml_backend_sched_alloc_graph(ctx->sched_dec_step, gf)) { - LOG_ERR(ctx, "decoder step: failed to allocate graph"); - ggml_free(gctx); + if (!run_graph(ctx, ctx->sched_dec_step, gctx, gf, [&](ggml_cgraph * g) { + set_graph_input(g, "token_id", &token_id, sizeof(int32_t)); + set_graph_input(g, "position", &position, sizeof(int32_t)); + set_graph_input(g, "time_emb", ctx->time_emb_cpu.data(), VOXTRAL_DEC_DIM * sizeof(float)); + }, "decoder step")) { return false; } - // Set inputs - ggml_tensor * tok_t = find_tensor_in_graph(gf, "token_id"); - if (tok_t) { - ggml_backend_tensor_set(tok_t, &token_id, 0, sizeof(int32_t)); + // Read back the greedy token (4 bytes) and/or full logits if requested. + if (token_out) { + ggml_backend_tensor_get(ctx->decoder_argmax, token_out, 0, sizeof(int32_t)); } - - ggml_tensor * pos_t = find_tensor_in_graph(gf, "position"); - if (pos_t) { - ggml_backend_tensor_set(pos_t, &position, 0, sizeof(int32_t)); + if (logits_out) { + ggml_backend_tensor_get(ctx->decoder_logits, logits_out, 0, VOXTRAL_VOCAB_SIZE * sizeof(float)); } - ggml_tensor * time_t = find_tensor_in_graph(gf, "time_emb"); - if (time_t) { - ggml_backend_tensor_set(time_t, ctx->time_emb_cpu.data(), 0, VOXTRAL_DEC_DIM * sizeof(float)); + ctx->kv_used += 1; + return true; +} + +// Offline prefill: prefix tokens + audio embeddings + suffix tokens in one graph. +static bool run_offline_prefill(voxtral_context * ctx, + const int32_t * prefix_ids, int32_t n_prefix, + int32_t n_audio, + const int32_t * suffix_ids, int32_t n_suffix, + int32_t * token_out, float * logits_out) { + const int32_t n_tokens = n_prefix + n_audio + n_suffix; + static thread_local std::vector meta_buf; + ggml_context * gctx = init_graph_ctx(meta_buf, 8); + ggml_cgraph * gf = build_offline_prefill_graph(ctx, gctx, n_prefix, n_audio, n_suffix); + + if (!run_graph(ctx, ctx->sched_dec_pre, gctx, gf, [&](ggml_cgraph * g) { + set_graph_input(g, "prefix_ids", prefix_ids, n_prefix * sizeof(int32_t)); + set_graph_input(g, "suffix_ids", suffix_ids, n_suffix * sizeof(int32_t)); + std::vector pos(n_tokens); + std::iota(pos.begin(), pos.end(), 0); + set_graph_input(g, "positions", pos.data(), n_tokens * sizeof(int32_t)); + std::vector mask; + fill_causal_mask(mask, n_tokens); + set_graph_input(g, "causal_mask", mask.data(), mask.size() * sizeof(float)); + }, "offline prefill")) { + return false; } - // Compute - ggml_backend_sched_graph_compute(ctx->sched_dec_step, gf); + if (token_out) ggml_backend_tensor_get(ctx->decoder_argmax, token_out, 0, sizeof(int32_t)); + if (logits_out) ggml_backend_tensor_get(ctx->decoder_logits, logits_out, 0, VOXTRAL_VOCAB_SIZE * sizeof(float)); + ctx->kv_used = n_tokens; + return true; +} - // Read logits - ggml_backend_tensor_get(ctx->decoder_logits, logits_out, 0, VOXTRAL_VOCAB_SIZE * sizeof(float)); +// Offline single-token step. +static bool run_offline_step(voxtral_context * ctx, int32_t token_id, int32_t position, int32_t * token_out) { + static thread_local std::vector step_meta_buf; + ggml_context * gctx = init_graph_ctx(step_meta_buf, 4); + ggml_cgraph * gf = build_offline_step_graph(ctx, gctx); + if (!run_graph(ctx, ctx->sched_dec_step, gctx, gf, [&](ggml_cgraph * g) { + set_graph_input(g, "token_id", &token_id, sizeof(int32_t)); + set_graph_input(g, "position", &position, sizeof(int32_t)); + }, "offline step")) { + return false; + } + + if (token_out) ggml_backend_tensor_get(ctx->decoder_argmax, token_out, 0, sizeof(int32_t)); ctx->kv_used += 1; + return true; +} - ggml_backend_sched_reset(ctx->sched_dec_step); - ggml_free(gctx); +// ============================================================================ +// High-level: Transcribe (offline Voxtral-Mini-3B-2507) +// ============================================================================ + +static constexpr int32_t VOXTRAL_OFFLINE_WINDOW_SEC = 30; +static constexpr int32_t VOXTRAL_OFFLINE_MAX_DECODE = 448; + +static void compute_mel_even(voxtral_context & ctx, const float * samples, int32_t n_samples, + std::vector & mel_data, int32_t & n_frames) { + const int32_t max_frames = n_samples / VOXTRAL_HOP_LENGTH + 1; + mel_data.assign((size_t) VOXTRAL_NUM_MEL_BINS * max_frames, 0.0f); + n_frames = 0; + compute_mel_spectrogram(samples, n_samples, ctx.mel_filters_cpu.data(), + ctx.hann_window.data(), mel_data.data(), &n_frames); + if (n_frames % 2 != 0) { + for (int32_t m = 0; m < VOXTRAL_NUM_MEL_BINS; m++) + memmove(mel_data.data() + (size_t) m * (n_frames - 1), + mel_data.data() + (size_t) m * n_frames + 1, + (size_t) (n_frames - 1) * sizeof(float)); + n_frames -= 1; + } +} +static bool transcribe_offline_window( + voxtral_context & ctx, const float * wav, int32_t n_wav, + int32_t max_tokens, float * first_logits_or_null, + std::vector & out_tokens) +{ + const int32_t win = VOXTRAL_OFFLINE_WINDOW_SEC * VOXTRAL_SAMPLE_RATE; + std::vector padded((size_t) win, 0.0f); + const int32_t ncopy = std::min(n_wav, win); + memcpy(padded.data(), wav, (size_t) ncopy * sizeof(float)); + + int32_t n_frames = 0; + std::vector mel_data; + compute_mel_even(ctx, padded.data(), win, mel_data, n_frames); + + if (!run_encoder_offline(&ctx, mel_data.data(), n_frames)) return false; + if (!run_adapter(&ctx)) return false; + + const auto & hp = ctx.model->hp; + const int32_t n_audio = ctx.dec_seq_len; + // Prompt: [INST][BEGIN_AUDIO] [AUDIO]xN [/INST] lang:en [TRANSCRIBE] + // The middle three suffix ids are the tekken ranks for the text "lang:en" + // (language is currently fixed to English). + static const int32_t kLangEn[3] = { 9909, 1058, 1262 }; + const int32_t prefix[3] = { hp.tok_bos, hp.tok_inst, hp.tok_begin_audio }; + const int32_t suffix[5] = { hp.tok_inst_end, kLangEn[0], kLangEn[1], kLangEn[2], hp.tok_transcribe }; + const int32_t L = 3 + n_audio + 5; + + clear_kv_cache(&ctx); + int32_t token = 0; + if (!run_offline_prefill(&ctx, prefix, 3, n_audio, suffix, 5, &token, first_logits_or_null)) return false; + if (token != VOXTRAL_TOKEN_EOS) out_tokens.push_back(token); + + const int32_t cap = (max_tokens > 0) ? max_tokens : VOXTRAL_OFFLINE_MAX_DECODE; + for (int32_t i = 0; i < cap && token != VOXTRAL_TOKEN_EOS; ++i) { + if (!run_offline_step(&ctx, token, L + i, &token)) return false; + if (token == VOXTRAL_TOKEN_EOS) break; + out_tokens.push_back(token); + } + return true; +} + +static std::string trim_copy(const std::string & s) { + size_t a = s.find_first_not_of(" \t\r\n"); + if (a == std::string::npos) return ""; + size_t b = s.find_last_not_of(" \t\r\n"); + return s.substr(a, b - a + 1); +} + +static bool voxtral_transcribe_offline( + voxtral_context & ctx, + const float * audio, + int32_t n_samples, + int32_t max_tokens, + voxtral_result & result) +{ + const int32_t win = VOXTRAL_OFFLINE_WINDOW_SEC * VOXTRAL_SAMPLE_RATE; + int32_t n_windows = (n_samples + win - 1) / win; + if (n_windows < 1) n_windows = 1; + LOG_INFO(&ctx, "offline: %d samples (%.1fs) -> %d window(s) of %ds", + n_samples, (float) n_samples / VOXTRAL_SAMPLE_RATE, n_windows, VOXTRAL_OFFLINE_WINDOW_SEC); + + auto t_all = std::chrono::steady_clock::now(); + std::string full; + int32_t total_steps = 0; + for (int32_t w = 0; w < n_windows; ++w) { + const int32_t start = w * win; + const int32_t len = std::min(win, n_samples - start); + if (len <= 0) break; + + std::vector wtoks; + // Capture the first window's prefill logits for diagnostics (--dump-logits). + float * flog = nullptr; + if (w == 0) { + result.first_step_logits.resize(VOXTRAL_VOCAB_SIZE); + flog = result.first_step_logits.data(); + } + auto tw = std::chrono::steady_clock::now(); + if (!transcribe_offline_window(ctx, audio + start, len, max_tokens, flog, wtoks)) return false; + total_steps += (int32_t) wtoks.size(); + + std::string wtext = trim_copy(decode_tokens(*ctx.model, wtoks)); + LOG_INFO(&ctx, "window %d/%d: %.1f-%.1fs -> %d tokens, %.0f ms", + w + 1, n_windows, (float) start / VOXTRAL_SAMPLE_RATE, + (float) (start + len) / VOXTRAL_SAMPLE_RATE, (int) wtoks.size(), elapsed_ms(tw)); + + result.tokens.insert(result.tokens.end(), wtoks.begin(), wtoks.end()); + if (!wtext.empty()) { + if (!full.empty()) full += " "; + full += wtext; + } + } + result.text = full; + const double ms = elapsed_ms(t_all); + LOG_INFO(&ctx, "offline total: %d windows, %d tokens, %.0f ms (RTF %.3f)", + n_windows, total_steps, ms, ms / 1000.0 / ((double) n_samples / VOXTRAL_SAMPLE_RATE)); return true; } @@ -1941,7 +2667,12 @@ static bool voxtral_transcribe_from_audio( (float)n_samples / VOXTRAL_SAMPLE_RATE); } - // 2. Streaming padding (matching Python pad_audio_streaming) + // Offline: different preprocessing, prompt and decode. + if (ctx.model->hp.is_offline) { + return voxtral_transcribe_offline(ctx, audio, n_samples, max_tokens, result); + } + + // Streaming padding (matching Python pad_audio_streaming) constexpr int32_t mult_of = VOXTRAL_RAW_AUDIO_LENGTH_PER_TOK; // 1280 const int32_t n_raw = n_samples; const int32_t align_pad = (mult_of - (n_raw % mult_of)) % mult_of; @@ -1953,45 +2684,29 @@ static bool voxtral_transcribe_from_audio( LOG_INFO(&ctx, "padded audio: %d samples (left=%d, right=%d)", (int)padded.size(), left_pad, right_pad); - // 3. Compute mel spectrogram + // Compute mel spectrogram (truncated to an even frame count for conv stride 2) int32_t n_frames = 0; - const int32_t max_frames = (int32_t)padded.size() / VOXTRAL_HOP_LENGTH + 1; - std::vector mel_data(VOXTRAL_NUM_MEL_BINS * max_frames); - - compute_mel_spectrogram( - padded.data(), (int32_t)padded.size(), - ctx.mel_filters_cpu.data(), - ctx.hann_window.data(), - mel_data.data(), &n_frames); - + std::vector mel_data; + compute_mel_even(ctx, padded.data(), (int32_t) padded.size(), mel_data, n_frames); LOG_INFO(&ctx, "mel spectrogram: %d frames", n_frames); - // Truncate to even number of frames (for conv stride=2) - if (n_frames % 2 != 0) { - // Drop first frame (matching Python mel[:, 1:]) - // Shift mel data left by 1 frame - for (int32_t m = 0; m < VOXTRAL_NUM_MEL_BINS; m++) { - memmove(mel_data.data() + m * (n_frames - 1), - mel_data.data() + m * n_frames + 1, - (n_frames - 1) * sizeof(float)); - } - n_frames -= 1; - LOG_INFO(&ctx, "mel truncated to %d frames (even)", n_frames); - } - - // 4. Run encoder - if (!run_encoder(&ctx, mel_data.data(), n_frames)) { + // Run encoder (chunked for arbitrarily long audio) + auto t_encoder = std::chrono::steady_clock::now(); + if (!run_encoder_chunked(&ctx, mel_data.data(), n_frames)) { return false; } + LOG_INFO(&ctx, "encoder time: %.1f ms", elapsed_ms(t_encoder)); - // 5. Run adapter + // Run adapter + auto t_adapter = std::chrono::steady_clock::now(); if (!run_adapter(&ctx)) { return false; } + LOG_INFO(&ctx, "adapter time: %.1f ms", elapsed_ms(t_adapter)); const int32_t n_audio = ctx.dec_seq_len; - // 6. Build prompt tokens: [BOS] + [STREAMING_PAD] * (N_LEFT_PAD_TOKENS + N_DELAY_TOKENS) + // Build prompt tokens: [BOS] + [STREAMING_PAD] * (N_LEFT_PAD_TOKENS + N_DELAY_TOKENS) std::vector prompt_ids; prompt_ids.push_back(VOXTRAL_TOKEN_BOS); for (int32_t i = 0; i < VOXTRAL_N_LEFT_PAD_TOKENS + VOXTRAL_N_DELAY_TOKENS; i++) { @@ -2006,10 +2721,11 @@ static bool voxtral_transcribe_from_audio( return false; } - // 7. Reset KV cache + // Reset KV cache clear_kv_cache(&ctx); - // 8. Decoder prefill + // Decoder prefill + auto t_prefill = std::chrono::steady_clock::now(); std::vector logits(VOXTRAL_VOCAB_SIZE); if (L > 1) { if (!run_decoder_prefill(&ctx, prompt_ids.data(), L - 1, logits.data())) { @@ -2017,20 +2733,13 @@ static bool voxtral_transcribe_from_audio( } } - // 8b. One step with last prefix token (matches Python prefill + forward_one) - if (!run_decoder_step(&ctx, prompt_ids[L - 1], L - 1, L - 1, logits.data())) { - return false; - } - - // First token from prefill + // One step with last prefix token + // Request full logits here so we can store first_step_logits for diagnostics. int32_t token = 0; - float max_logit = -INFINITY; - for (int32_t i = 0; i < VOXTRAL_VOCAB_SIZE; i++) { - if (logits[i] > max_logit) { - max_logit = logits[i]; - token = i; - } + if (!run_decoder_step(&ctx, prompt_ids[L - 1], L - 1, L - 1, logits.data(), &token)) { + return false; } + LOG_INFO(&ctx, "prefill time: %.1f ms", elapsed_ms(t_prefill)); // Store first step logits result.first_step_logits = logits; @@ -2038,26 +2747,34 @@ static bool voxtral_transcribe_from_audio( LOG_INFO(&ctx, "first token: %d", token); - // 9. Autoregressive decoding - for (int32_t pos = L; pos < n_audio && (int32_t)result.tokens.size() < max_tokens; pos++) { + // Autoregressive decoding + // + // The realtime model emits one token per audio frame. To transcribe the + // whole file we must decode every audio position and only stop on EOS (or + // the optional max_tokens cap, where <= 0 means until end of audio). + // + // we deliberately do NOT early-stop on a run of consecutive pad + // tokens. A pad run only means a pause in speech, which is indistinguishable + // from end-of-audio without lookahead, so a pad-based early stop truncates + // longform transcripts at the first pause. Trailing pads are filtered out + // during detokenization, so decoding to the end costs only the (bounded) + // trailing-silence frames. + const bool unlimited = (max_tokens <= 0); + auto t_decode = std::chrono::steady_clock::now(); + for (int32_t pos = L; pos < n_audio && (unlimited || (int32_t)result.tokens.size() < max_tokens); pos++) { if (token == VOXTRAL_TOKEN_EOS) break; - if (!run_decoder_step(&ctx, token, pos, pos, logits.data())) { + // Hot path: read back only the on-device greedy argmax token (4 bytes), + // not the full 131072-float logits vector. + if (!run_decoder_step(&ctx, token, pos, pos, /*logits_out=*/nullptr, &token)) { return false; } - // Greedy argmax - token = 0; - max_logit = -INFINITY; - for (int32_t i = 0; i < VOXTRAL_VOCAB_SIZE; i++) { - if (logits[i] > max_logit) { - max_logit = logits[i]; - token = i; - } - } - result.tokens.push_back(token); } + LOG_INFO(&ctx, "decode time: %.1f ms (%d steps, %.1f ms/step)", + elapsed_ms(t_decode), (int)result.tokens.size() - 1, + result.tokens.size() > 1 ? elapsed_ms(t_decode) / (result.tokens.size() - 1) : 0.0); // Remove trailing EOS if (!result.tokens.empty() && result.tokens.back() == VOXTRAL_TOKEN_EOS) { diff --git a/tools/convert_voxtral_to_gguf.py b/tools/convert_voxtral_to_gguf.py index 30fd91a..a5bb3f6 100755 --- a/tools/convert_voxtral_to_gguf.py +++ b/tools/convert_voxtral_to_gguf.py @@ -1,14 +1,5 @@ #!/usr/bin/env python3 -"""Convert Voxtral-Mini-4B-Realtime safetensors to GGUF. - -Features: -- deterministic tensor naming -- strict shape/count validation -- metadata export (architecture/hparams/tokenizer/checksums) -- --verify mode with safetensors + optional GGUF shape checks - -This implementation intentionally avoids PyTorch and the `gguf` python package. -""" +"""Convert Voxtral models safetensors to GGUF.""" from __future__ import annotations @@ -254,7 +245,32 @@ def _flatten_dict(prefix: str, obj: Dict[str, Any]) -> Dict[str, Any]: return out -def expected_tensor_specs(params: Dict[str, Any]) -> Dict[str, TensorSpec]: +# Embedding-module prefix differs between the two model families: +# realtime (Voxtral-Mini-4B-Realtime): mm_streams_embeddings.embedding_module. +# offline (Voxtral-Mini-3B-2507): mm_whisper_embeddings. +REALTIME_EMB_PREFIX = "mm_streams_embeddings.embedding_module." +OFFLINE_EMB_PREFIX = "mm_whisper_embeddings." + + +def detect_offline(source_names: Iterable[str]) -> bool: + return any(n.startswith(OFFLINE_EMB_PREFIX) for n in source_names) + + +def arch_prefixes(offline: bool) -> Tuple[str, str]: + """(embedding-module prefix, conv-layer name infix) for the model family. + + Offline (Whisper) conv tensors are conv_layers.N.weight; realtime are + conv_layers.N.conv.weight. + """ + return ( + OFFLINE_EMB_PREFIX if offline else REALTIME_EMB_PREFIX, + "" if offline else ".conv", + ) + + +def expected_tensor_specs(params: Dict[str, Any], offline: bool = False) -> Dict[str, TensorSpec]: + emb, conv_infix = arch_prefixes(offline) + has_ada = not offline enc = params["multimodal"]["whisper_model_args"]["encoder_args"] enc_dim = int(enc["dim"]) @@ -262,7 +278,7 @@ def expected_tensor_specs(params: Dict[str, Any]) -> Dict[str, TensorSpec]: enc_heads = int(enc["n_heads"]) enc_head_dim = int(enc["head_dim"]) enc_hidden = int(enc["hidden_dim"]) - enc_kv_heads = int(enc["n_kv_heads"]) + enc_kv_heads = int(enc.get("n_kv_heads", enc["n_heads"])) dec_dim = int(params["dim"]) dec_layers = int(params["n_layers"]) @@ -282,25 +298,23 @@ def expected_tensor_specs(params: Dict[str, Any]) -> Dict[str, TensorSpec]: def add(name: str, shape: Sequence[int]) -> None: s[name] = TensorSpec(name=name, shape=tuple(int(x) for x in shape)) + # Offline (true Whisper) encoder: conv tensors are conv_layers.N.weight (no + # ".conv." infix), norms are LayerNorm (weight+bias), FFN is a plain GELU MLP + # (w1/w2, no w3). Realtime encoder: conv_layers.N.conv.weight, RMSNorm (weight + # only), SwiGLU FFN (w1/w2/w3). add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.0.conv.weight", + f"{emb}whisper_encoder.conv_layers.0{conv_infix}.weight", (enc_dim, int(enc["audio_encoding_args"]["num_mel_bins"]), 3), ) + add(f"{emb}whisper_encoder.conv_layers.0{conv_infix}.bias", (enc_dim,)) add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.0.conv.bias", - (enc_dim,), - ) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.1.conv.weight", + f"{emb}whisper_encoder.conv_layers.1{conv_infix}.weight", (enc_dim, enc_dim, 3), ) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.1.conv.bias", - (enc_dim,), - ) + add(f"{emb}whisper_encoder.conv_layers.1{conv_infix}.bias", (enc_dim,)) for i in range(enc_layers): - p = f"mm_streams_embeddings.embedding_module.whisper_encoder.transformer.layers.{i}" + p = f"{emb}whisper_encoder.transformer.layers.{i}" add(f"{p}.attention_norm.weight", (enc_dim,)) add(f"{p}.attention.wq.weight", (enc_heads * enc_head_dim, enc_dim)) add(f"{p}.attention.wq.bias", (enc_heads * enc_head_dim,)) @@ -313,24 +327,32 @@ def add(name: str, shape: Sequence[int]) -> None: add(f"{p}.feed_forward.w1.weight", (enc_hidden, enc_dim)) add(f"{p}.feed_forward.w2.weight", (enc_dim, enc_hidden)) add(f"{p}.feed_forward.w2.bias", (enc_dim,)) - add(f"{p}.feed_forward.w3.weight", (enc_hidden, enc_dim)) + if offline: + # LayerNorm biases + MLP w1 bias (no SwiGLU gate) + add(f"{p}.attention_norm.bias", (enc_dim,)) + add(f"{p}.ffn_norm.bias", (enc_dim,)) + add(f"{p}.feed_forward.w1.bias", (enc_hidden,)) + else: + add(f"{p}.feed_forward.w3.weight", (enc_hidden, enc_dim)) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.transformer.norm.weight", - (enc_dim,), - ) + add(f"{emb}whisper_encoder.transformer.norm.weight", (enc_dim,)) + if offline: + add(f"{emb}whisper_encoder.transformer.norm.bias", (enc_dim,)) add( - "mm_streams_embeddings.embedding_module.audio_language_projection.0.weight", + f"{emb}audio_language_projection.0.weight", (dec_dim, enc_dim * downsample_factor), ) add( - "mm_streams_embeddings.embedding_module.audio_language_projection.2.weight", + f"{emb}audio_language_projection.2.weight", (dec_dim, dec_dim), ) - add("mm_streams_embeddings.embedding_module.tok_embeddings.weight", (dec_vocab, dec_dim)) + add(f"{emb}tok_embeddings.weight", (dec_vocab, dec_dim)) add("norm.weight", (dec_dim,)) + # Note: the offline untied output projection `output.weight` is handled via the + # optional-tensor list in convert()/verify() (it also exists for realtime), so it + # is intentionally NOT added here to avoid a duplicate tensor entry. for i in range(dec_layers): p = f"layers.{i}" @@ -343,8 +365,9 @@ def add(name: str, shape: Sequence[int]) -> None: add(f"{p}.feed_forward.w1.weight", (dec_hidden, dec_dim)) add(f"{p}.feed_forward.w2.weight", (dec_dim, dec_hidden)) add(f"{p}.feed_forward.w3.weight", (dec_hidden, dec_dim)) - add(f"{p}.ada_rms_norm_t_cond.0.weight", (ada_dim, dec_dim)) - add(f"{p}.ada_rms_norm_t_cond.2.weight", (dec_dim, ada_dim)) + if has_ada: + add(f"{p}.ada_rms_norm_t_cond.0.weight", (ada_dim, dec_dim)) + add(f"{p}.ada_rms_norm_t_cond.2.weight", (dec_dim, ada_dim)) return s @@ -416,6 +439,32 @@ def make_mel_filter_plan(params: Dict[str, Any]) -> Tuple[TensorPlan, str]: return plan, sha +def make_enc_pos_embedding_plan(params: Dict[str, Any]) -> TensorPlan: + """Whisper sinusoidal positional embeddings for the offline encoder. + + The Mistral consolidated.safetensors omits these (they are a fixed buffer); the + HF checkpoint stores them as audio_tower.embed_positions.weight, which matches the + standard Whisper sinusoids exactly. We generate them so no extra download is needed. + Stored as [length, channels] row-major -> ggml ne=[channels, length]. + """ + enc = params["multimodal"]["whisper_model_args"]["encoder_args"] + channels = int(enc["dim"]) # 1280 + length = int(enc["max_source_positions"]) # 1500 + log_inc = np.log(10000.0) / (channels // 2 - 1) + inv = np.exp(-log_inc * np.arange(channels // 2)) + scaled = np.arange(length)[:, None] * inv[None, :] + pos = np.concatenate([np.sin(scaled), np.cos(scaled)], axis=1).astype(np.float32) # [length, channels] + raw = pos.tobytes(order="C") + return TensorPlan( + name="enc.pos_embedding", + src=None, + ggml_type=GGMLType.F32, + dims=gguf_dims_from_shape((length, channels)), + out_nbytes=len(raw), + inline_data=raw, + ) + + def build_compact_name_map( params: Dict[str, Any], source_names: Iterable[str], @@ -423,6 +472,9 @@ def build_compact_name_map( src_set = set(source_names) out: Dict[str, str] = {} + offline = detect_offline(src_set) + emb, conv_infix = arch_prefixes(offline) + def add(src: str, dst: str) -> None: if src in src_set: if len(dst) >= 64: @@ -432,44 +484,26 @@ def add(src: str, dst: str) -> None: enc_layers = int(params["multimodal"]["whisper_model_args"]["encoder_args"]["n_layers"]) dec_layers = int(params["n_layers"]) - add("mm_streams_embeddings.embedding_module.tok_embeddings.weight", "tok_embeddings.weight") + add(f"{emb}tok_embeddings.weight", "tok_embeddings.weight") add("norm.weight", "norm.weight") - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.0.conv.weight", - "enc.conv0.weight", - ) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.0.conv.bias", - "enc.conv0.bias", - ) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.1.conv.weight", - "enc.conv1.weight", - ) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.1.conv.bias", - "enc.conv1.bias", - ) - add( - "mm_streams_embeddings.embedding_module.whisper_encoder.transformer.norm.weight", - "enc.norm.weight", - ) + add(f"{emb}whisper_encoder.conv_layers.0{conv_infix}.weight", "enc.conv0.weight") + add(f"{emb}whisper_encoder.conv_layers.0{conv_infix}.bias", "enc.conv0.bias") + add(f"{emb}whisper_encoder.conv_layers.1{conv_infix}.weight", "enc.conv1.weight") + add(f"{emb}whisper_encoder.conv_layers.1{conv_infix}.bias", "enc.conv1.bias") + add(f"{emb}whisper_encoder.transformer.norm.weight", "enc.norm.weight") + add(f"{emb}whisper_encoder.transformer.norm.bias", "enc.norm.bias") - add( - "mm_streams_embeddings.embedding_module.audio_language_projection.0.weight", - "adapter.0.weight", - ) - add( - "mm_streams_embeddings.embedding_module.audio_language_projection.2.weight", - "adapter.2.weight", - ) + add(f"{emb}audio_language_projection.0.weight", "adapter.0.weight") + add(f"{emb}audio_language_projection.2.weight", "adapter.2.weight") for i in range(enc_layers): - src = f"mm_streams_embeddings.embedding_module.whisper_encoder.transformer.layers.{i}" + src = f"{emb}whisper_encoder.transformer.layers.{i}" dst = f"enc.blk.{i}" add(f"{src}.attention_norm.weight", f"{dst}.attn_norm.weight") + add(f"{src}.attention_norm.bias", f"{dst}.attn_norm.bias") add(f"{src}.ffn_norm.weight", f"{dst}.ffn_norm.weight") + add(f"{src}.ffn_norm.bias", f"{dst}.ffn_norm.bias") add(f"{src}.attention.wq.weight", f"{dst}.attn_q.weight") add(f"{src}.attention.wq.bias", f"{dst}.attn_q.bias") add(f"{src}.attention.wk.weight", f"{dst}.attn_k.weight") @@ -478,6 +512,7 @@ def add(src: str, dst: str) -> None: add(f"{src}.attention.wo.weight", f"{dst}.attn_o.weight") add(f"{src}.attention.wo.bias", f"{dst}.attn_o.bias") add(f"{src}.feed_forward.w1.weight", f"{dst}.ffn_w1.weight") + add(f"{src}.feed_forward.w1.bias", f"{dst}.ffn_w1.bias") add(f"{src}.feed_forward.w2.weight", f"{dst}.ffn_w2.weight") add(f"{src}.feed_forward.w2.bias", f"{dst}.ffn_w2.bias") add(f"{src}.feed_forward.w3.weight", f"{dst}.ffn_w3.weight") @@ -494,12 +529,13 @@ def add(src: str, dst: str) -> None: add(f"{src}.feed_forward.w1.weight", f"{dst}.ffn_w1.weight") add(f"{src}.feed_forward.w2.weight", f"{dst}.ffn_w2.weight") add(f"{src}.feed_forward.w3.weight", f"{dst}.ffn_w3.weight") - add(f"{src}.ada_rms_norm_t_cond.0.weight", f"{dst}.ada0.weight") - add(f"{src}.ada_rms_norm_t_cond.2.weight", f"{dst}.ada2.weight") + if not offline: + add(f"{src}.ada_rms_norm_t_cond.0.weight", f"{dst}.ada0.weight") + add(f"{src}.ada_rms_norm_t_cond.2.weight", f"{dst}.ada2.weight") add("output.weight", "output.weight") add("output.bias", "output.bias") - add("mm_streams_embeddings.embedding_module.output.weight", "output_mm.weight") + add(f"{emb}output.weight", "output_mm.weight") missing_map = sorted(src_set - set(out.keys())) if missing_map: @@ -528,7 +564,9 @@ def validate_safetensors( *, verbose: bool = True, ) -> Tuple[List[str], Dict[str, str], Dict[str, TensorSpec]]: - expected = expected_tensor_specs(params) + offline = detect_offline(reader.keys()) + emb, conv_infix = arch_prefixes(offline) + expected = expected_tensor_specs(params, offline=offline) names = sorted(reader.keys()) actual = set(names) @@ -537,7 +575,7 @@ def validate_safetensors( optional = { "output.weight", "output.bias", - "mm_streams_embeddings.embedding_module.output.weight", + f"{emb}output.weight", } missing = sorted(mandatory - actual) @@ -562,11 +600,11 @@ def validate_safetensors( checksums: Dict[str, str] = {} checksum_names = [ - "mm_streams_embeddings.embedding_module.tok_embeddings.weight", + f"{emb}tok_embeddings.weight", "layers.0.attention.wq.weight", f"layers.{params['n_layers'] - 1}.feed_forward.w2.weight", - "mm_streams_embeddings.embedding_module.whisper_encoder.conv_layers.0.conv.weight", - "mm_streams_embeddings.embedding_module.whisper_encoder.transformer.norm.weight", + f"{emb}whisper_encoder.conv_layers.0{conv_infix}.weight", + f"{emb}whisper_encoder.transformer.norm.weight", ] for name in checksum_names: @@ -657,6 +695,7 @@ def metadata_from_model_info( out_type: str, name_map: Dict[str, str], mel_filters_sha256: str, + offline: bool = False, ) -> List[Tuple[str, int, Any]]: params = model_info["params"] tekken = model_info["tekken"] @@ -679,8 +718,8 @@ def metadata_from_model_info( vocab_limit = max(0, min(vocab_limit, len(vocab))) vocab_b64 = [str(vocab[i]["token_bytes"]) for i in range(vocab_limit)] - sample_rate = int(audio_cfg.get("sampling_rate", aud["sampling_rate"])) - frame_rate = float(audio_cfg.get("frame_rate", aud["frame_rate"])) + sample_rate = int(audio_cfg.get("sampling_rate", aud.get("sampling_rate", 16000))) + frame_rate = float(audio_cfg.get("frame_rate", aud.get("frame_rate", 12.5))) hop_length = int(aud["hop_length"]) raw_audio_per_tok = int(sample_rate // frame_rate) audio_len_per_tok = raw_audio_per_tok // hop_length @@ -696,11 +735,26 @@ def metadata_from_model_info( special_lookup = {str(st.get("token_str", "")): int(st.get("rank", -1)) for st in special_tokens} + # Encoder defaults differ between families: the realtime params.json sets these + # explicitly (causal, sliding_window, rope_theta, norm_eps); the offline one omits + # them → standard Whisper-style encoder (non-causal, full attention, RoPE theta 1e6). + enc_causal = bool(enc.get("causal", not offline)) + enc_sliding_window = int(enc.get("sliding_window", 0)) + enc_norm_eps = float(enc.get("norm_eps", 1e-5)) + enc_rope_theta = float(enc.get("rope_theta", 1e6)) + dec_sliding_window = int(params.get("sliding_window", 0) or 0) + global_log_mel_max = float(aud.get("global_log_mel_max", 1.5)) + ada_enabled = bool(params.get("ada_rms_norm_t_cond", not offline)) + + arch_str = "voxtral" if offline else "voxtral_realtime" + name_str = "Voxtral-Mini-3B-2507" if offline else "Voxtral-Mini-4B-Realtime-2602" + kv: List[Tuple[str, int, Any]] = [ - ("general.architecture", GGUFType.STRING, "voxtral_realtime"), - ("general.name", GGUFType.STRING, "Voxtral-Mini-4B-Realtime-2602"), + ("general.architecture", GGUFType.STRING, arch_str), + ("general.name", GGUFType.STRING, name_str), ("general.file_type", GGUFType.UINT32, 0 if out_type == "f32" else 1), ("voxtral.format_version", GGUFType.INT32, 1), + ("voxtral.decode.offline", GGUFType.BOOL, offline), ("voxtral.decoder.dim", GGUFType.INT32, int(params["dim"])), ("voxtral.decoder.n_layers", GGUFType.INT32, int(params["n_layers"])), ("voxtral.decoder.n_heads", GGUFType.INT32, int(params["n_heads"])), @@ -709,17 +763,21 @@ def metadata_from_model_info( ("voxtral.decoder.n_kv_heads", GGUFType.INT32, int(params["n_kv_heads"])), ("voxtral.decoder.norm_eps", GGUFType.FLOAT32, float(params["norm_eps"])), ("voxtral.decoder.rope_theta", GGUFType.FLOAT32, float(params["rope_theta"])), - ("voxtral.decoder.sliding_window", GGUFType.INT32, int(params["sliding_window"])), + ("voxtral.decoder.sliding_window", GGUFType.INT32, dec_sliding_window), ("voxtral.encoder.dim", GGUFType.INT32, int(enc["dim"])), ("voxtral.encoder.n_layers", GGUFType.INT32, int(enc["n_layers"])), ("voxtral.encoder.n_heads", GGUFType.INT32, int(enc["n_heads"])), ("voxtral.encoder.head_dim", GGUFType.INT32, int(enc["head_dim"])), ("voxtral.encoder.hidden_dim", GGUFType.INT32, int(enc["hidden_dim"])), - ("voxtral.encoder.n_kv_heads", GGUFType.INT32, int(enc["n_kv_heads"])), - ("voxtral.encoder.norm_eps", GGUFType.FLOAT32, float(enc["norm_eps"])), - ("voxtral.encoder.rope_theta", GGUFType.FLOAT32, float(enc["rope_theta"])), - ("voxtral.encoder.sliding_window", GGUFType.INT32, int(enc["sliding_window"])), - ("voxtral.ada_rms_norm_t_cond", GGUFType.BOOL, bool(params.get("ada_rms_norm_t_cond", True))), + ("voxtral.encoder.n_kv_heads", GGUFType.INT32, int(enc.get("n_kv_heads", enc["n_heads"]))), + ("voxtral.encoder.norm_eps", GGUFType.FLOAT32, enc_norm_eps), + ("voxtral.encoder.rope_theta", GGUFType.FLOAT32, enc_rope_theta), + ("voxtral.encoder.sliding_window", GGUFType.INT32, enc_sliding_window), + ("voxtral.encoder.causal", GGUFType.BOOL, enc_causal), + # Offline = true Whisper block (LayerNorm + GELU MLP); realtime = RMSNorm + SwiGLU + ("voxtral.encoder.norm_type", GGUFType.STRING, "layernorm" if offline else "rmsnorm"), + ("voxtral.encoder.ffn_type", GGUFType.STRING, "mlp_gelu" if offline else "swiglu"), + ("voxtral.ada_rms_norm_t_cond", GGUFType.BOOL, ada_enabled), ("voxtral.ada_rms_norm_t_cond_dim", GGUFType.INT32, int(params.get("ada_rms_norm_t_cond_dim", 32))), ("voxtral.vocab_size", GGUFType.INT32, int(params["vocab_size"])), ("voxtral.audio.sample_rate", GGUFType.INT32, sample_rate), @@ -727,7 +785,7 @@ def metadata_from_model_info( ("voxtral.audio.num_mel_bins", GGUFType.INT32, int(aud["num_mel_bins"])), ("voxtral.audio.hop_length", GGUFType.INT32, int(aud["hop_length"])), ("voxtral.audio.window_size", GGUFType.INT32, int(aud["window_size"])), - ("voxtral.audio.global_log_mel_max", GGUFType.FLOAT32, float(aud["global_log_mel_max"])), + ("voxtral.audio.global_log_mel_max", GGUFType.FLOAT32, global_log_mel_max), ("voxtral.audio.mel_filters_tensor", GGUFType.STRING, "audio.mel_filters"), ("voxtral.audio.mel_filters_sha256", GGUFType.STRING, mel_filters_sha256), ("voxtral.audio.downsample_factor", GGUFType.INT32, downsample_factor), @@ -1063,6 +1121,9 @@ def convert(args: argparse.Namespace) -> None: write_out_path = temp_fp_path with SafeTensorReader(st_path) as reader: + offline = detect_offline(reader.keys()) + emb, _ = arch_prefixes(offline) + print(f"[convert] detected architecture: {'voxtral (offline)' if offline else 'voxtral_realtime'}") names, checksums, expected = validate_safetensors(reader, model_info["params"], verbose=True) mandatory = sorted(expected.keys()) @@ -1071,7 +1132,7 @@ def convert(args: argparse.Namespace) -> None: for n in [ "output.weight", "output.bias", - "mm_streams_embeddings.embedding_module.output.weight", + f"{emb}output.weight", ] if n in names ] @@ -1081,12 +1142,17 @@ def convert(args: argparse.Namespace) -> None: plans = make_plans(reader, tensor_names, base_out_type, name_map) mel_plan, mel_sha = make_mel_filter_plan(model_info["params"]) plans.append(mel_plan) + if offline: + # Offline Whisper encoder needs sinusoidal positional embeddings (the + # consolidated checkpoint omits them); generate and embed them. + plans.append(make_enc_pos_embedding_plan(model_info["params"])) kv = metadata_from_model_info( model_info, checksums, base_out_type, name_map, mel_sha, + offline=offline, ) print(f"[convert] writing {len(plans)} tensors to {write_out_path}") diff --git a/tools/download_model.sh b/tools/download_model.sh index 50b4c9b..ba9ff12 100755 --- a/tools/download_model.sh +++ b/tools/download_model.sh @@ -1,19 +1,24 @@ #!/bin/bash -# Download Voxtral Realtime 4B GGUF model from HuggingFace +# Download a Voxtral GGUF model from HuggingFace # -# Usage: ./download_model.sh [QUANT] [--dir DIR] -# QUANT Quantization precision (default: Q4_0) -# --dir DIR Download to DIR (default: models/voxtral) +# Usage: ./download_model.sh [QUANT] [--model realtime|mini] [--dir DIR] +# QUANT Quantization precision (default: Q4_K_M) +# --model VARIANT realtime (4B streaming, default) or mini (3B offline) +# --dir DIR Download to DIR (default depends on --model) set -e -MODEL_ID="andrijdavid/Voxtral-Mini-4B-Realtime-2602-GGUF" -MODEL_DIR="models/voxtral" -QUANT="Q4_0" +MODEL_VARIANT="realtime" +MODEL_DIR="" +QUANT="Q4_K_M" # Simple argument parsing while [[ $# -gt 0 ]]; do case $1 in + --model) + MODEL_VARIANT="$2" + shift 2 + ;; --dir) MODEL_DIR="$2" shift 2 @@ -30,11 +35,26 @@ while [[ $# -gt 0 ]]; do esac done +case "$MODEL_VARIANT" in + realtime) + MODEL_ID="andrijdavid/Voxtral-Mini-4B-Realtime-2602-GGUF" + : "${MODEL_DIR:=models/voxtral}" + ;; + mini) + MODEL_ID="andrijdavid/Voxtral-Mini-3B-2507-GGUF" + : "${MODEL_DIR:=models/voxtral-3b}" + ;; + *) + echo "Unknown --model '$MODEL_VARIANT' (expected: realtime|mini)" + exit 1 + ;; +esac + # Standardize quant naming (if user passes q4_0 instead of Q4_0) QUANT=$(echo "$QUANT" | tr '[:lower:]' '[:upper:]') FILE_NAME="${QUANT}.gguf" -echo "Downloading Voxtral Realtime 4B GGUF to ${MODEL_DIR}/" +echo "Downloading Voxtral ${MODEL_VARIANT} GGUF to ${MODEL_DIR}/" echo "Model: ${MODEL_ID}" echo "Quantization: ${QUANT}" echo "" @@ -53,7 +73,7 @@ else HTTP_CODE=$(curl -s -o /dev/null -I -L -w "%{http_code}" "${BASE_URL}/${FILE_NAME}") if [[ "$HTTP_CODE" -ne 200 && "$HTTP_CODE" -ne 302 ]]; then echo "Error: Quantization '${QUANT}' not found in repository (HTTP ${HTTP_CODE})." - echo "Available quants include: Q2_K, Q4_0, Q4_1, Q5_0, Q5_1" + echo "Available quants include: Q2_K, Q3_K, Q4_0, Q4_1, Q4_K, Q4_K_M, Q5_0, Q5_1, Q5_K, Q6_K, Q8_0" exit 1 fi