From c2524de8639e1d6a5f66e8eeb69bb9ba574c65b3 Mon Sep 17 00:00:00 2001 From: wOvAN Date: Tue, 20 Oct 2026 01:58:40 +0500 Subject: [PATCH] feat(qwen3_tts): stream the RKNN decoder with window semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stateful RKNN vocoder voices each token once through a carried cache, which is why streaming on the NPU beats the CPU decoder on speed (57.7 s against 151.2 s) and misses it on waveform (16.4 dB SNR): the ggml streaming graph re-voices the 25-frame left context inside every window and drops its audio, a structurally different function. Add an optional `window` artifact to the `speech_decoder` RKNN plan. When present, every streaming event runs a full decode of its whole [left context + new] window from a zeroed cache (the same stateful graph, `cache_out` ignored, RoPE restarting at 0), reproducing the ggml streaming function per chunk. Naming the main engine aliases it — one resident context voices groups offline and windows streaming, and `engine_resources` prices it once. A dedicated smaller export is the per-event optimization. Measured on the RK3588 board, 307 Russian characters, seed 42, greedy, conv f16: streaming SNR against the ggml streaming render 47.4 dB (max sample delta 731/32767) against 16.4 dB for the stateful path, at 93.2 s wall against 44 s — the re-voicing price, and under the CPU's 106 s. Stateful streaming stays the default; offline decoding is untouched. The converter gains a small fix: with `--output *.rknn` the ONNX export now lands beside the engine path instead of under it. --- docs/build/rockchip-npu.md | 18 ++++- docs/models/qwen3_tts.md | 12 ++- .../models/qwen3_tts/rknn_speech_decoder.h | 17 +++- .../qwen3_tts/tokenizer_speech_decoder.h | 4 + src/models/qwen3_tts/loader.cpp | 6 +- src/models/qwen3_tts/rknn_speech_decoder.cpp | 79 +++++++++++++++++++ src/models/qwen3_tts/session.cpp | 6 +- .../qwen3_tts/tokenizer_speech_decoder.cpp | 9 ++- tools/rknn/prepare_qwen3_tts_rknn_vocoder.py | 3 + 9 files changed, 143 insertions(+), 11 deletions(-) diff --git a/docs/build/rockchip-npu.md b/docs/build/rockchip-npu.md index 4c4a78476..9119189e6 100644 --- a/docs/build/rockchip-npu.md +++ b/docs/build/rockchip-npu.md @@ -383,7 +383,7 @@ the configurable components: | Piper `decoder` | `piper_tts.exec_plan` | `ggml`, `rknn` | RKNN: required `main` | | Piper `duration_encoder` / `duration_flows` | `piper_tts.exec_plan` | `ggml` | None (`backend` may name `cpu`) | | Kokoro `generator` | `kokoro_tts.exec_plan` | `ggml`, `rknn` | RKNN: required `main` | -| Qwen3 `speech_decoder` | `qwen3_tts.exec_plan` | `ggml`, `rknn` | RKNN: required `main`, optional `first` | +| Qwen3 `speech_decoder` | `qwen3_tts.exec_plan` | `ggml`, `rknn` | RKNN: required `main`, optional `first`, `window` | | sanoTTS `decoder` | `sanotts.exec_plan` | `default`, `ggml`, `neon` | None | | Inflect v2 `decoder` | `inflect_v2.exec_plan` | `ggml`, `rknn` | RKNN: required `main` | | CosyVoice3 `flow` | `cosyvoice3.exec_plan` | `ggml`, `rknn` | RKNN: required `main` (comma-separated ladder of per-length engines) | @@ -443,6 +443,16 @@ CLI session options carry JSON text. Server configs also accept a JSON object: small-batch model, which is what cuts time-to-first-audio. When the RKNN decoder is active the ggml decoder weights are not loaded at all, which is where most of the memory saving comes from. +`artifacts.window` switches streaming to the ggml path's window semantics: every event runs a +full decode of its whole `[left context + new]` window from a zeroed cache (the stateful graph +again, so `cache_out` is ignored) and the left context's audio is dropped. The stateful default +voices each token once, which is why it beats the CPU streaming decoder on speed and misses it on +waveform; the window role re-voices the context and reproduces the CPU streaming function +per chunk. It costs one engine step of `25 + frames_per_event` tokens per event. Naming the main +engine (`"window":"vocoder_stateful_64b_fp16.rknn"`) aliases it — the resident B=64 step covers +the default 25 + 12 window at one resident context; a dedicated smaller export compiled with +`--codes-length 25+stream_frames_per_event` is cheaper per event. Offline decoding is unaffected. + Piper takes one option and is offline only: ```bash @@ -504,9 +514,12 @@ model's latent channel count and its samples-per-frame hop at init. | offline, RKNN B=64 | 2.17 | 2560 MB | 64.1 | | streaming, ggml decoder | 151.2 s | - | 365 | | streaming, RKNN 16+64 | 57.7 s, first audio 0.88 s | - | 62.4 | +| streaming, RKNN 16+64 window (main aliased) | 93.2 s, first chunk 3.07 s | - | 47.9 | Streaming gains more than offline because the ggml streaming decoder re-voices the left-context -tokens in every window, while the stateful model voices each token once. +tokens in every window, while the stateful model voices each token once. The window role buys the +quality back at that same price: the same run re-voicing every window costs the extra 48 s of +decoder and lands at 47.4 dB against the ggml streaming render instead of 16.4 dB. Quality, split by what varies, because the numbers below are not comparable to each other and one of the rows is not about the NPU at all: @@ -516,6 +529,7 @@ of the rows is not about the NPU at all: | backend, offline, 185 chars | 27 dB | - | | backend, offline, 307 chars | 25.1 dB | 0.9999 | | backend, streaming, 307 chars | 16.4 dB | 0.9988 | +| backend, streaming window, 307 chars | 47.4 dB | - | | backend, RKNN B=16 vs B=64 | 57 dB | - | | mode, same backend (streaming vs offline), 307 chars | -2.2 dB | 0.187 | diff --git a/docs/models/qwen3_tts.md b/docs/models/qwen3_tts.md index 32ff59a32..c3823f14a 100644 --- a/docs/models/qwen3_tts.md +++ b/docs/models/qwen3_tts.md @@ -170,7 +170,7 @@ constant-context sizes are in MiB. | `qwen3_tts.talker_constant_context_mb` | positive integer | `4` | Talker constant tensor context. | | `qwen3_tts.code_predictor_constant_context_mb` | positive integer | `4` | Code predictor constant tensor context. | | `qwen3_tts.speech_decoder_constant_context_mb` | positive integer | `4` | Speech decoder constant tensor context. | -| `qwen3_tts.exec_plan` | JSON object of component plans, encoded as a string | `{}` | `{"speech_decoder":{"impl":"rknn","artifacts":{"main":"/models/decoder.rknn","first":"/models/decoder-first.rknn"}}}` selects the NPU decoder and skips GGML decoder weights. `main` is required; `first` optionally uses a smaller engine for the first streaming group. No decoder-specific params. See [component execution plans](../build/rockchip-npu.md#component-execution-plans). | +| `qwen3_tts.exec_plan` | JSON object of component plans, encoded as a string | `{}` | `{"speech_decoder":{"impl":"rknn","artifacts":{"main":"/models/decoder.rknn","first":"/models/decoder-first.rknn","window":"/models/decoder-window.rknn"}}}` selects the NPU decoder and skips GGML decoder weights. `main` is required; `first` optionally uses a smaller engine for the first streaming group; `window` optionally streams with window semantics. No decoder-specific params. See [component execution plans](../build/rockchip-npu.md#component-execution-plans). | ## Rockchip NPU decoder @@ -189,7 +189,15 @@ outputs `audio_values` and `cache_out`; 1920 samples per token at 24 kHz, with the convolution and KV cache carried across steps in the graph I/O. The optional first-group model runs the first token group at a smaller compiled batch; the cache layout is batch-independent, so its cache feeds the main -model directly. A `.rknn` is compiled per SoC and is not portable between +model directly. The optional window model (`artifacts.window`) streams with the +ggml path's window semantics instead: every event runs a full decode of its +whole `[left context + new]` window from a zeroed cache and drops the context +frames' audio, so each event reproduces the CPU streaming function exactly +(25 frames re-voiced per event, at one NPU step of `25 + frames_per_event` +tokens). Naming the same file as `main` aliases it: the resident engine covers +the default 25 + 12 window; a dedicated export compiled at `--codes-length +25 + stream_frames_per_event` is cheaper per event. Offline decoding keeps +using `main`. A `.rknn` is compiled per SoC and is not portable between chips. Build the file with [`tools/rknn/prepare_qwen3_tts_rknn_vocoder.py`](../../tools/rknn/prepare_qwen3_tts_rknn_vocoder.py) and see [the Rockchip NPU build guide](../build/rockchip-npu.md) for the toolkit diff --git a/include/engine/models/qwen3_tts/rknn_speech_decoder.h b/include/engine/models/qwen3_tts/rknn_speech_decoder.h index 17da80393..84d348a18 100644 --- a/include/engine/models/qwen3_tts/rknn_speech_decoder.h +++ b/include/engine/models/qwen3_tts/rknn_speech_decoder.h @@ -31,9 +31,22 @@ class RknnSpeechDecoder { // waits for. The cache layout is B-independent, so the small model's cache // feeds the big one. `num_quantizers` and `sliding_window` come from the // checkpoint's speech_tokenizer_config. + // + // `window_path` (optional) switches streaming to window semantics: the same + // stateful export run at zero cache and `cache_valid` 0 over the whole + // [left context + new] window, which is exactly the full decode over the + // window — the function the ggml streaming path computes per chunk, context + // re-voiced every event. The first window (no context) rides the + // first-group model when one is loaded. The context frames' audio is + // dropped, flush is a no-op, and the stateful streaming state stays unused. + // A `window_path` naming the same file as `model_path` (compared as files, + // as the residency accounting compares them, not as strings) aliases the + // main engine (one resident context voices groups offline and windows + // streaming). RknnSpeechDecoder( const std::string & model_path, const std::string & first_model_path, + const std::string & window_path, int64_t num_quantizers, int64_t head_dim, int64_t sliding_window, @@ -55,7 +68,9 @@ class RknnSpeechDecoder { // Streaming step. `codes` holds `context_frames + new_frames` row-major // frames of which the first `context_frames` were already voiced and live in // the cache; only the tail is fed. `flush` voices the buffered remainder - // (< one group) with zero padding and resets the utterance. + // (< one group) with zero padding and resets the utterance. With a window + // engine loaded the context is fed too (re-voiced, not cache-covered), its + // audio is dropped, and `flush` returns nothing. std::vector step( const int32_t * codes, int64_t context_frames, diff --git a/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h b/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h index c9240ca95..3ead94d2c 100644 --- a/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h +++ b/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h @@ -25,9 +25,13 @@ class RknnSpeechDecoder; // Optional Rockchip NPU backend for the decoder stage: the path(s) of the // stateful step export. Empty `rknn_model_path` keeps the ggml graph. +// `window_path` (optional) is a larger-B export used to voice each whole +// streaming window from a zeroed cache — window semantics matching the ggml +// streaming path, at the cost of re-voicing the left context per event. struct Qwen3SpeechTokenizerDecoderRknn { std::string model_path; std::string first_model_path; + std::string window_path; }; class Qwen3SpeechTokenizerDecoderRuntime { diff --git a/src/models/qwen3_tts/loader.cpp b/src/models/qwen3_tts/loader.cpp index 54d5f830d..71538feb2 100644 --- a/src/models/qwen3_tts/loader.cpp +++ b/src/models/qwen3_tts/loader.cpp @@ -170,8 +170,10 @@ class Qwen3TTSLoader final : public runtime::IVoiceModelLoader { } const auto & artifacts = decoder->require("artifacts"); paths.push_back(io::json::require_string(artifacts, "main")); - if (const auto * first = artifacts.find("first")) { - paths.push_back(first->as_string()); + for (const char * const role : {"first", "window"}) { + if (const auto * artifact = artifacts.find(role)) { + paths.push_back(artifact->as_string()); + } } } catch (const std::runtime_error &) { // Session loading reports invalid execution plans. diff --git a/src/models/qwen3_tts/rknn_speech_decoder.cpp b/src/models/qwen3_tts/rknn_speech_decoder.cpp index d423dfbec..99ac77111 100644 --- a/src/models/qwen3_tts/rknn_speech_decoder.cpp +++ b/src/models/qwen3_tts/rknn_speech_decoder.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -136,6 +137,11 @@ struct RknnSpeechDecoder::Impl { Step main; Step first; + // Streaming window engine: a stateful export run at a zeroed cache so each + // call is a full decode over its whole input. It may be the main model's + // file itself, which is aliased rather than loaded a second time. + Step window; + bool window_from_main = false; int64_t num_quantizers = 0; int64_t kv_len = 0; std::vector inv_freq; @@ -211,6 +217,7 @@ void RknnSpeechDecoder::Impl::voice( RknnSpeechDecoder::RknnSpeechDecoder( const std::string & model_path, const std::string & first_model_path, + const std::string & window_path, int64_t num_quantizers, int64_t head_dim, int64_t sliding_window, @@ -237,14 +244,36 @@ RknnSpeechDecoder::RknnSpeechDecoder( throw std::runtime_error("Qwen3 speech decoder RKNN: first-group cache size differs from the main model"); } } + if (!window_path.empty()) { + // Files are compared, not strings: the residency accounting canonicalizes paths + // (`engine_resources`), so a string comparison here lets the same file spelled two + // ways load two contexts and be counted as one. + std::error_code ec; + if (std::filesystem::equivalent(window_path, model_path, ec)) { + // The main engine voices the windows too (its B covers the default + // 25 + 12): one resident context, not two. + impl_->window_from_main = true; + } else { + impl_->window.load(window_path, num_quantizers, head_dim / 2); + if (impl_->window.cache_numel != impl_->main.cache_numel) { + throw std::runtime_error("Qwen3 speech decoder RKNN: window cache size differs from the main model"); + } + } + } debug::timing_log_scalar("qwen3_tts.speech_decoder.rknn.group", impl_->main.group); debug::timing_log_scalar( "qwen3_tts.speech_decoder.rknn.first_group", impl_->first.engine.loaded() ? impl_->first.group : 0); + debug::timing_log_scalar( + "qwen3_tts.speech_decoder.rknn.window_group", + impl_->window_from_main + ? impl_->main.group + : (impl_->window.engine.loaded() ? impl_->window.group : 0)); debug::timing_log_scalar("qwen3_tts.speech_decoder.rknn.cache_numel", impl_->main.cache_numel); #else (void)model_path; (void)first_model_path; + (void)window_path; (void)num_quantizers; (void)head_dim; (void)sliding_window; @@ -321,6 +350,56 @@ std::vector RknnSpeechDecoder::step( throw std::runtime_error("Qwen3 speech decoder RKNN: invalid streaming codec shape"); } std::vector out; + // Windowed streaming: voice the whole [context + new] window on the + // fixed-shape engine from a zeroed cache — a standalone full decode over + // the window, the function the ggml streaming path computes per chunk, + // context re-voiced every event — and drop the context's audio. Nothing is + // buffered, so flush voices nothing. + if (s.window_from_main || s.window.engine.loaded()) { + if (codes == nullptr || flush) { + return out; + } + const int64_t total = context_frames + new_frames; + // The context-free first window rides the small first-group engine: + // fewer frames per step is the audio the listener waits for. + Impl::Step & voice_step = + s.first.engine.loaded() && context_frames == 0 && total <= s.first.group + ? s.first + : (s.window_from_main ? s.main : s.window); + if (total > voice_step.group) { + throw std::runtime_error( + "Qwen3 speech decoder RKNN: window of " + std::to_string(total) + + " frames exceeds the " + std::to_string(voice_step.group) + "-frame window engine"); + } + std::vector fed(static_cast(s.num_quantizers) * voice_step.group, 0); + for (int64_t frame = 0; frame < total; ++frame) { + for (int64_t g = 0; g < s.num_quantizers; ++g) { + fed[static_cast(g * voice_step.group + frame)] = codes[frame * num_quantizers + g]; + } + } + // RoPE positions restart at 0 every window: each event is a full decode + // of its own window, not a continuation of the last one. + std::vector freqs(static_cast(voice_step.group * voice_step.freqs_width), 0.0F); + for (int64_t frame = 0; frame < total; ++frame) { + for (int64_t i = 0; i < voice_step.freqs_width; ++i) { + freqs[static_cast(frame * voice_step.freqs_width + i)] = static_cast( + std::fmod(static_cast(frame) * s.inv_freq[static_cast(i)], 2.0 * M_PI)); + } + } + std::vector audio; + std::vector cache_out; + const auto start = Clock::now(); + voice_step.run(fed.data(), freqs.data(), 0.0F, s.cache.data(), audio, cache_out); + s.compute_ms += std::chrono::duration(Clock::now() - start).count(); + ++s.steps; + const int64_t keep = total * kSamplesPerToken; + const int64_t drop = context_frames * kSamplesPerToken; + if (keep > static_cast(audio.size())) { + throw std::runtime_error("Qwen3 speech decoder RKNN: window engine returned a short waveform"); + } + out.assign(audio.begin() + drop, audio.begin() + keep); + return out; + } // The window carries left context, and a stateful model cannot voice a token // twice (the cache would hold it twice and pos would run ahead), so the // context is normally skipped as already covered by the cache. The first diff --git a/src/models/qwen3_tts/session.cpp b/src/models/qwen3_tts/session.cpp index 2e41b735c..5f10418fe 100644 --- a/src/models/qwen3_tts/session.cpp +++ b/src/models/qwen3_tts/session.cpp @@ -337,7 +337,7 @@ Qwen3TTSSession::Qwen3TTSSession( "qwen3_tts.exec_plan", {{"speech_decoder", "ggml", {{"ggml", {options.backend.type}, {}, {}}, - {"rknn", {}, {"main"}, {"first"}}}}}); + {"rknn", {}, {"main"}, {"first", "window"}}}}}); const auto & decoder_plan = exec_plan.at("speech_decoder"); if (decoder_plan.impl == "rknn") { const auto & model_dir = assets_->resources.model_directory(); @@ -346,6 +346,10 @@ Qwen3TTSSession::Qwen3TTSSession( if (const auto first = decoder_plan.artifacts.find("first"); first != decoder_plan.artifacts.end()) { rknn_decoder_.first_model_path = engine::rknn::resolve_engine_path(first->second, model_dir); } + if (const auto window = decoder_plan.artifacts.find("window"); + window != decoder_plan.artifacts.end()) { + rknn_decoder_.window_path = engine::rknn::resolve_engine_path(window->second, model_dir); + } } if (perf_mode_ == Qwen3TTSPerfMode::FlashAttention && (!source_contains_q8_tensor(*assets_->model_weights) || diff --git a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp index c218b2380..3b8bf4ac0 100644 --- a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp +++ b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp @@ -1175,6 +1175,7 @@ Qwen3SpeechTokenizerDecoderRuntime::Qwen3SpeechTokenizerDecoderRuntime( rknn_ = std::make_unique( std::move(rknn.model_path), std::move(rknn.first_model_path), + std::move(rknn.window_path), weights_->config.num_quantizers, weights_->config.head_dim, weights_->config.sliding_window, @@ -1305,9 +1306,11 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode_stream_chunk( } const auto begin = Clock::now(); if (rknn_ != nullptr) { - // Stateful NPU path: the left context is already in the carried cache, - // so only the tail is fed, and audio comes out in whole model groups - // (the controller drops empty chunks). + // NPU path: on the stateful engines the left context is already in the + // carried cache, so only the tail is fed and audio comes out in whole + // model groups (the controller drops empty chunks); with a window engine + // loaded the whole window is voiced per event from a zeroed cache and + // the context's audio is dropped (see RknnSpeechDecoder::step). auto samples = rknn_->step( codec_codes.codes.data(), context_frames, diff --git a/tools/rknn/prepare_qwen3_tts_rknn_vocoder.py b/tools/rknn/prepare_qwen3_tts_rknn_vocoder.py index 0115945e0..3e17d14dd 100644 --- a/tools/rknn/prepare_qwen3_tts_rknn_vocoder.py +++ b/tools/rknn/prepare_qwen3_tts_rknn_vocoder.py @@ -307,6 +307,9 @@ def rope(i, L): return None out = args.output or f"vocoder_stateful_{B}.onnx" + if out.endswith(".rknn"): + # --output names the compiled engine; the ONNX lands beside it. + out = os.path.splitext(out)[0] + ".onnx" os.makedirs(os.path.dirname(out) or ".", exist_ok=True) dummy_c = torch.randint(0, 2048, (1, 16, B), dtype=torch.int64) dummy_cache = torch.zeros(step.cache_numel, dtype=torch.float32)