diff --git a/README.md b/README.md index c9237a39a..cf6370270 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,15 @@ loaded; write a new file and rename it over the old one. `--no-mmap` (`transcribe_model_load_params::no_mmap`, or `TRANSCRIBE_NO_MMAP=1`) copies the weights instead. +Models whose FFN gate and up weights are stored back to back (every packed +variant: granite, canary-qwen, qwen3-asr, voxtral, voxtral-realtime, moss, +funasr-nano) are fused at load time by pointing one tensor at those bytes +instead of packing a second copy, so the fusion usually costs no extra weight +memory. It falls back to the copying path when the device layout does not allow +that (tensor storage padded or the mapping split, e.g. a file larger than the +device's max buffer size). `TRANSCRIBE_NO_GATE_UP_ALIAS=1` forces the copying +path. + ## Bindings Official bindings wrap the C API for other languages: diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 4bb0336da..f9a58a55d 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -26,6 +26,7 @@ of tests. | --- | --- | | `TRANSCRIBE_BACKENDS=` | Restrict which backends may register: a comma-separated list of `cpu`, `metal`, `vulkan`, `cuda`, `rocm`, `other`, `all` (case-insensitive). An excluded backend never runs any code (no module load, no driver init). It can only narrow the mask a host passes to `transcribe_init_backends_ex()`, applies even if the host never calls it, and CPU is always kept. Unknown names are dropped and logged as an error naming what is still allowed, so a typo narrows rather than widens (`vulcan` means CPU-only). Unset or empty means `all`. Read once per process. | | `TRANSCRIBE_NO_MMAP` | Copy model weights into memory instead of memory-mapping the model file, for every model load in the process (same as `transcribe_model_load_params::no_mmap`, CLI `--no-mmap`). Read at each load. | +| `TRANSCRIBE_NO_GATE_UP_ALIAS` | Disable the load-time gate+up fusion shortcut that points one packed FFN weight tensor at the existing adjacent gate and up weights instead of copying them into a new buffer (granite, canary-qwen, qwen3-asr, voxtral, voxtral-realtime, moss, funasr-nano). Forces the copying path, i.e. the pre-mmapping memory cost. Read at each load; for A/B measurement only. | | `TRANSCRIBE_NO_FLASH` | Disable flash attention on encoder and decoder (forces the manual F32 path). | | `TRANSCRIBE_FORCE_FLASH` | Force flash attention on. Wins over `TRANSCRIBE_NO_FLASH` if both are set. | | `TRANSCRIBE_CONV_DIRECT_DW` / `TRANSCRIBE_CONV_NO_DIRECT_DW` | Force the depthwise-conv dispatch to the direct `conv_2d_dw` path / the im2col path, overriding the per-family backend default. | diff --git a/src/causal_lm/causal_lm.cpp b/src/causal_lm/causal_lm.cpp index 4db8c0730..08744f27b 100644 --- a/src/causal_lm/causal_lm.cpp +++ b/src/causal_lm/causal_lm.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include namespace transcribe::causal_lm { @@ -836,6 +837,132 @@ void PackedGateUpHandles::free() { } } +namespace { + +// Why a block cannot alias. Reported at INFO when a load falls back, because a +// silently lost memory win is exactly the regression this path can suffer from +// without a wrong answer anywhere. +enum class AliasSkip { + None, + DisabledByEnv, + NotInOneBuffer, + IsAView, + NotContiguous, + NotAdjacent, + SplitMapping, + PaddedStorage, +}; + +const char * alias_skip_name(AliasSkip why) { + switch (why) { + case AliasSkip::None: + return "aliased"; + case AliasSkip::DisabledByEnv: + return "disabled by TRANSCRIBE_NO_GATE_UP_ALIAS"; + case AliasSkip::NotInOneBuffer: + return "gate/up in different buffers"; + case AliasSkip::IsAView: + return "gate/up is a view"; + case AliasSkip::NotContiguous: + return "gate/up not contiguous"; + case AliasSkip::NotAdjacent: + return "up does not follow gate"; + case AliasSkip::SplitMapping: + return "mapping split into several views"; + case AliasSkip::PaddedStorage: + return "padded tensor storage"; + } + return "unknown"; +} + +// Whether a block's gate/up bytes are already in packed layout: gate_w's bytes +// immediately followed by up_w's inside one backend buffer, so a +// [hidden, 2*intermediate] tensor starting at gate_w->data is the fused tensor. +// +// That is the normal case. The converter writes `ffn.up.weight` straight after +// `ffn.gate.weight` (verified for granite 4.0/4.1/plus, canary-qwen, qwen3-asr, +// voxtral, voxtral-realtime, moss and funasr-nano, all quant types), and the +// loader keeps tensors either in that same order (a copied buffer) or in place +// (load_common's read-only mmap). Aliasing instead of copying means the packed +// path costs no weight memory at all: under mmap the fused tensor is the very +// pages the gate/up tensors already are. +// +// Deliberately conservative. Anything unexpected falls back to the copying +// path, which is what every model got before this existed, and says why. +AliasSkip gate_up_alias_skip(const GateUpEntry & e, size_t fused_bytes) { + ggml_backend_buffer_t buffer = e.gate_w->buffer; + if (buffer == nullptr || buffer != e.up_w->buffer) { + return AliasSkip::NotInOneBuffer; + } + if (e.gate_w->data == nullptr || e.up_w->data == nullptr) { + return AliasSkip::NotInOneBuffer; + } + if (e.gate_w->view_src != nullptr || e.up_w->view_src != nullptr) { + return AliasSkip::IsAView; + } + if (!ggml_is_contiguous(e.gate_w) || !ggml_is_contiguous(e.up_w)) { + return AliasSkip::NotContiguous; + } + if (static_cast(e.gate_w->data) + ggml_nbytes(e.gate_w) != e.up_w->data) { + return AliasSkip::NotAdjacent; + } + + // A device may keep several views of one buffer (Metal splits a mapping + // larger than its max buffer size into overlapping views, after rounding the + // length up to the page size) and a tensor has to sit inside a single one. + // The generic API does not expose view bounds, so only alias a buffer the + // device can hold as one view. Splitting rather than aliasing costs a copy; + // aliasing across a boundary would hand the kernel the wrong buffer, so the + // margin is deliberately generous. + constexpr size_t k_max_page_size = 16384; // largest page size we target + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buffer); + const size_t max_buffer_size = ggml_backend_buft_get_max_size(buft); + if (max_buffer_size != 0 && ggml_backend_buffer_get_size(buffer) + k_max_page_size > max_buffer_size) { + return AliasSkip::SplitMapping; + } + + // Some devices pad or reshape a tensor's storage (see load_common's mmap + // gating); the fused tensor is only valid if it occupies exactly its bytes. + if (ggml_backend_buffer_get_alloc_size(buffer, *e.gate_up_w_out) != fused_bytes) { + return AliasSkip::PaddedStorage; + } + return AliasSkip::None; +} + +// One line per load, at INFO: the aliased count, what it saved, and - when some +// blocks were copied - why, so a lost win is visible without a debug build. +void log_alias_summary(const char * error_tag, + size_t n_aliased, + size_t n_entries, + size_t aliased_bytes, + const size_t (&skipped)[static_cast(AliasSkip::PaddedStorage) + 1]) { + std::string why; + if (n_aliased < n_entries) { + for (size_t i = 1; i <= static_cast(AliasSkip::PaddedStorage); ++i) { + if (skipped[i] == 0) { + continue; + } + if (!why.empty()) { + why += ", "; + } + char buf[96]; + std::snprintf(buf, sizeof(buf), "%s x%zu", alias_skip_name(static_cast(i)), skipped[i]); + why += buf; + } + } + + if (n_aliased == 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: gate+up packed by copying (%zu blocks; %s)", error_tag, n_entries, + why.c_str()); + return; + } + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: gate+up aliased for %zu/%zu blocks (%.1f MiB not duplicated)%s%s", + error_tag, n_aliased, n_entries, static_cast(aliased_bytes) / (1024.0 * 1024.0), + why.empty() ? "" : "; copied ", why.c_str()); +} + +} // namespace + transcribe_status pack_gate_up(ggml_backend_t backend, int hidden, int intermediate, @@ -878,9 +1005,61 @@ transcribe_status pack_gate_up(ggml_backend_t backend, log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: pack_gate_up new_tensor_2d failed at %zu", error_tag, i); return TRANSCRIBE_ERR_OOM; } + // The fused tensor is the byte-concat of the two, so their bytes must add + // up to it. Each family's weights.cpp pins gate/up to + // [hidden, intermediate] with its own GET_LIN(...) shape check, which is + // what makes this true; verifying it here, before the alias/copy split, + // is what lets the aliased path trust the fused tensor to cover exactly + // those two and nothing else. + const size_t gate_bytes = ggml_nbytes(e.gate_w); + const size_t up_bytes = ggml_nbytes(e.up_w); + if (ggml_nbytes(t) != gate_bytes + up_bytes) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: pack_gate_up entry %zu size mismatch (%zu vs %zu + %zu)", + error_tag, i, ggml_nbytes(t), gate_bytes, up_bytes); + return TRANSCRIBE_ERR_GGUF; + } *e.gate_up_w_out = t; } + // Blocks whose weights are already laid out as gate||up point at those + // bytes instead of duplicating them. The rest are copied into a buffer of + // our own, as before. + constexpr size_t k_n_skip_reasons = static_cast(AliasSkip::PaddedStorage) + 1; + const bool alias_allowed = !env::flag("TRANSCRIBE_NO_GATE_UP_ALIAS"); + std::vector copied; + size_t n_aliased = 0; + size_t aliased_bytes = 0; + size_t skipped[k_n_skip_reasons] = {}; + copied.reserve(entries.size()); + + for (size_t i = 0; i < entries.size(); ++i) { + const GateUpEntry & e = entries[i]; + ggml_tensor * gate_up = *e.gate_up_w_out; + const size_t fused_size = ggml_nbytes(gate_up); + const AliasSkip skip = alias_allowed ? gate_up_alias_skip(e, fused_size) : AliasSkip::DisabledByEnv; + if (skip != AliasSkip::None) { + ++skipped[static_cast(skip)]; + copied.push_back(i); + continue; + } + // Same shape, dtype and byte layout as the copied tensor; only the + // storage differs. gate_w->data is inside gate_w->buffer, which ggml's + // own bounds assert in ggml_backend_tensor_alloc checks. + if (ggml_backend_tensor_alloc(e.gate_w->buffer, gate_up, e.gate_w->data) != GGML_STATUS_SUCCESS) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: pack_gate_up could not alias gate/up at block %zu", error_tag, i); + return TRANSCRIBE_ERR_BACKEND; + } + ++n_aliased; + aliased_bytes += fused_size; + } + + if (copied.empty()) { + log_alias_summary(error_tag, n_aliased, entries.size(), aliased_bytes, skipped); + return TRANSCRIBE_OK; + } + + // alloc_ctx_tensors skips tensors that already have storage (the aliased + // ones) and returns NULL when none are left, hence the early return above. out_handles.buffer = ggml_backend_alloc_ctx_tensors(out_handles.ctx, backend); if (out_handles.buffer == nullptr) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: pack_gate_up backend buffer alloc failed", error_tag); @@ -891,21 +1070,19 @@ transcribe_status pack_gate_up(ggml_backend_t backend, // Fill: concat-along-dim-1 is gate's bytes followed by up's bytes // (one CPU round-trip per block at load time). std::vector buf; - for (const auto & e : entries) { - ggml_tensor * gate_up = *e.gate_up_w_out; - const size_t gate_bytes = ggml_nbytes(e.gate_w); - const size_t up_bytes = ggml_nbytes(e.up_w); - if (ggml_nbytes(gate_up) != gate_bytes + up_bytes) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: pack_gate_up size mismatch (%zu vs %zu + %zu)", error_tag, - ggml_nbytes(gate_up), gate_bytes, up_bytes); - return TRANSCRIBE_ERR_GGUF; - } + for (const size_t i : copied) { + const GateUpEntry & e = entries[i]; + ggml_tensor * gate_up = *e.gate_up_w_out; + const size_t gate_bytes = ggml_nbytes(e.gate_w); + const size_t up_bytes = ggml_nbytes(e.up_w); buf.resize(std::max(gate_bytes, up_bytes)); ggml_backend_tensor_get(e.gate_w, buf.data(), 0, gate_bytes); ggml_backend_tensor_set(gate_up, buf.data(), 0, gate_bytes); ggml_backend_tensor_get(e.up_w, buf.data(), 0, up_bytes); ggml_backend_tensor_set(gate_up, buf.data(), gate_bytes, up_bytes); } + + log_alias_summary(error_tag, n_aliased, entries.size(), aliased_bytes, skipped); return TRANSCRIBE_OK; } diff --git a/src/causal_lm/causal_lm.h b/src/causal_lm/causal_lm.h index 1992f86bf..81724a6b1 100644 --- a/src/causal_lm/causal_lm.h +++ b/src/causal_lm/causal_lm.h @@ -294,13 +294,24 @@ struct PackedGateUpHandles { void free(); }; -// Allocate a packed context + a [hidden, 2·intermediate] tensor per block -// on `backend`, then copy gate's bytes followed by up's bytes into each. -// Compatible with row-wise quants (Q4/Q5/Q6/Q8) because concat-along-dim-1 -// is byte-concat for those types. Writes `*entries[i].gate_up_w_out` to the -// new tensor and marks the buffer GGML_BACKEND_BUFFER_USAGE_WEIGHTS. -// Returns TRANSCRIBE_ERR_OOM on allocation failure, TRANSCRIBE_ERR_GGUF on a -// gate/up shape or type mismatch; `out_handles` is left in a state safe to free. +// Allocate a packed context + a [hidden, 2·intermediate] tensor per block, +// then give each one the bytes of gate followed by up. Compatible with +// row-wise quants (Q4/Q5/Q6/Q8) because concat-along-dim-1 is byte-concat for +// those types. +// +// Where the block's gate and up tensors are already adjacent in one backend +// buffer (the layout the converter writes, and the one both the copying and +// the mmap load paths preserve), the packed tensor points straight at gate's +// bytes and nothing is copied or duplicated. `out_handles.buffer` then stays +// null, and the packed tensors' storage belongs to the weights buffer: they +// must not outlive it. Otherwise the bytes are copied into a buffer owned by +// `out_handles`, as before. `TRANSCRIBE_NO_GATE_UP_ALIAS` forces the copy path. +// +// Writes `*entries[i].gate_up_w_out` to the new tensor; a buffer created here +// is marked GGML_BACKEND_BUFFER_USAGE_WEIGHTS. Returns TRANSCRIBE_ERR_OOM on +// allocation failure, TRANSCRIBE_ERR_GGUF on a gate/up shape or type mismatch, +// TRANSCRIBE_ERR_BACKEND if a device rejects an aliased tensor; +// `out_handles` is left in a state safe to free. transcribe_status pack_gate_up(ggml_backend_t backend, int hidden, int intermediate, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b29e6cd29..04008a32c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -349,6 +349,23 @@ transcribe_apply_warnings(transcribe_conv_pw_promote_unit) add_test(NAME transcribe_conv_pw_promote_unit COMMAND transcribe_conv_pw_promote_unit) +# ----------------------------------------------------------------------------- +# causal_lm::pack_gate_up: alias / copy / mismatch, no model needed +# ----------------------------------------------------------------------------- + +add_executable(transcribe_pack_gate_up_unit + pack_gate_up_unit.cpp) + +target_link_libraries(transcribe_pack_gate_up_unit PRIVATE transcribe ggml) + +target_include_directories(transcribe_pack_gate_up_unit PRIVATE + ${CMAKE_SOURCE_DIR}/src) + +transcribe_apply_warnings(transcribe_pack_gate_up_unit) + +add_test(NAME transcribe_pack_gate_up_unit + COMMAND transcribe_pack_gate_up_unit) + # ----------------------------------------------------------------------------- # Dispatcher-level transcribe_run behavior # ----------------------------------------------------------------------------- diff --git a/tests/pack_gate_up_unit.cpp b/tests/pack_gate_up_unit.cpp new file mode 100644 index 000000000..ecbbcb70b --- /dev/null +++ b/tests/pack_gate_up_unit.cpp @@ -0,0 +1,367 @@ +// pack_gate_up_unit.cpp - unit test for causal_lm::pack_gate_up. +// +// The packing step has three behaviours and no fixture pins them: +// +// - alias: gate and up already sit back to back in one backend buffer, so +// the fused tensor points at gate's bytes and nothing is +// duplicated (PackedGateUpHandles::buffer stays null, and the +// weights buffer is what the fused tensor lives in). +// - copy: anything else (a gap between gate and up, non-adjacent +// buffers) is copied into a buffer owned by the handles. +// - mismatch: gate and up do not add up to the fused shape -> TRANSCRIBE_ERR_GGUF, +// reported the same way whichever path the block would have taken. +// +// It also covers the mixed case, where some blocks alias and others copy in +// the same call, because the two storage kinds have to coexist in one call and +// the handles must free only what they allocated. +// +// Tensors are built by hand on the CPU backend rather than through a GGUF: +// adjacency is a property of how tensors land in a buffer, which a synthetic +// catalog controls exactly, and no model is needed. + +#include "causal_lm/causal_lm.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml.h" + +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +// Bail macro for setup prerequisites: log and return rather than crash on a +// null deref. +#define REQUIRE(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "SETUP FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + return EXIT_FAILURE; \ + } \ + } while (0) + +constexpr int k_hidden = 4; +constexpr int k_intermediate = 8; + +// setenv/unsetenv do not exist on MSVC (same helper shape as +// weights_mmap_smoke). A null value unsets. +void set_env(const char * name, const char * value) { +#if defined(_WIN32) + _putenv_s(name, value != nullptr ? value : ""); +#else + if (value != nullptr) { + setenv(name, value, 1); + } else { + unsetenv(name); + } +#endif +} + +struct Pair { + ggml_tensor * gate = nullptr; + ggml_tensor * up = nullptr; + ggml_tensor * fused = nullptr; +}; + +// Fill a [hidden, intermediate] F32 tensor with a value derived from `seed` and +// the element index, so a wrong source tensor (or a shifted copy) is detected +// by the byte comparison rather than by luck. +void fill(ggml_tensor * t, int seed) { + std::vector data(static_cast(ggml_nelements(t))); + for (size_t i = 0; i < data.size(); ++i) { + data[i] = static_cast(seed) * 1000.0f + static_cast(i); + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); +} + +std::vector bytes_of(const ggml_tensor * t) { + std::vector out(ggml_nbytes(t)); + ggml_backend_tensor_get(t, out.data(), 0, out.size()); + return out; +} + +// The fused tensor must be gate's bytes followed by up's, whichever storage it +// ended up with. +void check_fused_bytes(const Pair & p) { + const std::vector gate = bytes_of(p.gate); + const std::vector up = bytes_of(p.up); + std::vector want = gate; + want.insert(want.end(), up.begin(), up.end()); + const std::vector got = bytes_of(p.fused); + CHECK(want.size() == got.size()); + CHECK(want.size() == got.size() && std::memcmp(want.data(), got.data(), got.size()) == 0); + CHECK(ggml_nbytes(p.fused) == ggml_nbytes(p.gate) + ggml_nbytes(p.up)); +} + +} // namespace + +int main() { + using transcribe::causal_lm::GateUpEntry; + using transcribe::causal_lm::pack_gate_up; + using transcribe::causal_lm::PackedGateUpHandles; + + ggml_backend_t cpu_be = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + if (cpu_be == nullptr) { + std::fprintf(stderr, "SKIP: could not init CPU backend\n"); + return 77; + } + + // --------------------------------------------------------------- + // Case 1: adjacent gate/up in one buffer -> aliased, no new buffer. + // --------------------------------------------------------------- + { + ggml_init_params params = { 2 * ggml_tensor_overhead() + 256, nullptr, true }; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + Pair p; + p.gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.gate != nullptr); + p.up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.up != nullptr); + + ggml_backend_buffer_t weights = ggml_backend_alloc_ctx_tensors(ctx, cpu_be); + REQUIRE(weights != nullptr); + fill(p.gate, 1); + fill(p.up, 2); + + // alloc_ctx_tensors lays tensors out in catalog order, so up starts + // where gate ends. + REQUIRE(static_cast(p.gate->data) + ggml_nbytes(p.gate) == p.up->data); + + const std::vector gate_bytes_before = bytes_of(p.gate); + const std::vector up_bytes_before = bytes_of(p.up); + + std::vector entries = { + { p.gate, p.up, &p.fused } + }; + PackedGateUpHandles handles; + + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + const transcribe_status st = pack_gate_up(cpu_be, k_hidden, k_intermediate, entries, handles, "unit"); + CHECK(st == TRANSCRIBE_OK); + CHECK(p.fused != nullptr); + CHECK(handles.buffer == nullptr); + if (p.fused != nullptr) { + CHECK(p.fused->data == p.gate->data); + CHECK(p.fused->buffer == p.gate->buffer); + check_fused_bytes(p); + } + + // free() must not free the weights buffer the tensors point into: the + // aliased path owns nothing. Everything the handles owned (the packed + // ctx, holding `fused`'s metadata) is gone, so only the weights-side + // pointers may be touched from here on. + handles.free(); + CHECK(bytes_of(p.gate) == gate_bytes_before); + CHECK(bytes_of(p.up) == up_bytes_before); + + ggml_backend_buffer_free(weights); + ggml_free(ctx); + } + + // --------------------------------------------------------------- + // Case 2: a gap between gate and up -> copied. + // --------------------------------------------------------------- + { + ggml_init_params params = { 3 * ggml_tensor_overhead() + 256, nullptr, true }; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + Pair p; + p.gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.gate != nullptr); + // Sits between the two in catalog order, so up does not follow gate. + ggml_tensor * filler = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(filler != nullptr); + p.up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.up != nullptr); + + ggml_backend_buffer_t weights = ggml_backend_alloc_ctx_tensors(ctx, cpu_be); + REQUIRE(weights != nullptr); + fill(p.gate, 3); + fill(p.up, 4); + + REQUIRE(static_cast(p.gate->data) + ggml_nbytes(p.gate) != p.up->data); + + std::vector entries = { + { p.gate, p.up, &p.fused } + }; + PackedGateUpHandles handles; + + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + const transcribe_status st = pack_gate_up(cpu_be, k_hidden, k_intermediate, entries, handles, "unit"); + CHECK(st == TRANSCRIBE_OK); + CHECK(p.fused != nullptr); + CHECK(handles.buffer != nullptr); + if (p.fused != nullptr) { + CHECK(p.fused->data != p.gate->data); + check_fused_bytes(p); + } + + handles.free(); + ggml_backend_buffer_free(weights); + ggml_free(ctx); + } + + // --------------------------------------------------------------- + // Case 3: mixed - one adjacent block and one gapped block in one call. + // --------------------------------------------------------------- + { + ggml_init_params params = { 6 * ggml_tensor_overhead() + 256, nullptr, true }; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + // Catalog order: gate0, up0, gate1, filler, up1. + Pair a, b; + a.gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(a.gate != nullptr); + a.up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(a.up != nullptr); + b.gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(b.gate != nullptr); + ggml_tensor * filler = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(filler != nullptr); + b.up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(b.up != nullptr); + + ggml_backend_buffer_t weights = ggml_backend_alloc_ctx_tensors(ctx, cpu_be); + REQUIRE(weights != nullptr); + fill(a.gate, 5); + fill(a.up, 6); + fill(b.gate, 7); + fill(b.up, 8); + + REQUIRE(static_cast(a.gate->data) + ggml_nbytes(a.gate) == a.up->data); + REQUIRE(static_cast(b.gate->data) + ggml_nbytes(b.gate) != b.up->data); + + const std::vector gate_before = bytes_of(a.gate); + const std::vector b_gate_before = bytes_of(b.gate); + + std::vector entries = { + { a.gate, a.up, &a.fused }, + { b.gate, b.up, &b.fused } + }; + PackedGateUpHandles handles; + + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + const transcribe_status st = pack_gate_up(cpu_be, k_hidden, k_intermediate, entries, handles, "unit"); + CHECK(st == TRANSCRIBE_OK); + CHECK(a.fused != nullptr); + CHECK(b.fused != nullptr); + // The buffer is allocated for the copied block only. + CHECK(handles.buffer != nullptr); + if (a.fused != nullptr && b.fused != nullptr) { + CHECK(a.fused->data == a.gate->data); + CHECK(a.fused->buffer == a.gate->buffer); + CHECK(b.fused->data != b.gate->data); + CHECK(b.fused->buffer == handles.buffer); + check_fused_bytes(a); + check_fused_bytes(b); + } + + handles.free(); + // The aliased block's bytes were never ours, so they survive the handles; + // the copied block's buffer went with them. Only the weights-side + // pointers are legal to touch now (the packed ctx held `a.fused`). + CHECK(bytes_of(a.gate) == gate_before); + CHECK(bytes_of(b.gate) == b_gate_before); + ggml_backend_buffer_free(weights); + ggml_free(ctx); + } + + // --------------------------------------------------------------- + // Case 4: TRANSCRIBE_NO_GATE_UP_ALIAS forces the copy path. + // --------------------------------------------------------------- + { + ggml_init_params params = { 2 * ggml_tensor_overhead() + 256, nullptr, true }; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + Pair p; + p.gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.gate != nullptr); + p.up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.up != nullptr); + + ggml_backend_buffer_t weights = ggml_backend_alloc_ctx_tensors(ctx, cpu_be); + REQUIRE(weights != nullptr); + fill(p.gate, 9); + fill(p.up, 10); + + std::vector entries = { + { p.gate, p.up, &p.fused } + }; + PackedGateUpHandles handles; + + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", "1"); + const transcribe_status st = pack_gate_up(cpu_be, k_hidden, k_intermediate, entries, handles, "unit"); + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + CHECK(st == TRANSCRIBE_OK); + CHECK(handles.buffer != nullptr); + if (p.fused != nullptr) { + CHECK(p.fused->data != p.gate->data); + check_fused_bytes(p); + } + + handles.free(); + ggml_backend_buffer_free(weights); + ggml_free(ctx); + } + + // --------------------------------------------------------------- + // Case 5: gate+up do not add up to the fused shape -> error, not a + // silently short tensor (the aliased path would read past up otherwise). + // --------------------------------------------------------------- + { + ggml_init_params params = { 2 * ggml_tensor_overhead() + 256, nullptr, true }; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + + Pair p; + // gate/up are [hidden, intermediate], but the caller claims a fused + // tensor of 2 * (intermediate + 1) rows. + p.gate = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.gate != nullptr); + p.up = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, k_hidden, k_intermediate); + REQUIRE(p.up != nullptr); + + ggml_backend_buffer_t weights = ggml_backend_alloc_ctx_tensors(ctx, cpu_be); + REQUIRE(weights != nullptr); + fill(p.gate, 11); + fill(p.up, 12); + + std::vector entries = { + { p.gate, p.up, &p.fused } + }; + PackedGateUpHandles handles; + + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + const transcribe_status st = pack_gate_up(cpu_be, k_hidden, k_intermediate + 1, entries, handles, "unit"); + CHECK(st == TRANSCRIBE_ERR_GGUF); + + handles.free(); + ggml_backend_buffer_free(weights); + ggml_free(ctx); + } + + ggml_backend_free(cpu_be); + + if (g_failures > 0) { + std::fprintf(stderr, "pack_gate_up_unit: %d failures\n", g_failures); + return EXIT_FAILURE; + } + std::fprintf(stdout, "pack_gate_up_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/qwen3_asr_real_smoke_0_6b.cpp b/tests/qwen3_asr_real_smoke_0_6b.cpp index bd71699e4..399460d06 100644 --- a/tests/qwen3_asr_real_smoke_0_6b.cpp +++ b/tests/qwen3_asr_real_smoke_0_6b.cpp @@ -33,6 +33,7 @@ #include "arch/qwen3_asr/qwen3_asr.h" #include "arch/qwen3_asr/weights.h" +#include "ggml-backend.h" #include "ggml.h" #include "transcribe-model.h" #include "transcribe.h" @@ -253,6 +254,36 @@ int main() { } } + // ---- gate+up fusion covers real converter output ---- + // A mapped model keeps tensors at their file offsets, so the alias path + // applies exactly when the converter wrote ffn.up directly after ffn.gate. + // Asserting it here (not just on the hand-generated fixture) is what would + // catch a converter reordering: the memory win would disappear otherwise, + // with no wrong answers anywhere to notice it by. + { + const auto & b0 = qm->weights.dec_blocks[0]; + CHECK(b0.ffn_gate_w != nullptr && b0.ffn_up_w != nullptr && b0.ffn_gate_up_w != nullptr); + if (b0.ffn_gate_w != nullptr && b0.ffn_up_w != nullptr && b0.ffn_gate_up_w != nullptr) { + if (qm->weight_storage.is_mapped(b0.ffn_gate_w)) { + CHECK(qm->weight_storage.is_mapped(b0.ffn_up_w)); + CHECK(qm->packed_gate_up.buffer == nullptr); + CHECK(b0.ffn_gate_up_w->data == b0.ffn_gate_w->data); + } else { + // Copied weights: the packed path may copy, but the fused bytes + // still have to be gate then up. + std::vector want(ggml_nbytes(b0.ffn_gate_up_w)); + if (want.size() == ggml_nbytes(b0.ffn_gate_w) + ggml_nbytes(b0.ffn_up_w)) { + ggml_backend_tensor_get(b0.ffn_gate_w, want.data(), 0, ggml_nbytes(b0.ffn_gate_w)); + ggml_backend_tensor_get(b0.ffn_up_w, want.data() + ggml_nbytes(b0.ffn_gate_w), 0, + ggml_nbytes(b0.ffn_up_w)); + std::vector got(want.size()); + ggml_backend_tensor_get(b0.ffn_gate_up_w, got.data(), 0, got.size()); + CHECK(std::memcmp(want.data(), got.data(), got.size()) == 0); + } + } + } + } + // ---- Input-length contract (docs/input-limits.md) ---- // qwen3_asr is a hard-context-cap family: it publishes a finite audio // bound, the session limits query tracks the n_ctx cap, and an over-cap diff --git a/tests/qwen3_asr_smoke.cpp b/tests/qwen3_asr_smoke.cpp index 98ac5ba22..7a95e76fb 100644 --- a/tests/qwen3_asr_smoke.cpp +++ b/tests/qwen3_asr_smoke.cpp @@ -104,9 +104,27 @@ const transcribe::qwen3_asr::QwenAsrModel * qwen_view(const struct transcribe_mo return static_cast(m); } +// setenv/unsetenv do not exist on MSVC; same helper shape as weights_mmap_smoke. +// A null value unsets. +void set_env(const char * name, const char * value) { +#if defined(_WIN32) + _putenv_s(name, value != nullptr ? value : ""); +#else + if (value != nullptr) { + setenv(name, value, 1); + } else { + unsetenv(name); + } +#endif +} + } // namespace int main() { + // The gate+up assertions below describe the default configuration; a caller + // that exported the fork switch would otherwise fail them spuriously. + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + const std::string fixture = g_fixtures_dir + "/arch_qwen3_asr_minimal.gguf"; if (!file_exists(fixture)) { @@ -304,6 +322,35 @@ int main() { CHECK(b.ffn_gate_up_w != nullptr); } + // ----- gate+up packing: aliased storage, exact bytes ----- + // + // The catalog stores ffn.up straight after ffn.gate, so the packed tensor + // should alias those bytes instead of duplicating them (the whole point of + // the aliasing path: a copy doubles the FFN gate+up weight memory on every + // packed family). Its bytes must still be gate followed by up. + std::vector> packed_bytes; + { + CHECK(qm->packed_gate_up.buffer == nullptr); + for (const auto & b : qm->weights.dec_blocks) { + if (b.ffn_gate_up_w == nullptr) { + continue; + } + CHECK(b.ffn_gate_up_w->data == b.ffn_gate_w->data); + CHECK(b.ffn_gate_up_w->buffer == b.ffn_gate_w->buffer); + + const size_t want_gate = ggml_nbytes(b.ffn_gate_w); + const size_t want_up = ggml_nbytes(b.ffn_up_w); + std::vector want(want_gate + want_up); + std::vector got(ggml_nbytes(b.ffn_gate_up_w)); + ggml_backend_tensor_get(b.ffn_gate_w, want.data(), 0, want_gate); + ggml_backend_tensor_get(b.ffn_up_w, want.data() + want_gate, 0, want_up); + ggml_backend_tensor_get(b.ffn_gate_up_w, got.data(), 0, got.size()); + CHECK(want.size() == got.size() && std::memcmp(want.data(), got.data(), got.size()) == 0); + packed_bytes.push_back(std::move(got)); + } + CHECK(!packed_bytes.empty()); + } + // ----- Spot-check tensor shapes against the catalog ----- if (es.conv0_w != nullptr) { CHECK_EQ_INT(es.conv0_w->ne[0], 3); @@ -364,6 +411,38 @@ int main() { transcribe_model_free(model); + // ----- TRANSCRIBE_NO_GATE_UP_ALIAS forces the copying path ----- + // Same fused bytes either way: the switch must not change results. + { + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", "1"); + transcribe_model_load_params mp_copy; + transcribe_model_load_params_init(&mp_copy); + mp_copy.backend = TRANSCRIBE_BACKEND_CPU; + struct transcribe_model * copied_m = nullptr; + const transcribe_status st_copy = transcribe_model_load_file(fixture.c_str(), &mp_copy, &copied_m); + set_env("TRANSCRIBE_NO_GATE_UP_ALIAS", nullptr); + CHECK(st_copy == TRANSCRIBE_OK); + const auto * qm_copy = qwen_view(copied_m); + if (qm_copy != nullptr) { + CHECK(qm_copy->packed_gate_up.buffer != nullptr); + CHECK(qm_copy->packed_gate_up.ctx != nullptr); + size_t i = 0; + for (const auto & b : qm_copy->weights.dec_blocks) { + if (b.ffn_gate_up_w == nullptr || i >= packed_bytes.size()) { + continue; + } + std::vector got(ggml_nbytes(b.ffn_gate_up_w)); + ggml_backend_tensor_get(b.ffn_gate_up_w, got.data(), 0, got.size()); + const auto & want = packed_bytes[i]; + CHECK(want.size() == got.size() && std::memcmp(want.data(), got.data(), got.size()) == 0); + ++i; + } + } else { + CHECK(qm_copy != nullptr); + } + transcribe_model_free(copied_m); + } + if (g_failures > 0) { std::fprintf(stderr, "qwen3_asr_smoke: %d failures\n", g_failures); return EXIT_FAILURE;