From d77d19c05d6efabe1e845da011aa4a90455edc5d Mon Sep 17 00:00:00 2001 From: montvid Date: Mon, 5 Oct 2026 18:21:27 +0300 Subject: [PATCH 1/7] mmap model weights instead of copying them into anonymous memory Weights are currently read tensor-by-tensor with fin.read() into a staging vector and copied in with ggml_backend_tensor_set, so the whole model lives in anonymous heap. Anonymous pages cannot be reclaimed, so on Android every weight byte is charged to the process by the low-memory killer; file-backed pages can simply be dropped and re-read. On a 5.5 GB phone this is the difference between a 1.7B model running and being killed mid-session. Add load_common::map_tensor_data_cpu(), which mmaps the GGUF read-only, wraps the tensor-data region in ggml_backend_cpu_buffer_from_ptr (whose free_buffer is NULL, so it does not own the pages) and points each tensor at its file offset with ggml_backend_tensor_alloc. A MappedWeights RAII member on the model owns the mapping and is released after the buffer and ctx_meta, both of which point into it. It returns bool rather than transcribe_status: mapping is an optimization, so every failure - unsupported platform, non-CPU backend, open/mmap failure, or a misaligned or truncated file - falls back to the existing alloc_ctx_tensors + stream_tensor_data path, which re-validates and reports real corruption itself. Every tensor is checked before any is allocated so a bad file falls back cleanly rather than half-mapping ctx_meta. Wired up for qwen3_asr, parakeet and whisper. Each was audited first for writes to ctx_meta tensors after load, since mapping read-only turns such a write into a crash. Notably whisper's legacy .bin loader does write frontend.mel_filterbank and frontend.window in place, so only the GGUF path is mapped and bin_load.cpp keeps its own allocation. voxtral, granite, funasr_nano and medasr are untouched and still stream. Peak RssAnon, x86 Linux, Release, 4 threads, one 8.6 s clip: model file before after qwen3-asr 1.7B Q4_0 1114 MiB 1936 MiB 817 MiB parakeet 0.6B Q8_0 705 MiB 1128 MiB 423 MiB whisper distil Q8_0 792 MiB 897 MiB 102 MiB The reduction tracks file size, as expected. Peak RSS is roughly unchanged (file pages still count) but the anonymous share, which is what the OOM killer charges, drops by the size of the model. Transcripts are identical on all three, and dropping the staging copy makes load slightly faster; the only cost is page-fault latency on the first run against a cold page cache. Co-Authored-By: Claude Opus 5 (1M context) --- src/arch/parakeet/model.cpp | 47 ++++++++----- src/arch/parakeet/parakeet.h | 4 ++ src/arch/qwen3_asr/model.cpp | 40 +++++++---- src/arch/qwen3_asr/qwen3_asr.h | 4 ++ src/arch/whisper/model.cpp | 41 +++++++---- src/arch/whisper/whisper.h | 4 ++ src/transcribe-load-common.cpp | 121 +++++++++++++++++++++++++++++++++ src/transcribe-load-common.h | 47 +++++++++++++ 8 files changed, 266 insertions(+), 42 deletions(-) diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 12463145e..830706a01 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -131,6 +131,8 @@ ParakeetModel::~ParakeetModel() { safe_buffer_free(backend_buffer); backend_buffer = nullptr; } + // After the buffer and ctx_meta, both of which point into these pages. + weights_map.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -538,23 +540,36 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par // Allocate a backend buffer for every tensor in ctx_meta on the // primary backend; the weight bytes are streamed in below. - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "parakeet: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + // Prefer mapping the weights on a CPU backend: the file pages back the + // tensors directly, keeping the model out of anonymous memory and so out of + // Android's low-memory-killer budget. Safe here — BN fusion only reads the + // raw BN tensors and writes into its own bn_fused buffer, the conv_pw F32 + // promotion emits new tensors, and the decoder's tensor_set targets are its + // own joint/LSTM buffers. + // Falls back to allocate-and-copy whenever mapping is unavailable. + ggml_backend_buffer_t weights_buffer = nullptr; + if (transcribe::load_common::map_tensor_data_cpu(loader.path(), gguf_data, m->ctx_meta, m->plan, &weights_buffer, + &m->weights_map, "parakeet")) { + m->backend_buffer = weights_buffer; + } else { + weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); + if (weights_buffer == nullptr) { + gguf_free(gguf_data); + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "parakeet: ggml_backend_alloc_ctx_tensors failed"); + return TRANSCRIBE_ERR_OOM; + } + m->backend_buffer = weights_buffer; + ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - // Stream tensor data from the GGUF into the backend buffer slots - // (shared loop in transcribe-load-common.h; works on host-memory - // backends and discrete GPUs). - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "parakeet"); - st != TRANSCRIBE_OK) { - gguf_free(gguf_data); - return st; + // Stream tensor data from the GGUF into the backend buffer slots + // (shared loop in transcribe-load-common.h; works on host-memory + // backends and discrete GPUs). + if (const transcribe_status st = + transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "parakeet"); + st != TRANSCRIBE_OK) { + gguf_free(gguf_data); + return st; + } } // Multitalker bundle: claim the embedded Sortformer diarizer. The diff --git a/src/arch/parakeet/parakeet.h b/src/arch/parakeet/parakeet.h index ca8479b70..7942155ce 100644 --- a/src/arch/parakeet/parakeet.h +++ b/src/arch/parakeet/parakeet.h @@ -21,6 +21,7 @@ #include #include #include +#include "transcribe-load-common.h" struct ggml_context; struct ggml_tensor; @@ -86,6 +87,9 @@ struct ParakeetModel final : public transcribe_model { // reverse order. transcribe::BackendPlan plan; ggml_backend_buffer_t backend_buffer = nullptr; + // Owns the weight mmap when backend_buffer is a cpu-buffer-from-ptr over it. + // Must outlive backend_buffer and ctx_meta: both point into these pages. + transcribe::load_common::MappedWeights weights_map; // Fused BN parameters live in a separate ggml context + buffer, // computed at load time from the raw BN tensors. Freed in dtor. diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index a8d6d977d..fde6167bd 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -57,6 +57,8 @@ QwenAsrModel::~QwenAsrModel() { safe_buffer_free(backend_buffer); backend_buffer = nullptr; } + // After the buffer and ctx_meta, both of which point into these pages. + weights_map.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -247,20 +249,32 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + // Prefer mapping the weights on a CPU backend: the file pages back the + // tensors directly, keeping the model out of anonymous memory and so out of + // Android's low-memory-killer budget. Safe here — every + // ggml_backend_tensor_set in this arch targets graph-input tensors in their + // own contexts, and the packed gate/up weights are new tensors. + // Falls back to allocate-and-copy whenever mapping is unavailable. + ggml_backend_buffer_t weights_buffer = nullptr; + if (transcribe::load_common::map_tensor_data_cpu(loader.path(), gguf_data, m->ctx_meta, m->plan, &weights_buffer, + &m->weights_map, "qwen3_asr")) { + m->backend_buffer = weights_buffer; + } else { + weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); + if (weights_buffer == nullptr) { + gguf_free(gguf_data); + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr: ggml_backend_alloc_ctx_tensors failed"); + return TRANSCRIBE_ERR_OOM; + } + m->backend_buffer = weights_buffer; + ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "qwen3_asr"); - st != TRANSCRIBE_OK) { - gguf_free(gguf_data); - return st; + if (const transcribe_status st = + transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "qwen3_asr"); + st != TRANSCRIBE_OK) { + gguf_free(gguf_data); + return st; + } } gguf_free(gguf_data); diff --git a/src/arch/qwen3_asr/qwen3_asr.h b/src/arch/qwen3_asr/qwen3_asr.h index 3bf8f4a0e..5b064f562 100644 --- a/src/arch/qwen3_asr/qwen3_asr.h +++ b/src/arch/qwen3_asr/qwen3_asr.h @@ -20,6 +20,7 @@ #include #include #include +#include "transcribe-load-common.h" struct ggml_context; struct ggml_tensor; @@ -78,6 +79,9 @@ struct QwenAsrModel final : public transcribe_model { transcribe::BackendPlan plan; ggml_backend_buffer_t backend_buffer = nullptr; + // Owns the weight mmap when backend_buffer is a cpu-buffer-from-ptr over it. + // Must outlive backend_buffer and ctx_meta: both point into these pages. + transcribe::load_common::MappedWeights weights_map; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index 66e855ac9..dcb256632 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -61,6 +61,8 @@ WhisperModel::~WhisperModel() { safe_buffer_free(backend_buffer); backend_buffer = nullptr; } + // After the buffer and ctx_meta, both of which point into these pages. + weights_map.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -479,20 +481,33 @@ transcribe_status whisper_load(Loader & loader, m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "whisper: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + // Prefer mapping the weights on a CPU backend: the file pages back the + // tensors directly, keeping the model out of anonymous memory and so out of + // Android's low-memory-killer budget. Safe for this GGUF path, which only + // reads frontend.mel_filterbank and frontend.window into host buffers. The + // legacy .bin loader DOES write those two tensors in place, so it keeps its + // own allocation and is untouched. + // Falls back to allocate-and-copy whenever mapping is unavailable. + ggml_backend_buffer_t weights_buffer = nullptr; + if (transcribe::load_common::map_tensor_data_cpu(loader.path(), gguf_data, m->ctx_meta, m->plan, &weights_buffer, + &m->weights_map, "whisper")) { + m->backend_buffer = weights_buffer; + } else { + weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); + if (weights_buffer == nullptr) { + gguf_free(gguf_data); + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "whisper: ggml_backend_alloc_ctx_tensors failed"); + return TRANSCRIBE_ERR_OOM; + } + m->backend_buffer = weights_buffer; + ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "whisper"); - st != TRANSCRIBE_OK) { - gguf_free(gguf_data); - return st; + if (const transcribe_status st = + transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "whisper"); + st != TRANSCRIBE_OK) { + gguf_free(gguf_data); + return st; + } } gguf_free(gguf_data); diff --git a/src/arch/whisper/whisper.h b/src/arch/whisper/whisper.h index 184fb9eb2..ca8803fd2 100644 --- a/src/arch/whisper/whisper.h +++ b/src/arch/whisper/whisper.h @@ -18,6 +18,7 @@ #include #include #include +#include "transcribe-load-common.h" struct ggml_context; struct ggml_tensor; @@ -270,6 +271,9 @@ struct WhisperModel final : public transcribe_model { // Runtime backend plan. See transcribe-backend.h. transcribe::BackendPlan plan; ggml_backend_buffer_t backend_buffer = nullptr; + // Owns the weight mmap when backend_buffer is a cpu-buffer-from-ptr over it. + // Must outlive backend_buffer and ctx_meta: both point into these pages. + transcribe::load_common::MappedWeights weights_map; // Language token ids keyed by BCP-47 short code (from general.languages). // Resolves a params.language string to the <|lang_xx|> token id (whisper diff --git a/src/transcribe-load-common.cpp b/src/transcribe-load-common.cpp index 0f17b856b..6287e2a95 100644 --- a/src/transcribe-load-common.cpp +++ b/src/transcribe-load-common.cpp @@ -4,6 +4,12 @@ // common backend-init and tensor-stream logic shared by per-family load(). #include "transcribe-load-common.h" +#if !defined(_WIN32) +# include +# include +# include +# include +#endif #include "ggml-alloc.h" #include "ggml-backend.h" @@ -440,6 +446,121 @@ transcribe_status init_backends(transcribe_backend_request requested, return TRANSCRIBE_ERR_INVALID_ARG; } + +void MappedWeights::reset() { +#if !defined(_WIN32) + if (addr != nullptr) { + munmap(addr, len); + } +#endif + addr = nullptr; + len = 0; +} + +bool map_tensor_data_cpu(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + ggml_backend_buffer_t * out_buffer, + MappedWeights * out_map, + const char * error_tag) { +#if defined(_WIN32) + (void) path; (void) gguf_data; (void) ctx_meta; (void) plan; + (void) out_buffer; (void) out_map; (void) error_tag; + return false; +#else + if (plan.primary_kind != BackendKind::Cpu || plan.primary == nullptr) { + return false; + } + + const int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: mmap open failed; streaming instead", error_tag); + return false; + } + struct stat sb {}; + if (::fstat(fd, &sb) != 0 || sb.st_size <= 0) { + ::close(fd); + return false; + } + const size_t file_size = static_cast(sb.st_size); + + void * base = ::mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0); + ::close(fd); // the mapping holds its own reference + if (base == MAP_FAILED) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: mmap failed; streaming instead", error_tag); + return false; + } + + const size_t data_offset = gguf_get_data_offset(gguf_data); + if (data_offset >= file_size) { + ::munmap(base, file_size); + return false; + } + + // Weights are touched in essentially random order during a run, and we do + // not want readahead pulling in pages that may never be used. + (void) ::madvise(static_cast(base) + data_offset, file_size - data_offset, MADV_RANDOM); + + ggml_backend_buffer_t buffer = + ggml_backend_cpu_buffer_from_ptr(static_cast(base) + data_offset, file_size - data_offset); + if (buffer == nullptr) { + ::munmap(base, file_size); + return false; + } + const size_t alignment = ggml_backend_buffer_get_alignment(buffer); + + // Validate every tensor before allocating any: a half-mapped ctx_meta would + // be worse than falling back cleanly. + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + const int64_t idx = gguf_find_tensor(gguf_data, t->name); + if (idx < 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: tensor \"%s\" not in gguf data; streaming instead", error_tag, + t->name); + safe_buffer_free(buffer); + ::munmap(base, file_size); + return false; + } + const size_t toffset = gguf_get_tensor_offset(gguf_data, idx); + if (data_offset + toffset + ggml_nbytes(t) > file_size) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: tensor \"%s\" runs past EOF; streaming instead", error_tag, + t->name); + safe_buffer_free(buffer); + ::munmap(base, file_size); + return false; + } + const auto addr_val = reinterpret_cast(static_cast(base) + data_offset + toffset); + if (alignment > 1 && (addr_val % alignment) != 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: tensor \"%s\" misaligned for mmap; streaming instead", error_tag, + t->name); + safe_buffer_free(buffer); + ::munmap(base, file_size); + return false; + } + } + + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + const int64_t idx = gguf_find_tensor(gguf_data, t->name); + const size_t toffset = gguf_get_tensor_offset(gguf_data, idx); + if (ggml_backend_tensor_alloc(buffer, t, static_cast(base) + data_offset + toffset) != + GGML_STATUS_SUCCESS) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: tensor_alloc failed for \"%s\"", error_tag, t->name); + safe_buffer_free(buffer); + ::munmap(base, file_size); + return false; + } + } + + ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + *out_buffer = buffer; + out_map->addr = base; + out_map->len = file_size; + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: weights mapped (%zu MiB file-backed)", error_tag, + (file_size - data_offset) / (1024 * 1024)); + return true; +#endif +} + transcribe_status stream_tensor_data(const std::string & path, const gguf_context * gguf_data, ggml_context * ctx_meta, diff --git a/src/transcribe-load-common.h b/src/transcribe-load-common.h index 4d3eb7aa7..13eb6c539 100644 --- a/src/transcribe-load-common.h +++ b/src/transcribe-load-common.h @@ -98,6 +98,53 @@ transcribe_status init_backends(transcribe_backend_request requested, // I/O or tensor-not-found failure. On failure ctx_meta's tensors // may be partially populated; the caller should discard the // whole model state on error. +// A read-only mapping of a GGUF file, used to back model weights directly +// instead of copying them into anonymous memory. Anonymous pages cannot be +// reclaimed under pressure, so on Android every weight byte counts against the +// low-memory killer's budget for the process; file-backed pages can be dropped +// and re-read instead. Move-only: the mapping must outlive every tensor that +// points into it. +struct MappedWeights { + void * addr = nullptr; + size_t len = 0; + + MappedWeights() = default; + MappedWeights(const MappedWeights &) = delete; + MappedWeights & operator=(const MappedWeights &) = delete; + MappedWeights(MappedWeights && o) noexcept : addr(o.addr), len(o.len) { o.addr = nullptr; o.len = 0; } + MappedWeights & operator=(MappedWeights && o) noexcept { + if (this != &o) { reset(); addr = o.addr; len = o.len; o.addr = nullptr; o.len = 0; } + return *this; + } + ~MappedWeights() { reset(); } + + bool valid() const { return addr != nullptr; } + void reset(); +}; + +// Point every tensor in `ctx_meta` at an mmap of `path` instead of allocating a +// backend buffer and copying into it. CPU primary backend only: anything else +// needs a real upload, and there is nothing to gain from mapping first. +// +// Returns true on success, with `*out_buffer` a cpu-buffer-from-ptr over the +// tensor-data region (it does not own the pages) and `*out_map` owning the +// mapping. Returns false for every other outcome — unsupported platform, +// non-CPU backend, open/mmap failure, or a file whose tensors are misaligned or +// truncated — leaving nothing allocated and ctx_meta untouched, so the caller +// falls back to ggml_backend_alloc_ctx_tensors + stream_tensor_data. Mapping is +// purely an optimization, so a false return is never an error: the streaming +// path re-validates and reports real corruption itself. +// +// Only safe for architectures that never write to ctx_meta tensors after load. +// Check every ggml_backend_tensor_set target in the arch before adopting it. +bool map_tensor_data_cpu(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + ggml_backend_buffer_t * out_buffer, + MappedWeights * out_map, + const char * error_tag); + transcribe_status stream_tensor_data(const std::string & path, const gguf_context * gguf_data, ggml_context * ctx_meta, From 10f3caa6e955adff5fe3a8758ed5a37ef0aae996 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 18 Oct 2026 03:17:04 -0700 Subject: [PATCH 2/7] ggml: return NULL when Metal cannot wrap a host pointer ggml_backend_metal_device_buffer_mapped wrapped the result of ggml_metal_buffer_map in a backend buffer without checking it, so a failed mapping (newBufferWithBytesNoCopy or residency-set setup returning nil) produced a buffer with a NULL context that crashed on first use. Return NULL instead, as ggml_backend_dev_buffer_from_host_ptr's other implementations do, so a caller can fall back to a copied buffer. Downstream patch 0003, recorded in ggml/UPSTREAM. --- ggml/UPSTREAM | 1 + ggml/src/ggml-metal/ggml-metal.cpp | 5 +++++ patches/ggml/0003-metal-mapped-buffer-null.patch | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 patches/ggml/0003-metal-mapped-buffer-null.patch diff --git a/ggml/UPSTREAM b/ggml/UPSTREAM index 6bcede247..923e32dff 100644 --- a/ggml/UPSTREAM +++ b/ggml/UPSTREAM @@ -3,6 +3,7 @@ sha: d7cb574130e6f01ad25b3289685489200febcd74 patches: patches/ggml/0001-fix-threadpool-oversubscription.patch patches/ggml/0002-backend-reg-filter.patch + patches/ggml/0003-metal-mapped-buffer-null.patch This directory is generated from the upstream ggml tree at the SHA above, minus .github/, with the listed downstream patches applied in order. Do not edit it by diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 344882e04..9e4d970e0 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -744,6 +744,11 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size); + if (res == NULL) { + // transcribe.cpp patch: report the failed mapping instead of wrapping + // a NULL context in a buffer that crashes on first use. + return NULL; + } const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev); diff --git a/patches/ggml/0003-metal-mapped-buffer-null.patch b/patches/ggml/0003-metal-mapped-buffer-null.patch new file mode 100644 index 000000000..175c73d55 --- /dev/null +++ b/patches/ggml/0003-metal-mapped-buffer-null.patch @@ -0,0 +1,16 @@ +diff --git a/src/ggml-metal/ggml-metal.cpp b/src/ggml-metal/ggml-metal.cpp +index 344882e0..9e4d970e 100644 +--- a/src/ggml-metal/ggml-metal.cpp ++++ b/src/ggml-metal/ggml-metal.cpp +@@ -744,6 +744,11 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen + ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; + + ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size); ++ if (res == NULL) { ++ // transcribe.cpp patch: report the failed mapping instead of wrapping ++ // a NULL context in a buffer that crashes on first use. ++ return NULL; ++ } + + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev); + From cea3bb2f022c8e42f21c531e964ebab3ece00cef Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 18 Oct 2026 03:17:24 -0700 Subject: [PATCH 3/7] cohere: fold the encoder Q bias into new tensors, not the GGUF weights fuse_encoder_q_bias added each block's linear_q bias into pos_bias_u / pos_bias_v in place with ggml_backend_tensor_set. That is the one place any family writes into a GGUF-backed weight after load; with the weights memory-mapped read-only it would crash, and it made the loaded tensors differ from the file. Write the sums into new F32 tensors (same shape and name) in their own q_bias_fused ctx + buffer on the primary backend, the way fuse_batch_norm handles BN, and repoint the block's pos_u/pos_v slots. Inputs that are missing or not F32 [d_model] leave that block unfused (q_b add kept). Same arithmetic, so outputs are unchanged. --- src/arch/cohere/cohere.h | 5 +++ src/arch/cohere/model.cpp | 81 +++++++++++++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/src/arch/cohere/cohere.h b/src/arch/cohere/cohere.h index 0a7e98604..b759ea3c9 100644 --- a/src/arch/cohere/cohere.h +++ b/src/arch/cohere/cohere.h @@ -138,6 +138,11 @@ struct CohereModel final : public transcribe_model { ggml_context * bn_fused_ctx = nullptr; ggml_backend_buffer_t bn_fused_buffer = nullptr; + // pos_bias_u/v with the encoder Q bias folded in (fuse_encoder_q_bias); + // block attn_pos_u/v slots point here once the fusion has run. + ggml_context * q_bias_fused_ctx = nullptr; + ggml_backend_buffer_t q_bias_fused_buffer = nullptr; + // On CPU primary backend, the conformer 1×1 pointwise conv weights // are dequantized F16->F32 at load time (Zen 2 has no native F16 // compute). Tensors live here; CohereBlock slots point at them. diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index fc9cc8a02..dc2abc80d 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -187,6 +187,14 @@ bool kv_cache_init_batched(CohereKvCache & cache, } CohereModel::~CohereModel() { + if (q_bias_fused_ctx != nullptr) { + ggml_free(q_bias_fused_ctx); + q_bias_fused_ctx = nullptr; + } + if (q_bias_fused_buffer != nullptr) { + safe_buffer_free(q_bias_fused_buffer); + q_bias_fused_buffer = nullptr; + } if (bn_fused_ctx != nullptr) { ggml_free(bn_fused_ctx); bn_fused_ctx = nullptr; @@ -336,12 +344,29 @@ transcribe_status fuse_batch_norm(CohereModel & m) { return TRANSCRIBE_OK; } +// A block whose Q-bias fusion inputs are all present as F32 [d_model] +// (weights.cpp binds them as F32 already; re-checked because the fusion reads +// raw floats). +bool q_bias_fusable(const CohereBlock & b, int64_t d_model) { + const ggml_tensor * ts[] = { b.attn_q_b, b.attn_pos_u, b.attn_pos_v }; + for (const ggml_tensor * t : ts) { + if (t == nullptr || t->type != GGML_TYPE_F32 || ggml_nelements(t) != d_model) { + return false; + } + } + return true; +} + // Fold each encoder layer's Q bias into its pos_bias_u / pos_bias_v at // load time. In rel_pos_mhsa: // q_u = (W_q x + q_b) + pos_u = W_q x + (q_b + pos_u) // q_v = (W_q x + q_b) + pos_v = W_q x + (q_b + pos_v) // so pre-adding q_b into pos_u/pos_v is identical and drops the explicit -// `q = q + q_b` graph op. We null attn_q_b after so the builder skips the add. +// `q = q + q_b` graph op. The sums go into new tensors in q_bias_fused_ctx / +// q_bias_fused_buffer (the GGUF-backed pos_bias tensors may be a read-only +// memory map, and weights are never written after load); the block's +// pos_u/pos_v slots are repointed at them and attn_q_b is nulled so the +// builder skips the add. transcribe_status fuse_encoder_q_bias(CohereModel & m) { const size_t n_blocks = m.weights.blocks.size(); if (n_blocks == 0) { @@ -360,15 +385,49 @@ transcribe_status fuse_encoder_q_bias(CohereModel & m) { return TRANSCRIBE_OK; } + size_t n_fusable = 0; + for (const auto & b : m.weights.blocks) { + n_fusable += q_bias_fusable(b, d_model) ? 1 : 0; + } + if (n_fusable == 0) { + return TRANSCRIBE_OK; + } + + const size_t ctx_size = n_fusable * 2 * ggml_tensor_overhead() + 256; + ggml_init_params params = { ctx_size, nullptr, true }; + m.q_bias_fused_ctx = ggml_init(params); + if (m.q_bias_fused_ctx == nullptr) { + return TRANSCRIBE_ERR_OOM; + } + + // Same shape and name as the tensors they replace, on the same backend. + std::vector fused_u(n_blocks, nullptr); + std::vector fused_v(n_blocks, nullptr); + for (size_t i = 0; i < n_blocks; ++i) { + const auto & b = m.weights.blocks[i]; + if (!q_bias_fusable(b, d_model)) { + continue; + } + fused_u[i] = ggml_new_tensor(m.q_bias_fused_ctx, GGML_TYPE_F32, ggml_n_dims(b.attn_pos_u), b.attn_pos_u->ne); + fused_v[i] = ggml_new_tensor(m.q_bias_fused_ctx, GGML_TYPE_F32, ggml_n_dims(b.attn_pos_v), b.attn_pos_v->ne); + ggml_set_name(fused_u[i], ggml_get_name(b.attn_pos_u)); + ggml_set_name(fused_v[i], ggml_get_name(b.attn_pos_v)); + } + + m.q_bias_fused_buffer = ggml_backend_alloc_ctx_tensors(m.q_bias_fused_ctx, m.plan.primary); + if (m.q_bias_fused_buffer == nullptr) { + return TRANSCRIBE_ERR_OOM; + } + ggml_backend_buffer_set_usage(m.q_bias_fused_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + const size_t nbytes = static_cast(d_model) * sizeof(float); std::vector q_bias(d_model); std::vector pos_u(d_model); std::vector pos_v(d_model); - size_t fused = 0; for (size_t i = 0; i < n_blocks; ++i) { auto & b = m.weights.blocks[i]; - if (b.attn_q_b == nullptr || b.attn_pos_u == nullptr || b.attn_pos_v == nullptr) { + if (fused_u[i] == nullptr) { continue; } @@ -383,18 +442,16 @@ transcribe_status fuse_encoder_q_bias(CohereModel & m) { pos_v[j] += q_bias[j]; } - ggml_backend_tensor_set(b.attn_pos_u, pos_u.data(), 0, nbytes); - ggml_backend_tensor_set(b.attn_pos_v, pos_v.data(), 0, nbytes); + ggml_backend_tensor_set(fused_u[i], pos_u.data(), 0, nbytes); + ggml_backend_tensor_set(fused_v[i], pos_v.data(), 0, nbytes); - // Drop the reference so the graph builder skips the add. - // Tensor memory stays allocated inside ctx_meta. - b.attn_q_b = nullptr; - ++fused; + // Repoint the slots; drop q_b so the graph builder skips the add. + b.attn_pos_u = fused_u[i]; + b.attn_pos_v = fused_v[i]; + b.attn_q_b = nullptr; } - if (fused > 0) { - log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "cohere: fused Q bias into pos_u/pos_v for %zu encoder blocks", fused); - } + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "cohere: fused Q bias into pos_u/pos_v for %zu encoder blocks", n_fusable); return TRANSCRIBE_OK; } From 0120500e9f9160c6014079f0268a93c260e91eca Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 18 Oct 2026 03:17:51 -0700 Subject: [PATCH 4/7] load: memory-map model weights for every family, on CPU and Metal Builds on the previous commit's approach (map the GGUF, wrap it as a host-pointer buffer, point the tensors into it) and makes it the one weight load every family uses, everywhere the device can use host memory. Loader (src/transcribe-mmap.*, transcribe-load-common.*): - MappedFile maps the whole file read-only: mmap(PROT_READ, MAP_SHARED) on POSIX, CreateFileMappingW + MapViewOfFile on Windows (UTF-8 path through the wide conversion). It then starts reading the file in the background: F_RDADVISE on Apple, POSIX_FADV_SEQUENTIAL + WILLNEED on Linux/Android, PrefetchVirtualMemory on Windows. MADV_RANDOM made a cold-cache first run 70%+ slower; MADV_WILLNEED left macOS ~15% behind the copying load; F_RDADVISE matches it. - load_common::load_weights / allocate_weights replace the per-family alloc_ctx_tensors + stream_tensor_data pair. When mapping is allowed and the primary device reports buffer_from_host_ptr + mmap_support (CPU everywhere, Apple7+ Metal), the mapping is wrapped with ggml_backend_dev_buffer_from_host_ptr and every tensor whose file bytes are usable as-is (same type and size as in the file, in bounds, aligned for the device) points into it. Anything else goes into an ordinary buffer and is streamed, so a family that converts tensors at load (ecapa_tdnn widens Q8_0 to F16) maps the rest. Vulkan/CUDA/ROCm keep copying into device memory. - Wrapping the page-aligned mapping instead of base + data_offset fixes an abort: ggml_backend_cpu_buffer_from_ptr asserts a 32-byte-aligned base, which a valid GGUF with general.alignment < 32 does not give. - WeightStorage owns the mapped and copied buffers plus the mapping and frees them in order; it replaces each family's backend_buffer (and the previous commit's MappedWeights). Mapping or wrapping failures log and fall back to copying. - Legacy whisper.cpp / Silero .bin weights are converted on the host and stay copied (allocate_copied_weights). Mapped weights are read-only. An audit of every family found no writes into GGUF-backed tensors after load besides cohere's Q-bias fusion (fixed in the previous commit); CONTRIBUTING and the porting skill now say so. API: transcribe_model_load_params gains `bool no_mmap` (default false, zero-default like every field); TRANSCRIBE_NO_MMAP=1 forces it process wide. transcribe_model_load_file now hands families a full-size params view so callers built before the field existed read the default. CLI and transcribe-bench take --no-mmap. The header documents the mmap hazard: do not truncate or rewrite a loaded model file in place. Measured (same binary, --no-mmap vs default), peak footprint / RssAnon: macOS M4 CPU parakeet-tdt-0.6b Q8_0 1198 -> 504 MiB, qwen3-asr-0.6b Q8_0 1705 -> 745, whisper-large-v3-turbo F16 1860 -> 182, cohere Q8_0 3481 -> 1173 macOS M4 Metal parakeet 868 -> 172, qwen3-asr 1522 -> 550, whisper turbo 1877 -> 189, cohere 2493 -> 170 Linux aarch64 RssAnon parakeet 1092 -> 397, qwen3-asr 1471 -> 655, whisper turbo 1684 -> 129 Warm loads are 3-10x faster; steady-state encode/decode is unchanged; cold-cache first runs match the copying load. Every family's output is identical with and without --no-mmap on CPU and Metal (one model each, plus the multitalker bundle and a whisper.cpp .bin). Tests: transcribe_weights_mmap_smoke (fixtures, CI) checks the mapping decision for every loadable fixture family, bit-identical results mapped vs copied (silero VAD, ecapa langid incl. the partly-mapped Q8_0 model), the old struct_size, TRANSCRIBE_NO_MMAP, a misaligned GGUF (arch_silero_vad_misaligned.gguf: copied, same results, no abort), a truncated file, two models on one file, a non-ASCII path, and Metal when present. It crashes with SIGBUS if cohere's fusion writes in place again. Passes on macOS (CPU + Metal), Linux aarch64, and Windows x86_64 (MinGW build under Wine); the mmap code also compiles for Android API 21. Co-authored-by: montvid --- .claude/skills/porting-4-cpp/SKILL.md | 10 + CONTRIBUTING.md | 4 + README.md | 8 + docs/environment-variables.md | 1 + examples/cli/asr.cpp | 1 + examples/cli/cli.h | 3 +- examples/cli/main.cpp | 5 + include/transcribe.h | 25 + src/CMakeLists.txt | 1 + src/arch/canary/canary.h | 5 +- src/arch/canary/model.cpp | 18 +- src/arch/canary_qwen/canary_qwen.h | 5 +- src/arch/canary_qwen/model.cpp | 17 +- src/arch/cohere/cohere.h | 5 +- src/arch/cohere/model.cpp | 20 +- src/arch/ecapa_tdnn/ecapa_tdnn.h | 9 +- src/arch/ecapa_tdnn/model.cpp | 30 +- src/arch/funasr_nano/funasr_nano.h | 3 +- src/arch/funasr_nano/model.cpp | 17 +- src/arch/gigaam/gigaam.h | 5 +- src/arch/gigaam/model.cpp | 17 +- src/arch/granite/granite.h | 5 +- src/arch/granite/model.cpp | 18 +- src/arch/granite5_ctc/granite5_ctc.h | 5 +- src/arch/granite5_ctc/model.cpp | 17 +- src/arch/granite_nar/granite_nar.h | 5 +- src/arch/granite_nar/model.cpp | 18 +- src/arch/medasr/medasr.h | 5 +- src/arch/medasr/model.cpp | 17 +- src/arch/moonshine/model.cpp | 17 +- src/arch/moonshine/moonshine.h | 5 +- src/arch/moonshine_streaming/model.cpp | 18 +- .../moonshine_streaming/moonshine_streaming.h | 5 +- src/arch/moss/model.cpp | 18 +- src/arch/moss/moss.h | 3 +- src/arch/parakeet/model.cpp | 59 +-- src/arch/parakeet/parakeet.h | 9 +- src/arch/qwen3_asr/model.cpp | 38 +- src/arch/qwen3_asr/qwen3_asr.h | 7 +- src/arch/sensevoice/model.cpp | 17 +- src/arch/sensevoice/sensevoice.h | 5 +- src/arch/silero_vad/model.cpp | 29 +- src/arch/silero_vad/silero_vad.h | 11 +- src/arch/sortformer/model.cpp | 17 +- src/arch/sortformer/sortformer.h | 7 +- src/arch/voxtral/model.cpp | 18 +- src/arch/voxtral/voxtral.h | 3 +- src/arch/voxtral_realtime/model.cpp | 18 +- src/arch/voxtral_realtime/voxtral_realtime.h | 3 +- src/arch/whisper/bin_load.cpp | 12 +- src/arch/whisper/model.cpp | 39 +- src/arch/whisper/whisper.h | 9 +- src/transcribe-load-common.cpp | 340 +++++++++---- src/transcribe-load-common.h | 181 ++++--- src/transcribe-mmap.cpp | 248 ++++++++++ src/transcribe-mmap.h | 67 +++ src/transcribe.cpp | 15 + tests/CMakeLists.txt | 19 +- tests/api_smoke.c | 12 + tests/fixtures/make_gguf_fixtures.py | 53 +- tests/weights_mmap_smoke.cpp | 464 ++++++++++++++++++ tools/transcribe-bench/main.cpp | 8 +- 62 files changed, 1478 insertions(+), 595 deletions(-) create mode 100644 src/transcribe-mmap.cpp create mode 100644 src/transcribe-mmap.h create mode 100644 tests/weights_mmap_smoke.cpp diff --git a/.claude/skills/porting-4-cpp/SKILL.md b/.claude/skills/porting-4-cpp/SKILL.md index 8df400179..175c1c224 100644 --- a/.claude/skills/porting-4-cpp/SKILL.md +++ b/.claude/skills/porting-4-cpp/SKILL.md @@ -133,6 +133,16 @@ Pick the closest existing family by architecture pattern: not the whole file. Load the closest family's `src/arch//` as a working example. +Load weights the way every family does: `gguf_init_from_file` into +`ctx_meta`, bind the weight struct, `load_common::init_backends`, then +`load_common::load_weights` into a `load_common::WeightStorage` member that +the destructor resets before freeing the backends. Weights are memory-mapped +read-only wherever the device allows it, so never write a `ctx_meta` tensor +after load: fusions and promotions create new tensors in their own buffer and +repoint the weight slot (see `fuse_batch_norm`, cohere's +`fuse_encoder_q_bias`). A stray write crashes the process on CPU and Metal; +compare a default run against `--no-mmap` to confirm the outputs match. + Wire the new family into the arch dispatch in `src/` per the existing family wiring. Build iteratively: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2eb6e1cc2..94a194eba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,6 +128,10 @@ General rules: - Do not add new ggml operators, `ggml_type` values, or backend behavior without CPU validation, benchmark or accuracy evidence where relevant, and an upstream plan for ggml/llama.cpp when appropriate. +- Treat GGUF-backed weight tensors (a family's `ctx_meta`) as read-only after + load: they are memory-mapped from the file wherever the device allows it, so + a write crashes. Load-time transforms (fusions, promotions, packing) write + new tensors into their own buffer and repoint the weight slot. - Do not mix functional changes with broad reformatting or unrelated cleanup. - Keep comments concise. Explain non-obvious invariants, ABI contracts, tensor layout, numerical choices, and portability traps. Do not preserve local task diff --git a/README.md b/README.md index 098a27be4..c9237a39a 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,14 @@ Input must be 16 kHz mono WAV. Use `ffmpeg` or `sox` to convert other formats: ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav ``` +On the CPU and on Metal (Apple silicon), model weights are memory-mapped from +the GGUF file rather than copied into process memory: they live in the page +cache, which the OS can reclaim and share between processes, and loading is +faster. Don't truncate or rewrite a model file in place while a program has it +loaded; write a new file and rename it over the old one. `--no-mmap` +(`transcribe_model_load_params::no_mmap`, or `TRANSCRIBE_NO_MMAP=1`) copies the +weights instead. + ## Bindings Official bindings wrap the C API for other languages: diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 4e1ae2237..4bb0336da 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -25,6 +25,7 @@ of tests. | Variable | Effect | | --- | --- | | `TRANSCRIBE_BACKENDS=` | Restrict which backends may register: a comma-separated list of `cpu`, `metal`, `vulkan`, `cuda`, `rocm`, `other`, `all` (case-insensitive). An excluded backend never runs any code (no module load, no driver init). It can only narrow the mask a host passes to `transcribe_init_backends_ex()`, applies even if the host never calls it, and CPU is always kept. Unknown names are dropped and logged as an error naming what is still allowed, so a typo narrows rather than widens (`vulcan` means CPU-only). Unset or empty means `all`. Read once per process. | +| `TRANSCRIBE_NO_MMAP` | Copy model weights into memory instead of memory-mapping the model file, for every model load in the process (same as `transcribe_model_load_params::no_mmap`, CLI `--no-mmap`). Read at each load. | | `TRANSCRIBE_NO_FLASH` | Disable flash attention on encoder and decoder (forces the manual F32 path). | | `TRANSCRIBE_FORCE_FLASH` | Force flash attention on. Wins over `TRANSCRIBE_NO_FLASH` if both are set. | | `TRANSCRIBE_CONV_DIRECT_DW` / `TRANSCRIBE_CONV_NO_DIRECT_DW` | Force the depthwise-conv dispatch to the direct `conv_2d_dw` path / the im2col path, overriding the per-family backend default. | diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp index 071ba338b..3f8e49bfb 100644 --- a/examples/cli/asr.cpp +++ b/examples/cli/asr.cpp @@ -275,6 +275,7 @@ int run_asr_batch(const cli_args & args, std::ofstream * output) { transcribe_model_load_params_init(&mp); mp.backend = args.backend; mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + mp.no_mmap = args.no_mmap; if (args.device_index >= 0 && mp.device == nullptr) { std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); return EXIT_FAILURE; diff --git a/examples/cli/cli.h b/examples/cli/cli.h index a61f671da..96cbf5a90 100644 --- a/examples/cli/cli.h +++ b/examples/cli/cli.h @@ -36,7 +36,8 @@ struct cli_args { int n_ctx = 0; // 0 = model's true max; >0 lowers the cap transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + bool no_mmap = false; // --no-mmap: copy weights instead of mapping the file transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; // Generic prompting (transcribe_run_params::vocabulary / prompt / prefix). diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index aecb2e4de..ef7d4aa4b 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -66,6 +66,8 @@ void print_usage(const char * argv0) { " (default: auto)\n" " --device N exact device index from --list-devices, including 0\n" " (default: automatic device selection)\n" + " --no-mmap copy the weights into memory instead of memory-mapping\n" + " the model file (TRANSCRIBE_NO_MMAP=1 does the same)\n" " --timestamps TYPE timestamps: auto, none, segment, word, token (default: auto)\n" " --batch FILE batch mode: FILE has one wav path per line\n" " --batch-jsonl output one JSON line per file (for batch)\n" @@ -326,6 +328,8 @@ bool parse_args(int argc, char ** argv, cli_args & out) { std::fprintf(stderr, "error: --backend must be auto, cpu, cpu_accel, metal, vulkan, cuda, or rocm\n"); return false; } + } else if (a == "--no-mmap") { + out.no_mmap = true; } else if (a == "--device") { const char * v = take_value(a.c_str()); if (!v) { @@ -610,6 +614,7 @@ int run_file(const cli_args & args, std::ofstream * output) { transcribe_model_load_params_init(&mp); mp.backend = args.backend; mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + mp.no_mmap = args.no_mmap; if (args.device_index >= 0 && mp.device == nullptr) { std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); return EXIT_FAILURE; diff --git a/include/transcribe.h b/include/transcribe.h index 313c96e23..e690fe24c 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -1047,11 +1047,36 @@ TRANSCRIBE_API transcribe_device_t transcribe_model_device(const struct transcri * * Handles are process-local. Persist device_id (when available), then * enumerate and resolve a fresh handle in each process. + * + * no_mmap: false (the default) memory-maps the model file and serves the + * weights straight from it wherever the primary device can use host + * memory: the CPU on every platform, and Metal on Apple silicon. + * The weights then live in clean, file-backed pages that the OS can + * reclaim, share between models loaded from the same file, and does + * not count as the process's anonymous memory (what the Android + * low-memory killer and iOS/macOS jetsam charge). Loading is faster + * too; a first run on a cold page cache reads the weights from disk. + * Devices with their own memory (Vulkan, CUDA, ROCm) always get a + * copy, and weights a family converts at load are copied as well. + * true copies every weight into backend memory and closes the file, + * the behavior before memory mapping. + * + * While a memory-mapped model is loaded its file must not be + * truncated or rewritten in place: a later access to a changed page + * kills the process (SIGBUS; EXCEPTION_IN_PAGE_ERROR on Windows). + * Update a model file by writing a new file and renaming it over the + * old one. On Windows the file cannot be opened for writing or + * truncated until the model is freed. Use no_mmap for models on + * removable or network storage that may disappear. + * + * Setting the TRANSCRIBE_NO_MMAP environment variable (to anything + * but "0") forces no_mmap for every load in the process. */ struct transcribe_model_load_params { uint64_t struct_size; transcribe_backend_request backend; transcribe_device_t device; + bool no_mmap; }; TRANSCRIBE_API void transcribe_model_load_params_init(struct transcribe_model_load_params * params); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15c7685b8..68606dcc8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -31,6 +31,7 @@ add_library(transcribe transcribe-flash-policy.cpp transcribe-backend.cpp transcribe-load-common.cpp + transcribe-mmap.cpp transcribe-weights-util.cpp transcribe-bin-loader.cpp transcribe-batch-util.cpp diff --git a/src/arch/canary/canary.h b/src/arch/canary/canary.h index 27bac0115..7b0e9e7e9 100644 --- a/src/arch/canary/canary.h +++ b/src/arch/canary/canary.h @@ -9,6 +9,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -103,8 +104,8 @@ struct CanaryModel final : public transcribe_model { CanaryWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused BN parameters (same as parakeet/cohere). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index dbdbf488d..cdaf3ac64 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -198,10 +198,7 @@ CanaryModel::~CanaryModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -502,17 +499,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "canary"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "canary", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/canary_qwen/canary_qwen.h b/src/arch/canary_qwen/canary_qwen.h index 89ed2c87e..2674cae8f 100644 --- a/src/arch/canary_qwen/canary_qwen.h +++ b/src/arch/canary_qwen/canary_qwen.h @@ -19,6 +19,7 @@ #include "causal_lm/causal_lm.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -57,8 +58,8 @@ struct CanaryQwenModel final : public transcribe_model { CanaryQwenWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // BatchNorm fusion in the conformer conv module: fused (scale, bias) // computed at load time; running_mean/var unused at inference. diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 95698cbb0..5b00b4068 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -70,10 +70,7 @@ CanaryQwenModel::~CanaryQwenModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); if (bn_fused_buffer != nullptr) { safe_buffer_free(bn_fused_buffer); bn_fused_buffer = nullptr; @@ -648,16 +645,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary_qwen: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "canary_qwen"); + if (const transcribe_status st = load_common::load_weights(loader.path(), gguf_data, m->ctx_meta, m->plan, params, + "canary_qwen", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/cohere/cohere.h b/src/arch/cohere/cohere.h index b759ea3c9..09f1449e3 100644 --- a/src/arch/cohere/cohere.h +++ b/src/arch/cohere/cohere.h @@ -9,6 +9,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -131,8 +132,8 @@ struct CohereModel final : public transcribe_model { // Runtime backend plan — see transcribe-backend.h. scheduler_list // holds the backend handles plus a classified primary kind. - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused BN parameters (same as Parakeet). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index dc2abc80d..c38009f0d 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -215,10 +215,7 @@ CohereModel::~CohereModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -629,19 +626,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "cohere: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - // Stream tensor data from the GGUF file into the backend buffer - // slots. See transcribe-load-common.h for the shared loop. - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "cohere"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "cohere", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/ecapa_tdnn/ecapa_tdnn.h b/src/arch/ecapa_tdnn/ecapa_tdnn.h index f282ad5f0..00fba0014 100644 --- a/src/arch/ecapa_tdnn/ecapa_tdnn.h +++ b/src/arch/ecapa_tdnn/ecapa_tdnn.h @@ -7,6 +7,7 @@ #include "transcribe-backend.h" #include "transcribe-langid.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "weights.h" @@ -30,10 +31,10 @@ struct Model final : public transcribe_model { Weights weights; // GGUF metadata context: owns the ggml_tensor structs the weight slots - // above point at. Their data lives in backend_buffer. - ggml_context * ctx_meta = nullptr; - ggml_backend_buffer_t backend_buffer = nullptr; - BackendPlan plan; + // above point at. Their data lives in weight_storage. + ggml_context * ctx_meta = nullptr; + load_common::WeightStorage weight_storage; + BackendPlan plan; // Load-time derived weights (Weights::blk0_w_im2col): their own metadata // context and backend buffer, since ctx_meta is sized for the file. diff --git a/src/arch/ecapa_tdnn/model.cpp b/src/arch/ecapa_tdnn/model.cpp index f20293665..0b8451099 100644 --- a/src/arch/ecapa_tdnn/model.cpp +++ b/src/arch/ecapa_tdnn/model.cpp @@ -115,8 +115,13 @@ int widen_q8_0_weights(ggml_context * ctx_meta) { // load_common::stream_tensor_data, plus the Q8_0 -> F16 widening: a tensor // whose file type is Q8_0 and whose bound type is F16 is dequantized with // ggml's own Q8_0 reference and rounded to F16 on the way in. Every other -// tensor must be bound with its file type and is copied as-is. -transcribe_status stream_weights(const std::string & path, const gguf_context * gguf, ggml_context * ctx_meta) { +// tensor must be bound with its file type and is copied as-is, unless +// `storage` memory-mapped it (allocate_weights maps only tensors whose +// file bytes are usable as-is, so never a widened one). +transcribe_status stream_weights(const std::string & path, + const gguf_context * gguf, + ggml_context * ctx_meta, + const load_common::WeightStorage & storage) { std::ifstream fin(path_from_utf8(path), std::ios::binary); if (!fin) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to reopen %s for tensor data", kTag, path.c_str()); @@ -130,6 +135,9 @@ transcribe_status stream_weights(const std::string & path, const gguf_context * std::vector f16; for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (storage.is_mapped(t)) { + continue; // already backed by the file bytes + } const int64_t idx = gguf_find_tensor(gguf, t->name); if (idx < 0) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: tensor \"%s\" not in gguf data", kTag, t->name); @@ -310,14 +318,14 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - m->backend_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (m->backend_buffer == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", kTag); - return TRANSCRIBE_ERR_OOM; + // Tensors widened above differ from their file bytes, so allocate_weights + // copies them and maps the rest. + if (auto st = load_common::allocate_weights(loader.path(), guard.ctx, m->ctx_meta, m->plan, params, kTag, + m->weight_storage); + st != TRANSCRIBE_OK) { + return st; } - ggml_backend_buffer_set_usage(m->backend_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = stream_weights(loader.path(), guard.ctx, m->ctx_meta); st != TRANSCRIBE_OK) { + if (auto st = stream_weights(loader.path(), guard.ctx, m->ctx_meta, m->weight_storage); st != TRANSCRIBE_OK) { return st; } if (auto st = build_derived_weights(*m); st != TRANSCRIBE_OK) { @@ -519,9 +527,7 @@ Model::~Model() { if (ctx_meta != nullptr) { ggml_free(ctx_meta); } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - } + weight_storage.reset(); if (ctx_derived != nullptr) { ggml_free(ctx_derived); } diff --git a/src/arch/funasr_nano/funasr_nano.h b/src/arch/funasr_nano/funasr_nano.h index 31f765203..97ea4d515 100644 --- a/src/arch/funasr_nano/funasr_nano.h +++ b/src/arch/funasr_nano/funasr_nano.h @@ -8,6 +8,7 @@ #include "causal_lm/causal_lm.h" #include "transcribe-backend.h" #include "transcribe-kaldi-fbank.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -54,7 +55,7 @@ struct FunAsrNanoModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; // Constructed once at load() time; const-after-construction. Kaldi diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 8dfa567af..25af20815 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -57,10 +57,7 @@ FunAsrNanoModel::~FunAsrNanoModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -419,16 +416,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "funasr_nano: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "funasr_nano"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "funasr_nano", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/gigaam/gigaam.h b/src/arch/gigaam/gigaam.h index 8ceaaae16..3815b8395 100644 --- a/src/arch/gigaam/gigaam.h +++ b/src/arch/gigaam/gigaam.h @@ -12,6 +12,7 @@ #include "decoder.h" #include "mel.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -45,8 +46,8 @@ struct GigaamModel final : public transcribe_model { GigaamWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Family-specific mel frontend (HTK mel, power=2, log-clamp, // center=False, periodic Hann). See arch/gigaam/mel.{h,cpp}. diff --git a/src/arch/gigaam/model.cpp b/src/arch/gigaam/model.cpp index 42c4720ec..31195055f 100644 --- a/src/arch/gigaam/model.cpp +++ b/src/arch/gigaam/model.cpp @@ -73,10 +73,7 @@ GigaamModel::~GigaamModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -150,16 +147,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "gigaam: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "gigaam"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "gigaam", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite/granite.h b/src/arch/granite/granite.h index f5c6c1301..1e01166dd 100644 --- a/src/arch/granite/granite.h +++ b/src/arch/granite/granite.h @@ -27,6 +27,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -71,8 +72,8 @@ struct GraniteModel final : public transcribe_model { GraniteWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Pre-fused BatchNorm scale + bias for every encoder block. Lives // in a separate ggml context + backend buffer so we can write the diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 357a85b38..7005c537b 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -61,10 +61,7 @@ GraniteModel::~GraniteModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -420,17 +417,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "granite"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "granite", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite5_ctc/granite5_ctc.h b/src/arch/granite5_ctc/granite5_ctc.h index 072faa410..124b2b8b5 100644 --- a/src/arch/granite5_ctc/granite5_ctc.h +++ b/src/arch/granite5_ctc/granite5_ctc.h @@ -15,6 +15,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -42,8 +43,8 @@ struct Granite5CtcModel final : public transcribe_model { Granite5CtcWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Per-block fused BatchNorm scale/bias, precomputed at load // (model.cpp fuse_batch_norm). Lives in its own context/buffer so diff --git a/src/arch/granite5_ctc/model.cpp b/src/arch/granite5_ctc/model.cpp index ae4724dc0..7565077a1 100644 --- a/src/arch/granite5_ctc/model.cpp +++ b/src/arch/granite5_ctc/model.cpp @@ -46,10 +46,7 @@ Granite5CtcModel::~Granite5CtcModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); if (bn_fused_buffer != nullptr) { safe_buffer_free(bn_fused_buffer); bn_fused_buffer = nullptr; @@ -220,16 +217,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite5_ctc: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "granite5_ctc"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "granite5_ctc", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite_nar/granite_nar.h b/src/arch/granite_nar/granite_nar.h index 3e0a84bf6..500b86494 100644 --- a/src/arch/granite_nar/granite_nar.h +++ b/src/arch/granite_nar/granite_nar.h @@ -13,6 +13,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -40,8 +41,8 @@ struct GraniteNarModel final : public transcribe_model { GraniteNarWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Pre-fused BN (parakeet-style). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/granite_nar/model.cpp b/src/arch/granite_nar/model.cpp index 65c981a3a..0b02687a0 100644 --- a/src/arch/granite_nar/model.cpp +++ b/src/arch/granite_nar/model.cpp @@ -68,10 +68,7 @@ GraniteNarModel::~GraniteNarModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -340,17 +337,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite_nar: alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "granite_nar"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "granite_nar", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/medasr/medasr.h b/src/arch/medasr/medasr.h index 3d11881cf..0967d0016 100644 --- a/src/arch/medasr/medasr.h +++ b/src/arch/medasr/medasr.h @@ -6,6 +6,7 @@ #pragma once #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -35,8 +36,8 @@ struct MedAsrModel final : public transcribe_model { MedAsrWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; std::optional mel; diff --git a/src/arch/medasr/model.cpp b/src/arch/medasr/model.cpp index 2a1454c54..dff7be46b 100644 --- a/src/arch/medasr/model.cpp +++ b/src/arch/medasr/model.cpp @@ -61,10 +61,7 @@ MedAsrModel::~MedAsrModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -188,16 +185,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "medasr: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "medasr"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "medasr", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moonshine/model.cpp b/src/arch/moonshine/model.cpp index 2d93c23ec..00f61a647 100644 --- a/src/arch/moonshine/model.cpp +++ b/src/arch/moonshine/model.cpp @@ -53,10 +53,7 @@ MoonshineModel::~MoonshineModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -256,16 +253,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "moonshine: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "moonshine"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "moonshine", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moonshine/moonshine.h b/src/arch/moonshine/moonshine.h index e11044e65..b60d04ab3 100644 --- a/src/arch/moonshine/moonshine.h +++ b/src/arch/moonshine/moonshine.h @@ -14,6 +14,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -113,8 +114,8 @@ struct MoonshineModel final : public transcribe_model { MoonshineWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; MoonshineModel() = default; ~MoonshineModel() override; diff --git a/src/arch/moonshine_streaming/model.cpp b/src/arch/moonshine_streaming/model.cpp index 2c66045ca..7a6f150c3 100644 --- a/src/arch/moonshine_streaming/model.cpp +++ b/src/arch/moonshine_streaming/model.cpp @@ -69,10 +69,7 @@ MoonshineStreamingModel::~MoonshineStreamingModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -269,17 +266,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "moonshine_streaming: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "moonshine_streaming"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "moonshine_streaming", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moonshine_streaming/moonshine_streaming.h b/src/arch/moonshine_streaming/moonshine_streaming.h index 8cf76e9cb..cf08d28b9 100644 --- a/src/arch/moonshine_streaming/moonshine_streaming.h +++ b/src/arch/moonshine_streaming/moonshine_streaming.h @@ -13,6 +13,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -109,8 +110,8 @@ struct MoonshineStreamingModel final : public transcribe_model { MoonshineStreamingWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; MoonshineStreamingModel() = default; ~MoonshineStreamingModel() override; diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 6357a2f4b..f704cbdfe 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -109,10 +109,7 @@ MossModel::~MossModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -393,17 +390,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "moss: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "moss"); + if (const transcribe_status st = transcribe::load_common::load_weights(loader.path(), gguf_data, m->ctx_meta, + m->plan, params, "moss", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moss/moss.h b/src/arch/moss/moss.h index 9eef5aecd..01eba4f3a 100644 --- a/src/arch/moss/moss.h +++ b/src/arch/moss/moss.h @@ -16,6 +16,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -69,7 +70,7 @@ struct MossModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index 56dd4c4a3..a877781ea 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -1,11 +1,12 @@ // arch/parakeet/model.cpp - Parakeet family handler. // // load() binds a runtime backend, opens the GGUF with no_alloc=true so -// ggml builds the tensor catalog without touching the data section, -// allocates a backend buffer for every tensor, and streams the data -// section into it; afterward every borrowed ggml_tensor* in `weights` -// points at backend memory. The persistent model state is ctx_meta + -// backend_buffer + backends, freed in that order in the destructor. +// ggml builds the tensor catalog without touching the data section, then +// gives every tensor storage via load_common::load_weights (memory-mapped +// from the file where the device allows it, copied otherwise); afterward +// every borrowed ggml_tensor* in `weights` points at backend memory. The +// persistent model state is ctx_meta + weight_storage + backends, freed in +// that order in the destructor. // run() computes the mel front-end, runs the encoder graph, then // host-decodes (predictor + joint + TDT). @@ -95,8 +96,8 @@ void ParakeetSession::on_scratch_released() noexcept { } ParakeetModel::~ParakeetModel() { - // Teardown order: ctx_meta → backend_buffer → plan backends. The - // buffer must be freed before the backends (it holds a backend ref); + // Teardown order: ctx_meta → weight_storage → plan backends. The + // buffers must be freed before the backends (they hold a backend ref); // backends freed in reverse init order. if (bn_fused_ctx != nullptr) { ggml_free(bn_fused_ctx); @@ -129,12 +130,7 @@ ParakeetModel::~ParakeetModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } - // After the buffer and ctx_meta, both of which point into these pages. - weights_map.reset(); + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -540,38 +536,11 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - // Allocate a backend buffer for every tensor in ctx_meta on the - // primary backend; the weight bytes are streamed in below. - // Prefer mapping the weights on a CPU backend: the file pages back the - // tensors directly, keeping the model out of anonymous memory and so out of - // Android's low-memory-killer budget. Safe here — BN fusion only reads the - // raw BN tensors and writes into its own bn_fused buffer, the conv_pw F32 - // promotion emits new tensors, and the decoder's tensor_set targets are its - // own joint/LSTM buffers. - // Falls back to allocate-and-copy whenever mapping is unavailable. - ggml_backend_buffer_t weights_buffer = nullptr; - if (transcribe::load_common::map_tensor_data_cpu(loader.path(), gguf_data, m->ctx_meta, m->plan, &weights_buffer, - &m->weights_map, "parakeet")) { - m->backend_buffer = weights_buffer; - } else { - weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "parakeet: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - // Stream tensor data from the GGUF into the backend buffer slots - // (shared loop in transcribe-load-common.h; works on host-memory - // backends and discrete GPUs). - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "parakeet"); - st != TRANSCRIBE_OK) { - gguf_free(gguf_data); - return st; - } + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "parakeet", m->weight_storage); + st != TRANSCRIBE_OK) { + gguf_free(gguf_data); + return st; } // Multitalker bundle: claim the embedded Sortformer diarizer. The diff --git a/src/arch/parakeet/parakeet.h b/src/arch/parakeet/parakeet.h index e223becaf..30ae27439 100644 --- a/src/arch/parakeet/parakeet.h +++ b/src/arch/parakeet/parakeet.h @@ -10,6 +10,7 @@ #include "../sortformer/sortformer.h" // embedded diarizer (multitalker bundle) #include "decoder.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -21,7 +22,6 @@ #include #include #include -#include "transcribe-load-common.h" struct ggml_context; struct ggml_cgraph; @@ -105,11 +105,8 @@ struct ParakeetModel final : public transcribe_model { // load_common::init_backends (see transcribe-backend.h). plan's // scheduler_list owns the backends; the destructor frees them in // reverse order. - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; - // Owns the weight mmap when backend_buffer is a cpu-buffer-from-ptr over it. - // Must outlive backend_buffer and ctx_meta: both point into these pages. - transcribe::load_common::MappedWeights weights_map; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused BN parameters live in a separate ggml context + buffer, // computed at load time from the raw BN tensors. Freed in dtor. diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index d4a01f663..66a7868be 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -54,12 +54,7 @@ QwenAsrModel::~QwenAsrModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } - // After the buffer and ctx_meta, both of which point into these pages. - weights_map.reset(); + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -250,32 +245,11 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - // Prefer mapping the weights on a CPU backend: the file pages back the - // tensors directly, keeping the model out of anonymous memory and so out of - // Android's low-memory-killer budget. Safe here — every - // ggml_backend_tensor_set in this arch targets graph-input tensors in their - // own contexts, and the packed gate/up weights are new tensors. - // Falls back to allocate-and-copy whenever mapping is unavailable. - ggml_backend_buffer_t weights_buffer = nullptr; - if (transcribe::load_common::map_tensor_data_cpu(loader.path(), gguf_data, m->ctx_meta, m->plan, &weights_buffer, - &m->weights_map, "qwen3_asr")) { - m->backend_buffer = weights_buffer; - } else { - weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "qwen3_asr"); - st != TRANSCRIBE_OK) { - gguf_free(gguf_data); - return st; - } + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "qwen3_asr", m->weight_storage); + st != TRANSCRIBE_OK) { + gguf_free(gguf_data); + return st; } gguf_free(gguf_data); diff --git a/src/arch/qwen3_asr/qwen3_asr.h b/src/arch/qwen3_asr/qwen3_asr.h index 5b064f562..73b5634ba 100644 --- a/src/arch/qwen3_asr/qwen3_asr.h +++ b/src/arch/qwen3_asr/qwen3_asr.h @@ -10,6 +10,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -20,7 +21,6 @@ #include #include #include -#include "transcribe-load-common.h" struct ggml_context; struct ggml_tensor; @@ -78,10 +78,7 @@ struct QwenAsrModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; - // Owns the weight mmap when backend_buffer is a cpu-buffer-from-ptr over it. - // Must outlive backend_buffer and ctx_meta: both point into these pages. - transcribe::load_common::MappedWeights weights_map; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/sensevoice/model.cpp b/src/arch/sensevoice/model.cpp index 99c09c31f..5867a5069 100644 --- a/src/arch/sensevoice/model.cpp +++ b/src/arch/sensevoice/model.cpp @@ -50,10 +50,7 @@ SenseVoiceModel::~SenseVoiceModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -145,16 +142,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "sensevoice: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "sensevoice"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "sensevoice", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/sensevoice/sensevoice.h b/src/arch/sensevoice/sensevoice.h index 09564c8e5..bfdce0282 100644 --- a/src/arch/sensevoice/sensevoice.h +++ b/src/arch/sensevoice/sensevoice.h @@ -8,6 +8,7 @@ #include "transcribe-backend.h" #include "transcribe-kaldi-fbank.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -33,8 +34,8 @@ struct SenseVoiceModel final : public transcribe_model { SenseVoiceWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Constructed once at load() time; const-after-construction so // every context that derives from this model can share the cached diff --git a/src/arch/silero_vad/model.cpp b/src/arch/silero_vad/model.cpp index 3635e6de1..e2458d3dd 100644 --- a/src/arch/silero_vad/model.cpp +++ b/src/arch/silero_vad/model.cpp @@ -62,8 +62,13 @@ struct GgufGuard { GgufGuard & operator=(const GgufGuard &) = delete; }; -// Shared only by the two Silero file adapters. -transcribe_status init_storage(Model & m, const transcribe_model_load_params * params) { +// Shared only by the two Silero file adapters. `gguf` is null for the .bin +// adapter, whose weights are converted on the host and so always copied; the +// GGUF adapter memory-maps them where it can (load_common::allocate_weights). +transcribe_status init_storage(Model & m, + const transcribe_model_load_params * params, + const std::string & path, + const gguf_context * gguf) { // AUTO resolves to the strict CPU backend. The model is tiny and the CPU // path matches the reference to ~1e-6 in probability, while ggml-metal's // F32 mul_mm stages operands as half (about 2e-3). A specific GPU request @@ -81,13 +86,10 @@ transcribe_status init_storage(Model & m, const transcribe_model_load_params * p m.backend = ggml_backend_name(m.plan.primary); m.primary_backend = m.plan.primary; - m.backend_buffer = ggml_backend_alloc_ctx_tensors(m.ctx_meta, m.plan.primary); - if (m.backend_buffer == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", kTag); - return TRANSCRIBE_ERR_OOM; + if (gguf != nullptr) { + return load_common::allocate_weights(path, gguf, m.ctx_meta, m.plan, params, kTag, m.weight_storage); } - ggml_backend_buffer_set_usage(m.backend_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - return TRANSCRIBE_OK; + return load_common::allocate_copied_weights(m.ctx_meta, m.plan, kTag, m.weight_storage); } transcribe_status load(Loader & loader, const transcribe_model_load_params * params, transcribe_model ** out_model) { @@ -117,10 +119,11 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } - if (auto st = init_storage(*m, params); st != TRANSCRIBE_OK) { + if (auto st = init_storage(*m, params, loader.path(), guard.ctx); st != TRANSCRIBE_OK) { return st; } - if (auto st = load_common::stream_tensor_data(loader.path(), guard.ctx, m->ctx_meta, kTag); st != TRANSCRIBE_OK) { + if (auto st = load_common::stream_tensor_data(loader.path(), guard.ctx, m->ctx_meta, kTag, &m->weight_storage); + st != TRANSCRIBE_OK) { return st; } if (auto st = build_derived_weights(*m); st != TRANSCRIBE_OK) { @@ -428,7 +431,7 @@ transcribe_status load_from_bin(const char * path, if (auto st = bind_weights(m->ctx_meta, hp, m->weights); st != TRANSCRIBE_OK) { return st; } - if (auto st = init_storage(*m, params); st != TRANSCRIBE_OK) { + if (auto st = init_storage(*m, params, std::string(), nullptr); st != TRANSCRIBE_OK) { return st; } for (size_t i = 0; i < k_bin_count; ++i) { @@ -483,9 +486,7 @@ Model::~Model() { if (ctx_meta != nullptr) { ggml_free(ctx_meta); } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - } + weight_storage.reset(); if (ctx_derived != nullptr) { ggml_free(ctx_derived); } diff --git a/src/arch/silero_vad/silero_vad.h b/src/arch/silero_vad/silero_vad.h index 33ccdbe6a..911ce1f74 100644 --- a/src/arch/silero_vad/silero_vad.h +++ b/src/arch/silero_vad/silero_vad.h @@ -19,6 +19,7 @@ #pragma once #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-vad.h" @@ -102,11 +103,11 @@ struct Model final : public transcribe_model { HParams hparams; Weights weights; - ggml_context * ctx_meta = nullptr; // owns the file's tensor structs - ggml_backend_buffer_t backend_buffer = nullptr; - ggml_context * ctx_derived = nullptr; - ggml_backend_buffer_t derived_buffer = nullptr; - BackendPlan plan; + ggml_context * ctx_meta = nullptr; // owns the file's tensor structs + load_common::WeightStorage weight_storage; + ggml_context * ctx_derived = nullptr; + ggml_backend_buffer_t derived_buffer = nullptr; + BackendPlan plan; HostDecoder host; diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index dc8a5ca7d..09eb44557 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -53,9 +53,7 @@ SortformerModel::~SortformerModel() { if (ctx_meta != nullptr) { ggml_free(ctx_meta); } - if (backend_buffer != nullptr) { - transcribe::safe_buffer_free(backend_buffer); - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { transcribe::safe_backend_free(*it); } @@ -590,17 +588,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "sortformer: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "sortformer"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "sortformer", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/sortformer/sortformer.h b/src/arch/sortformer/sortformer.h index dd1bebeba..a9cb6a20e 100644 --- a/src/arch/sortformer/sortformer.h +++ b/src/arch/sortformer/sortformer.h @@ -15,6 +15,7 @@ #include "../parakeet/weights.h" // ParakeetHParams / ParakeetWeights (conformer) #include "transcribe-backend.h" #include "transcribe-diarize.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe/sortformer.h" // public preset enum + diarize ext @@ -52,9 +53,9 @@ struct SortformerModel final : public transcribe_model { // Sortformer-specific weights (encoder_proj + transformer + diar head). SortformerWeights weights; - ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + ggml_context * ctx_meta = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused conformer BN params (computed at load from the raw BN tensors). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index ae6575e19..577f8e883 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -62,10 +62,7 @@ VoxtralModel::~VoxtralModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -504,17 +501,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "voxtral"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "voxtral", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/voxtral/voxtral.h b/src/arch/voxtral/voxtral.h index 63adadee3..89876cc08 100644 --- a/src/arch/voxtral/voxtral.h +++ b/src/arch/voxtral/voxtral.h @@ -13,6 +13,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -60,7 +61,7 @@ struct VoxtralModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/voxtral_realtime/model.cpp b/src/arch/voxtral_realtime/model.cpp index eefcd228b..13bd06523 100644 --- a/src/arch/voxtral_realtime/model.cpp +++ b/src/arch/voxtral_realtime/model.cpp @@ -69,10 +69,7 @@ Model::~Model() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -263,17 +260,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral_realtime: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "voxtral_realtime"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "voxtral_realtime", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/voxtral_realtime/voxtral_realtime.h b/src/arch/voxtral_realtime/voxtral_realtime.h index 88a1ef49b..3a035eddd 100644 --- a/src/arch/voxtral_realtime/voxtral_realtime.h +++ b/src/arch/voxtral_realtime/voxtral_realtime.h @@ -10,6 +10,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -49,7 +50,7 @@ struct Model final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/whisper/bin_load.cpp b/src/arch/whisper/bin_load.cpp index 638fa6270..88e12278c 100644 --- a/src/arch/whisper/bin_load.cpp +++ b/src/arch/whisper/bin_load.cpp @@ -617,13 +617,13 @@ transcribe_status load_from_bin(const char * path m->primary_backend = m->plan.primary; // ---- Allocate backend buffer ---- - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (buf == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", kTag); - return TRANSCRIBE_ERR_OOM; + // Copied, never memory-mapped: the .bin payload is not laid out for ggml + // (unaligned offsets, renamed/reshaped tensors) and the frontend tensors + // below are written after allocation. + if (auto st = transcribe::load_common::allocate_copied_weights(m->ctx_meta, m->plan, kTag, m->weight_storage); + st != TRANSCRIBE_OK) { + return st; } - m->backend_buffer = buf; - ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // ---- Stream tensor bytes from the .bin ---- if (auto st = transcribe::bin_loader::stream_tensor_data_from_bin(bm.path, stream_slots, "whisper"); diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index 763b2abfb..435fb24b5 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -58,12 +58,7 @@ WhisperModel::~WhisperModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } - // After the buffer and ctx_meta, both of which point into these pages. - weights_map.reset(); + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -482,33 +477,11 @@ transcribe_status whisper_load(Loader & loader, m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - // Prefer mapping the weights on a CPU backend: the file pages back the - // tensors directly, keeping the model out of anonymous memory and so out of - // Android's low-memory-killer budget. Safe for this GGUF path, which only - // reads frontend.mel_filterbank and frontend.window into host buffers. The - // legacy .bin loader DOES write those two tensors in place, so it keeps its - // own allocation and is untouched. - // Falls back to allocate-and-copy whenever mapping is unavailable. - ggml_backend_buffer_t weights_buffer = nullptr; - if (transcribe::load_common::map_tensor_data_cpu(loader.path(), gguf_data, m->ctx_meta, m->plan, &weights_buffer, - &m->weights_map, "whisper")) { - m->backend_buffer = weights_buffer; - } else { - weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "whisper: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "whisper"); - st != TRANSCRIBE_OK) { - gguf_free(gguf_data); - return st; - } + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "whisper", m->weight_storage); + st != TRANSCRIBE_OK) { + gguf_free(gguf_data); + return st; } gguf_free(gguf_data); diff --git a/src/arch/whisper/whisper.h b/src/arch/whisper/whisper.h index ca8803fd2..bbdad49f3 100644 --- a/src/arch/whisper/whisper.h +++ b/src/arch/whisper/whisper.h @@ -7,6 +7,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -18,7 +19,6 @@ #include #include #include -#include "transcribe-load-common.h" struct ggml_context; struct ggml_tensor; @@ -269,11 +269,8 @@ struct WhisperModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; // Runtime backend plan. See transcribe-backend.h. - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; - // Owns the weight mmap when backend_buffer is a cpu-buffer-from-ptr over it. - // Must outlive backend_buffer and ctx_meta: both point into these pages. - transcribe::load_common::MappedWeights weights_map; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Language token ids keyed by BCP-47 short code (from general.languages). // Resolves a params.language string to the <|lang_xx|> token id (whisper diff --git a/src/transcribe-load-common.cpp b/src/transcribe-load-common.cpp index 6287e2a95..2edee65f1 100644 --- a/src/transcribe-load-common.cpp +++ b/src/transcribe-load-common.cpp @@ -4,12 +4,6 @@ // common backend-init and tensor-stream logic shared by per-family load(). #include "transcribe-load-common.h" -#if !defined(_WIN32) -# include -# include -# include -# include -#endif #include "ggml-alloc.h" #include "ggml-backend.h" @@ -21,9 +15,11 @@ # include "ggml-metal.h" #endif #include "transcribe-backend.h" +#include "transcribe-env.h" #include "transcribe-log.h" #include "transcribe-path.h" +#include #include #include #include @@ -446,125 +442,278 @@ transcribe_status init_backends(transcribe_backend_request requested, return TRANSCRIBE_ERR_INVALID_ARG; } +// --------------------------------------------------------------------------- +// Weight storage: memory-mapped where the device can use host memory, +// copied into a backend buffer otherwise. +// --------------------------------------------------------------------------- + +void WeightStorage::reset() noexcept { + // Buffers before the mapping. A host-pointer buffer only wraps the pages + // (its free never touches them), but it must not outlive them. + safe_buffer_free(mapped_buffer_); + mapped_buffer_ = nullptr; + safe_buffer_free(copy_buffer_); + copy_buffer_ = nullptr; + file_.reset(); +} -void MappedWeights::reset() { -#if !defined(_WIN32) - if (addr != nullptr) { - munmap(addr, len); - } -#endif - addr = nullptr; - len = 0; +bool WeightStorage::is_mapped(const ggml_tensor * t) const noexcept { + return t != nullptr && mapped_buffer_ != nullptr && t->buffer == mapped_buffer_; } -bool map_tensor_data_cpu(const std::string & path, - const gguf_context * gguf_data, - ggml_context * ctx_meta, - const BackendPlan & plan, - ggml_backend_buffer_t * out_buffer, - MappedWeights * out_map, - const char * error_tag) { -#if defined(_WIN32) - (void) path; (void) gguf_data; (void) ctx_meta; (void) plan; - (void) out_buffer; (void) out_map; (void) error_tag; - return false; -#else - if (plan.primary_kind != BackendKind::Cpu || plan.primary == nullptr) { +bool mmap_allowed(const transcribe_model_load_params * params) { + if (params != nullptr && params->no_mmap) { return false; } + return !env::flag("TRANSCRIBE_NO_MMAP"); +} + +namespace { - const int fd = ::open(path.c_str(), O_RDONLY); - if (fd < 0) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: mmap open failed; streaming instead", error_tag); +constexpr double k_mib = 1024.0 * 1024.0; + +// The primary device can keep weights in a wrapped host allocation. ggml says +// so through buffer_from_host_ptr (CPU, Metal) and mmap_support (false for +// integrated CUDA/Vulkan GPUs). Pre-Apple7 Metal GPUs are also excluded: they +// include the discrete AMD GPUs of Intel Macs, whose weights belong in VRAM +// rather than in shared system memory. +bool device_can_map(const BackendPlan & plan, ggml_backend_dev_t dev) { + if (dev == nullptr) { return false; } - struct stat sb {}; - if (::fstat(fd, &sb) != 0 || sb.st_size <= 0) { - ::close(fd); + ggml_backend_dev_props props{}; + ggml_backend_dev_get_props(dev, &props); + if (!props.caps.buffer_from_host_ptr || !props.caps.mmap_support) { return false; } - const size_t file_size = static_cast(sb.st_size); - - void * base = ::mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0); - ::close(fd); // the mapping holds its own reference - if (base == MAP_FAILED) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: mmap failed; streaming instead", error_tag); + if (plan.primary_kind == BackendKind::Metal && metal_backend_lacks_simdgroup_mm(plan.primary, dev)) { return false; } + return true; +} - const size_t data_offset = gguf_get_data_offset(gguf_data); - if (data_offset >= file_size) { - ::munmap(base, file_size); - return false; +struct MapCounts { + size_t n_mapped = 0; + size_t mapped_bytes = 0; + size_t n_layout = 0; // in-memory type/size differs from the file (converted at load) + size_t n_misaligned = 0; // file offset not aligned for the device +}; + +// Map every tensor of ctx_meta whose file bytes are usable as-is. When nothing +// can be mapped, `file` and `out_buffer` are left empty, `why` says why, and +// the result is still TRANSCRIBE_OK: the caller copies everything instead. +transcribe_status map_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + ggml_backend_dev_t dev, + const char * error_tag, + MappedFile & file, + ggml_backend_buffer_t & out_buffer, + MapCounts & counts, + std::string & why) { + if (!file.open(path, &why)) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: cannot memory-map %s (%s); copying weights instead", error_tag, + path.c_str(), why.c_str()); + why = "memory mapping failed"; + return TRANSCRIBE_OK; } - // Weights are touched in essentially random order during a run, and we do - // not want readahead pulling in pages that may never be used. - (void) ::madvise(static_cast(base) + data_offset, file_size - data_offset, MADV_RANDOM); + const size_t file_size = file.size(); + const size_t data_offset = gguf_get_data_offset(gguf_data); + char * base = static_cast(file.addr()); + + struct Candidate { + ggml_tensor * t; + char * addr; + }; + + std::vector candidates; + size_t max_tensor_size = 0; + if (data_offset <= file_size) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (t->data != nullptr || t->buffer != nullptr || t->view_src != nullptr) { + continue; // already has storage + } + const int64_t idx = gguf_find_tensor(gguf_data, ggml_get_name(t)); + if (idx < 0) { + continue; // stream_tensor_data reports it + } + const size_t nbytes = ggml_nbytes(t); + if (gguf_get_tensor_type(gguf_data, idx) != t->type || gguf_get_tensor_size(gguf_data, idx) != nbytes || + !ggml_is_contiguous(t)) { + ++counts.n_layout; + continue; + } + const size_t toffset = gguf_get_tensor_offset(gguf_data, idx); + if (toffset > file_size - data_offset || nbytes > file_size - data_offset - toffset) { + continue; // runs past EOF: the stream reports the truncation + } + candidates.push_back({ t, base + data_offset + toffset }); + max_tensor_size = std::max(max_tensor_size, nbytes); + } + } + if (candidates.empty()) { + file.reset(); + why = "no tensor is stored as-is in the file"; + return TRANSCRIBE_OK; + } - ggml_backend_buffer_t buffer = - ggml_backend_cpu_buffer_from_ptr(static_cast(base) + data_offset, file_size - data_offset); + // Wrap the whole page-aligned mapping, not just the data section: ggml's + // CPU host-pointer buffer asserts 32-byte alignment of its base, which a + // GGUF with general.alignment < 32 would not give data_offset. + ggml_backend_buffer_t buffer = ggml_backend_dev_buffer_from_host_ptr(dev, base, file_size, max_tensor_size); if (buffer == nullptr) { - ::munmap(base, file_size); - return false; + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: %s cannot wrap the memory-mapped file; copying weights instead", + error_tag, ggml_backend_dev_name(dev)); + file.reset(); + why = "device could not wrap the mapping"; + return TRANSCRIBE_OK; } - const size_t alignment = ggml_backend_buffer_get_alignment(buffer); - // Validate every tensor before allocating any: a half-mapped ctx_meta would - // be worse than falling back cleanly. - for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { - const int64_t idx = gguf_find_tensor(gguf_data, t->name); - if (idx < 0) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: tensor \"%s\" not in gguf data; streaming instead", error_tag, - t->name); - safe_buffer_free(buffer); - ::munmap(base, file_size); - return false; + const size_t alignment = ggml_backend_buffer_get_alignment(buffer); + for (const Candidate & c : candidates) { + const size_t nbytes = ggml_nbytes(c.t); + if (alignment > 1 && reinterpret_cast(c.addr) % alignment != 0) { + ++counts.n_misaligned; + continue; } - const size_t toffset = gguf_get_tensor_offset(gguf_data, idx); - if (data_offset + toffset + ggml_nbytes(t) > file_size) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: tensor \"%s\" runs past EOF; streaming instead", error_tag, - t->name); - safe_buffer_free(buffer); - ::munmap(base, file_size); - return false; + if (ggml_backend_buffer_get_alloc_size(buffer, c.t) != nbytes) { + ++counts.n_layout; + continue; } - const auto addr_val = reinterpret_cast(static_cast(base) + data_offset + toffset); - if (alignment > 1 && (addr_val % alignment) != 0) { - log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: tensor \"%s\" misaligned for mmap; streaming instead", error_tag, - t->name); - safe_buffer_free(buffer); - ::munmap(base, file_size); - return false; + if (ggml_backend_tensor_alloc(buffer, c.t, c.addr) != GGML_STATUS_SUCCESS) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: %s rejected memory-mapped tensor \"%s\"", error_tag, + ggml_backend_dev_name(dev), ggml_get_name(c.t)); + out_buffer = buffer; // owned by the storage from here; tensors may point into it + return TRANSCRIBE_ERR_BACKEND; } + ++counts.n_mapped; + counts.mapped_bytes += nbytes; } + if (counts.n_mapped == 0) { + safe_buffer_free(buffer); + file.reset(); + why = counts.n_misaligned > 0 ? "tensors misaligned in the file" : "no tensor is stored as-is in the file"; + return TRANSCRIBE_OK; + } + + ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + out_buffer = buffer; + return TRANSCRIBE_OK; +} +} // namespace + +transcribe_status allocate_copied_weights(ggml_context * ctx_meta, + const BackendPlan & plan, + const char * error_tag, + WeightStorage & out) { + bool any_unallocated = false; for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { - const int64_t idx = gguf_find_tensor(gguf_data, t->name); - const size_t toffset = gguf_get_tensor_offset(gguf_data, idx); - if (ggml_backend_tensor_alloc(buffer, t, static_cast(base) + data_offset + toffset) != - GGML_STATUS_SUCCESS) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: tensor_alloc failed for \"%s\"", error_tag, t->name); - safe_buffer_free(buffer); - ::munmap(base, file_size); - return false; + if (t->data == nullptr && t->view_src == nullptr) { + any_unallocated = true; + break; } } + if (!any_unallocated) { + return TRANSCRIBE_OK; + } + if (out.copy_buffer_ != nullptr || plan.primary == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: weight storage already allocated or no primary backend", error_tag); + return TRANSCRIBE_ERR_BACKEND; + } + // Allocates only the tensors that have no storage yet. + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_meta, plan.primary); + if (buffer == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", error_tag); + return TRANSCRIBE_ERR_OOM; + } ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - *out_buffer = buffer; - out_map->addr = base; - out_map->len = file_size; - log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: weights mapped (%zu MiB file-backed)", error_tag, - (file_size - data_offset) / (1024 * 1024)); - return true; -#endif + out.copy_buffer_ = buffer; + return TRANSCRIBE_OK; } -transcribe_status stream_tensor_data(const std::string & path, - const gguf_context * gguf_data, - ggml_context * ctx_meta, - const char * error_tag) { +transcribe_status allocate_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out) { + if (plan.primary == nullptr || gguf_data == nullptr || ctx_meta == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + ggml_backend_dev_t dev = ggml_backend_get_device(plan.primary); + + std::string why; + MapCounts counts; + if (!mmap_allowed(params)) { + why = "memory mapping disabled"; + } else if (!MappedFile::supported()) { + why = "memory mapping unsupported on this platform"; + } else if (!device_can_map(plan, dev)) { + why = "device keeps weights in its own memory"; + } else if (const transcribe_status st = + map_weights(path, gguf_data, ctx_meta, dev, error_tag, out.file_, out.mapped_buffer_, counts, why); + st != TRANSCRIBE_OK) { + return st; + } + + if (const transcribe_status st = allocate_copied_weights(ctx_meta, plan, error_tag, out); st != TRANSCRIBE_OK) { + return st; + } + + size_t n_copied = 0, copied_bytes = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (out.copy_buffer_ != nullptr && t->buffer == out.copy_buffer_) { + ++n_copied; + copied_bytes += ggml_nbytes(t); + } + } + if (counts.n_mapped == 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: weights copied: %.1f MiB in %s (%s)", error_tag, copied_bytes / k_mib, + out.copy_buffer_ != nullptr ? ggml_backend_buffer_name(out.copy_buffer_) : "-", why.c_str()); + } else if (n_copied == 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: weights memory-mapped: %.1f MiB in %s", error_tag, + counts.mapped_bytes / k_mib, ggml_backend_buffer_name(out.mapped_buffer_)); + } else { + char reasons[96]; + if (counts.n_layout > 0 && counts.n_misaligned > 0) { + std::snprintf(reasons, sizeof(reasons), "%zu converted at load, %zu misaligned in the file", + counts.n_layout, counts.n_misaligned); + } else if (counts.n_misaligned > 0) { + std::snprintf(reasons, sizeof(reasons), "misaligned in the file"); + } else { + std::snprintf(reasons, sizeof(reasons), "converted at load"); + } + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, + "%s: weights memory-mapped: %.1f MiB in %s; %zu tensors (%.1f MiB) copied (%s)", error_tag, + counts.mapped_bytes / k_mib, ggml_backend_buffer_name(out.mapped_buffer_), n_copied, + copied_bytes / k_mib, reasons); + } + return TRANSCRIBE_OK; +} + +transcribe_status load_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out) { + if (const transcribe_status st = allocate_weights(path, gguf_data, ctx_meta, plan, params, error_tag, out); + st != TRANSCRIBE_OK) { + return st; + } + return stream_tensor_data(path, gguf_data, ctx_meta, error_tag, &out); +} + +transcribe_status stream_tensor_data(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const char * error_tag, + const WeightStorage * skip) { // std::ifstream rather than going through gguf's loader a third // time: ifstream::seekg takes a streamoff (signed 64-bit on every // platform we target), so multi-GB tensor offsets work without @@ -579,6 +728,9 @@ transcribe_status stream_tensor_data(const std::string & path, std::vector staging; for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (skip != nullptr && skip->is_mapped(t)) { + continue; // already backed by the file bytes + } const int64_t idx = gguf_find_tensor(gguf_data, t->name); if (idx < 0) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: tensor \"%s\" not in gguf data", error_tag, t->name); diff --git a/src/transcribe-load-common.h b/src/transcribe-load-common.h index 13eb6c539..4cb065e04 100644 --- a/src/transcribe-load-common.h +++ b/src/transcribe-load-common.h @@ -2,14 +2,16 @@ // // INTERNAL, C++17. Not part of the public ABI. // -// Every per-family load() walks the same three steps after building the +// Every per-family load() walks the same two steps after building the // GGUF catalog: // -// 1. Resolve a BackendPlan from transcribe_model_load_params::backend. -// 2. Allocate a backend buffer for every tensor in ctx_meta on the plan's -// primary backend. -// 3. Stream each tensor's bytes from the GGUF data section into its -// backend slot via ggml_backend_tensor_set. +// 1. Resolve a BackendPlan from transcribe_model_load_params::backend +// (init_backends). +// 2. Give every tensor in ctx_meta storage on the plan's primary backend +// and fill it from the GGUF data section (load_weights). Where the +// device can use host memory directly the tensors point into a +// read-only memory map of the file; otherwise their bytes are copied +// into a backend buffer. // // These functions hoist that scaffolding; per-family load() still owns the // model, params validation, weights catalog, fusion, and destructor wiring. @@ -17,6 +19,7 @@ #pragma once #include "transcribe-backend.h" +#include "transcribe-mmap.h" #include "transcribe.h" #include @@ -79,11 +82,102 @@ transcribe_status init_backends(transcribe_backend_request requested, const char * error_tag, BackendPlan & out); -// Stream every tensor in `ctx_meta` from the GGUF data section on -// disk into its already-allocated backend buffer slot via -// ggml_backend_tensor_set. The backend buffer must already be -// allocated (typically via ggml_backend_alloc_ctx_tensors) before -// calling this. +// Storage behind a model's GGUF-backed weight tensors (its ctx_meta). +// +// Mapped tensors point straight into a read-only MappedFile of the GGUF, +// wrapped as a backend buffer (`mapped_buffer`). Every other tensor lives in +// an ordinary backend buffer filled from the file (`copy_buffer`). With +// mapping off, unavailable, or impossible for every tensor, copy_buffer +// holds them all: the classic path. +// +// Mapped weights are READ-ONLY: writing one (ggml_backend_tensor_set, an +// in-place graph op, a load-time fix-up) crashes. Load-time transforms must +// write their results into separate tensors in their own buffer, as +// fuse_batch_norm and promote_conv_pw_f16_to_f32_on_cpu do. A family should +// never write a ctx_meta tensor after load anyway: weights are shared by +// every session on the model. +class WeightStorage { + public: + WeightStorage() = default; + + ~WeightStorage() { reset(); } + + WeightStorage(const WeightStorage &) = delete; + WeightStorage & operator=(const WeightStorage &) = delete; + WeightStorage(WeightStorage &&) = delete; + WeightStorage & operator=(WeightStorage &&) = delete; + + // Free both buffers, then unmap. Call it where the family used to free + // its weight buffer (before the plan's backends); the destructor calls it + // again harmlessly. + void reset() noexcept; + + // True when `t`'s data lives in the file mapping. + bool is_mapped(const ggml_tensor * t) const noexcept; + + private: + friend transcribe_status allocate_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out); + friend transcribe_status allocate_copied_weights(ggml_context * ctx_meta, + const BackendPlan & plan, + const char * error_tag, + WeightStorage & out); + + MappedFile file_; + ggml_backend_buffer_t mapped_buffer_ = nullptr; + ggml_backend_buffer_t copy_buffer_ = nullptr; +}; + +// Whether a load may memory-map weights: false when params->no_mmap is set +// or the TRANSCRIBE_NO_MMAP environment flag is on. `params` may be NULL +// (defaults); otherwise it must be the full-size view that +// transcribe_model_load_file passes to the family loaders. +bool mmap_allowed(const transcribe_model_load_params * params); + +// Give every tensor in `ctx_meta` storage on plan.primary, without filling +// the copied ones. +// +// When mmap_allowed(params), the platform can map files, and the primary +// device can wrap host memory as a buffer (ggml caps buffer_from_host_ptr + +// mmap_support: the CPU everywhere and Apple-silicon Metal), the GGUF at +// `path` is mapped read-only and every tensor whose file bytes are usable +// as-is (same type and size as in the file, inside it, and aligned for the +// device) points into it. The rest, and everything on other devices (Vulkan, +// CUDA, ROCm upload into device memory), goes into an ordinary buffer. +// Mapping failures are not errors: they log and fall back to copying. +// +// Follow with stream_tensor_data(..., &out) (or a family streamer that skips +// out.is_mapped(t)) to fill the copied tensors. load_weights does both. +// +// Returns TRANSCRIBE_OK, TRANSCRIBE_ERR_OOM when the copy buffer cannot be +// allocated, or TRANSCRIBE_ERR_BACKEND when the device rejects a mapped +// tensor. On error `out` may hold partial storage; the caller discards the +// whole model, whose destructor resets it. +transcribe_status allocate_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out); + +// Allocate every still-unallocated tensor in `ctx_meta` in an ordinary +// buffer on plan.primary (no mapping). For loaders whose weights are not +// GGUF file bytes (legacy whisper.cpp / Silero .bin files). +transcribe_status allocate_copied_weights(ggml_context * ctx_meta, + const BackendPlan & plan, + const char * error_tag, + WeightStorage & out); + +// Stream tensor data from the GGUF data section on disk into +// already-allocated backend buffer slots via ggml_backend_tensor_set. +// Tensors that `skip` maps are left alone (they already hold the file +// bytes); with skip == nullptr every tensor in ctx_meta is streamed. // // path: filesystem path to the GGUF file, reopened fresh via // std::ifstream. Stable for the life of the call. @@ -98,57 +192,22 @@ transcribe_status init_backends(transcribe_backend_request requested, // I/O or tensor-not-found failure. On failure ctx_meta's tensors // may be partially populated; the caller should discard the // whole model state on error. -// A read-only mapping of a GGUF file, used to back model weights directly -// instead of copying them into anonymous memory. Anonymous pages cannot be -// reclaimed under pressure, so on Android every weight byte counts against the -// low-memory killer's budget for the process; file-backed pages can be dropped -// and re-read instead. Move-only: the mapping must outlive every tensor that -// points into it. -struct MappedWeights { - void * addr = nullptr; - size_t len = 0; - - MappedWeights() = default; - MappedWeights(const MappedWeights &) = delete; - MappedWeights & operator=(const MappedWeights &) = delete; - MappedWeights(MappedWeights && o) noexcept : addr(o.addr), len(o.len) { o.addr = nullptr; o.len = 0; } - MappedWeights & operator=(MappedWeights && o) noexcept { - if (this != &o) { reset(); addr = o.addr; len = o.len; o.addr = nullptr; o.len = 0; } - return *this; - } - ~MappedWeights() { reset(); } - - bool valid() const { return addr != nullptr; } - void reset(); -}; +transcribe_status stream_tensor_data(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const char * error_tag, + const WeightStorage * skip = nullptr); -// Point every tensor in `ctx_meta` at an mmap of `path` instead of allocating a -// backend buffer and copying into it. CPU primary backend only: anything else -// needs a real upload, and there is nothing to gain from mapping first. -// -// Returns true on success, with `*out_buffer` a cpu-buffer-from-ptr over the -// tensor-data region (it does not own the pages) and `*out_map` owning the -// mapping. Returns false for every other outcome — unsupported platform, -// non-CPU backend, open/mmap failure, or a file whose tensors are misaligned or -// truncated — leaving nothing allocated and ctx_meta untouched, so the caller -// falls back to ggml_backend_alloc_ctx_tensors + stream_tensor_data. Mapping is -// purely an optimization, so a false return is never an error: the streaming -// path re-validates and reports real corruption itself. -// -// Only safe for architectures that never write to ctx_meta tensors after load. -// Check every ggml_backend_tensor_set target in the arch before adopting it. -bool map_tensor_data_cpu(const std::string & path, - const gguf_context * gguf_data, - ggml_context * ctx_meta, - const BackendPlan & plan, - ggml_backend_buffer_t * out_buffer, - MappedWeights * out_map, - const char * error_tag); - -transcribe_status stream_tensor_data(const std::string & path, - const gguf_context * gguf_data, - ggml_context * ctx_meta, - const char * error_tag); +// allocate_weights + stream_tensor_data: the weight load every GGUF family +// does after init_backends. Same contract as allocate_weights, plus +// TRANSCRIBE_ERR_GGUF from the stream. +transcribe_status load_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out); // A single F16 → F32 promotion target: `dst_slot` is a pointer into a // family weights struct (e.g. &block.conv_pw1_w) that currently holds diff --git a/src/transcribe-mmap.cpp b/src/transcribe-mmap.cpp new file mode 100644 index 000000000..c13fa9baa --- /dev/null +++ b/src/transcribe-mmap.cpp @@ -0,0 +1,248 @@ +// transcribe-mmap.cpp - read-only memory map of a model file. +// +// See transcribe-mmap.h. The mapping follows llama.cpp's llama_mmap +// (MAP_SHARED + PROT_READ; CreateFileMapping + MapViewOfFile). Once mapped, +// the whole file is read in the background: F_RDADVISE on Apple, +// POSIX_FADV_SEQUENTIAL + POSIX_FADV_WILLNEED on Linux/Android, +// PrefetchVirtualMemory on Windows. On a cold page cache that keeps the first +// run as fast as the copying load's sequential read (MADV_WILLNEED was +// measurably slower on macOS). Never MADV_RANDOM: every family reads +// essentially all of its weights each run, so readahead is what keeps a cold +// start fast. No MAP_POPULATE: it blocks the load on the read and fills every +// page-table entry up front, which costs time even on a warm cache. + +#include "transcribe-mmap.h" + +#include "transcribe-path.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# define TRANSCRIBE_MMAP_WIN32 1 +#elif defined(__unix__) || defined(__APPLE__) +# include +# include +# include +# include +# define TRANSCRIBE_MMAP_POSIX 1 +#endif + +namespace transcribe { + +namespace { + +void set_why(std::string * why, const char * what, const char * detail) { + if (why != nullptr) { + *why = what; + if (detail != nullptr && detail[0] != '\0') { + *why += ": "; + *why += detail; + } + } +} + +#if defined(TRANSCRIBE_MMAP_WIN32) +void set_why_win32(std::string * why, const char * what) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "error %lu", static_cast(GetLastError())); + set_why(why, what, buf); +} + +// PrefetchVirtualMemory is Windows 8+; resolve it at runtime so the library +// still loads on anything older (where prefetching is a no-op). +struct PrefetchRange { + void * VirtualAddress; + SIZE_T NumberOfBytes; +}; + +typedef BOOL(WINAPI * PrefetchFn)(HANDLE, ULONG_PTR, PrefetchRange *, ULONG); + +PrefetchFn resolve_prefetch() { + HMODULE k32 = GetModuleHandleW(L"kernel32.dll"); + if (k32 == nullptr) { + return nullptr; + } + return reinterpret_cast(reinterpret_cast(GetProcAddress(k32, "PrefetchVirtualMemory"))); +} + +// Start reading [addr, addr + size) in the background. +void prefetch_view(void * addr, size_t size) noexcept { + static const PrefetchFn prefetch_fn = resolve_prefetch(); + if (prefetch_fn == nullptr) { + return; + } + PrefetchRange range{ addr, size }; + (void) prefetch_fn(GetCurrentProcess(), 1, &range, 0); +} +#endif + +} // namespace + +bool MappedFile::supported() noexcept { +#if defined(TRANSCRIBE_MMAP_WIN32) || defined(TRANSCRIBE_MMAP_POSIX) + return true; +#else + return false; +#endif +} + +#if defined(TRANSCRIBE_MMAP_POSIX) + +bool MappedFile::open(const std::string & path, std::string * why) { + reset(); + + const int fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) { + set_why(why, "open failed", std::strerror(errno)); + return false; + } + + struct stat sb{}; + + if (::fstat(fd, &sb) != 0) { + set_why(why, "fstat failed", std::strerror(errno)); + ::close(fd); + return false; + } + if (!S_ISREG(sb.st_mode) || sb.st_size <= 0) { + set_why(why, "not a non-empty regular file", nullptr); + ::close(fd); + return false; + } + if (static_cast(sb.st_size) > static_cast(SIZE_MAX)) { + set_why(why, "file larger than the address space", nullptr); + ::close(fd); + return false; + } + const size_t size = static_cast(sb.st_size); + + void * addr = ::mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + set_why(why, "mmap failed", std::strerror(errno)); + ::close(fd); + return false; + } + + // Start reading the file in the background. Hints: failure is harmless. +# if defined(__APPLE__) + // radvisory::ra_count is an int, so advise in chunks below 2 GiB. + const off_t chunk = off_t(1) << 30; + for (off_t off = 0; off < static_cast(size); off += chunk) { + struct radvisory ra{}; + ra.ra_offset = off; + ra.ra_count = static_cast(std::min(chunk, static_cast(size) - off)); + (void) ::fcntl(fd, F_RDADVISE, &ra); + } +# elif defined(__linux__) + (void) ::posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void) ::posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); +# else + (void) ::madvise(addr, size, MADV_WILLNEED); +# endif + ::close(fd); // the mapping holds its own reference to the file + + addr_ = addr; + size_ = size; + return true; +} + +void MappedFile::reset() noexcept { + if (addr_ != nullptr) { + (void) ::munmap(addr_, size_); + } + addr_ = nullptr; + size_ = 0; +} + +#elif defined(TRANSCRIBE_MMAP_WIN32) + +bool MappedFile::open(const std::string & path, std::string * why) { + reset(); + + // FILE_SHARE_DELETE lets a model manager rename the file away while it is + // loaded; no FILE_SHARE_WRITE, so nobody can rewrite it underneath us. + const std::filesystem::path p = path_from_utf8(path); + HANDLE file = CreateFileW(p.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) { + set_why_win32(why, "CreateFileW failed"); + return false; + } + + LARGE_INTEGER file_size{}; + if (!GetFileSizeEx(file, &file_size)) { + set_why_win32(why, "GetFileSizeEx failed"); + CloseHandle(file); + return false; + } + if (file_size.QuadPart <= 0) { + set_why(why, "empty file", nullptr); + CloseHandle(file); + return false; + } + if (static_cast(file_size.QuadPart) > static_cast(SIZE_MAX)) { + set_why(why, "file larger than the address space", nullptr); + CloseHandle(file); + return false; + } + + HANDLE mapping = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (mapping == nullptr) { + set_why_win32(why, "CreateFileMappingW failed"); + CloseHandle(file); + return false; + } + // The section keeps the file referenced and the view keeps the section + // alive, so neither handle is needed once the view exists. + void * addr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + if (addr == nullptr) { + set_why_win32(why, "MapViewOfFile failed"); + } + CloseHandle(mapping); + CloseHandle(file); + if (addr == nullptr) { + return false; + } + + addr_ = addr; + size_ = static_cast(file_size.QuadPart); + prefetch_view(addr_, size_); + return true; +} + +void MappedFile::reset() noexcept { + if (addr_ != nullptr) { + (void) UnmapViewOfFile(addr_); + } + addr_ = nullptr; + size_ = 0; +} + +#else + +bool MappedFile::open(const std::string & path, std::string * why) { + (void) path; + reset(); + set_why(why, "memory mapping is not supported on this platform", nullptr); + return false; +} + +void MappedFile::reset() noexcept { + addr_ = nullptr; + size_ = 0; +} + +#endif + +} // namespace transcribe diff --git a/src/transcribe-mmap.h b/src/transcribe-mmap.h new file mode 100644 index 000000000..46dd96244 --- /dev/null +++ b/src/transcribe-mmap.h @@ -0,0 +1,67 @@ +// transcribe-mmap.h - read-only memory map of a model file. +// +// INTERNAL, C++17. Not part of the public ABI. +// +// MappedFile maps a whole file read-only so model weights can be served +// straight from the page cache instead of being copied into anonymous memory. +// File-backed pages are clean and reclaimable: they do not count against the +// Android low-memory killer's anonymous budget or the iOS/macOS footprint, and +// several models loaded from one file share them. +// +// POSIX: open + mmap(PROT_READ, MAP_SHARED). +// Windows: CreateFileW + CreateFileMappingW(PAGE_READONLY) + +// MapViewOfFile(FILE_MAP_READ). +// +// The mapping is read-only: a write through it crashes. And it reflects the +// file: truncating or rewriting the file in place while it is mapped turns a +// later access into SIGBUS (POSIX) or EXCEPTION_IN_PAGE_ERROR (Windows). +// Replace a model file by writing a new one and renaming it over the old one. +// On Windows the mapped file cannot be opened for writing or truncated until +// the mapping is released. + +#pragma once + +#include +#include + +namespace transcribe { + +class MappedFile { + public: + MappedFile() = default; + + ~MappedFile() { reset(); } + + MappedFile(const MappedFile &) = delete; + MappedFile & operator=(const MappedFile &) = delete; + MappedFile(MappedFile &&) = delete; + MappedFile & operator=(MappedFile &&) = delete; + + // True when this build can map files at all. + static bool supported() noexcept; + + // Map the file at `path` (UTF-8) read-only and start reading it into the + // page cache in the background (the weights are read either way: mapped + // tensors fault in from it, copied ones are streamed from it). Replaces + // any previous mapping. On failure returns false, leaves the object empty + // and, when `why` is non-null, stores a short reason for the log. + bool open(const std::string & path, std::string * why); + + // Unmap. Every tensor pointing into the mapping must be dead by now. + void reset() noexcept; + + bool valid() const noexcept { return addr_ != nullptr; } + + // Page-aligned base of the mapping (MapViewOfFile: allocation-granularity + // aligned). Non-const because ggml's host-pointer buffers take void *; + // the pages are still read-only. + void * addr() const noexcept { return addr_; } + + size_t size() const noexcept { return size_; } + + private: + void * addr_ = nullptr; + size_t size_ = 0; +}; + +} // namespace transcribe diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 0525619c3..88cda4cd1 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -1024,6 +1024,21 @@ static transcribe_status transcribe_model_load_file_impl(const char * if (const auto st = check_input_struct_size(params->struct_size, k_min_model_params_size); st != TRANSCRIBE_OK) { return st; } + // Full-size view for the family loaders: defaults first, then only the + // prefix the caller's struct_size covers, so every trailing field is + // readable (no_mmap reads false for a caller built before it existed). + // The bool is re-derived from its raw byte: a C caller can store any + // value there, and loading a non-0/1 byte as bool is UB. + struct transcribe_model_load_params params_view; + transcribe_model_load_params_init(¶ms_view); + std::memcpy(¶ms_view, params, + static_cast(std::min(params->struct_size, sizeof(params_view)))); + { + unsigned char no_mmap_raw = 0; + std::memcpy(&no_mmap_raw, ¶ms_view.no_mmap, sizeof(no_mmap_raw)); + params_view.no_mmap = no_mmap_raw != 0; + } + params = ¶ms_view; // device is validated where the registry is available, in // load_common::init_backends. NULL means automatic selection; a non-NULL diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8ce587f21..b29e6cd29 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -545,7 +545,8 @@ set(TRANSCRIBE_FIXTURE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_ecapa_tdnn_q8_0_as_f16.gguf ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_ecapa_tdnn_bad_hop0.gguf ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_ecapa_tdnn_bad_win_gt_fft.gguf - ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_silero_vad.gguf) + ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_silero_vad.gguf + ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_silero_vad_misaligned.gguf) find_program(TRANSCRIBE_UV uv) @@ -819,6 +820,22 @@ if(TRANSCRIBE_UV) add_test(NAME transcribe_silero_vad_smoke COMMAND transcribe_silero_vad_smoke) endif() +# ----------------------------------------------------------------------------- +# Memory-mapped weights (no_mmap / TRANSCRIBE_NO_MMAP) across the fixture +# families: mapping decision, mapped-vs-copied equivalence, misaligned and +# truncated files, two models on one file, Metal when present. +# ----------------------------------------------------------------------------- + +if(TRANSCRIBE_UV) + add_executable(transcribe_weights_mmap_smoke weights_mmap_smoke.cpp) + target_link_libraries(transcribe_weights_mmap_smoke PRIVATE transcribe) + target_compile_definitions(transcribe_weights_mmap_smoke PRIVATE + "TRANSCRIBE_TEST_FIXTURES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/fixtures\"") + transcribe_apply_warnings(transcribe_weights_mmap_smoke) + add_dependencies(transcribe_weights_mmap_smoke fixtures) + add_test(NAME transcribe_weights_mmap_smoke COMMAND transcribe_weights_mmap_smoke) +endif() + # ----------------------------------------------------------------------------- # Real-model gated test (developer-local, never CI) # ----------------------------------------------------------------------------- diff --git a/tests/api_smoke.c b/tests/api_smoke.c index ed06e4403..d7419ecf6 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -237,6 +238,7 @@ static void test_init_macros(void) { CHECK(mp_macro.struct_size == sizeof(struct transcribe_model_load_params)); CHECK(mp_macro.backend == TRANSCRIBE_BACKEND_AUTO); CHECK(mp_macro.device == NULL); + CHECK(mp_macro.no_mmap == false); struct transcribe_session_params cp_macro; transcribe_session_params_init(&cp_macro); @@ -454,6 +456,16 @@ static void test_load_invalid(void) { TRANSCRIBE_ERR_FILE_NOT_FOUND); CHECK(m == NULL); + /* A caller built before no_mmap existed passes a shorter struct_size; + * it is accepted and the missing field reads as its default. */ + struct transcribe_model_load_params mp_old; + transcribe_model_load_params_init(&mp_old); + mp_old.struct_size = offsetof(struct transcribe_model_load_params, no_mmap); + m = (struct transcribe_model *) 0xdeadbeef; + CHECK(transcribe_model_load_file("/__transcribe_smoke_does_not_exist__.gguf", &mp_old, &m) == + TRANSCRIBE_ERR_FILE_NOT_FOUND); + CHECK(m == NULL); + /* Otherwise-valid call against a path that does not exist on disk * -> FILE_NOT_FOUND. */ m = (struct transcribe_model *) 0xdeadbeef; diff --git a/tests/fixtures/make_gguf_fixtures.py b/tests/fixtures/make_gguf_fixtures.py index f74ff1188..efb69d105 100644 --- a/tests/fixtures/make_gguf_fixtures.py +++ b/tests/fixtures/make_gguf_fixtures.py @@ -323,6 +323,8 @@ def _build_full_gguf( magic: bytes, kv_blobs: list[bytes], tensors: list[Tensor], + alignment: int = GGUF_DEFAULT_ALIGNMENT, + layout: dict | None = None, ) -> bytes: """Assemble a complete GGUF file with KV pairs AND tensor data. @@ -337,7 +339,10 @@ def _build_full_gguf( Each tensor's data offset within the data blob is aligned to GGUF_DEFAULT_ALIGNMENT. ggml_get_alignment defaults to 32 when general.alignment is absent (we don't write it), so emitting at the - same alignment matches what the loader expects. + same alignment matches what the loader expects. A caller that passes a + different `alignment` must also put a matching general.alignment KV in + kv_blobs. `layout`, when given, receives the file offset of the data + section under "data_offset". """ body = bytearray() body += magic @@ -355,18 +360,20 @@ def _build_full_gguf( offsets: list[int] = [] cursor = 0 for t in tensors: - cursor = _ggml_pad(cursor) + cursor = _ggml_pad(cursor, alignment) offsets.append(cursor) cursor += len(t.data) - data_blob_size = _ggml_pad(cursor) + data_blob_size = _ggml_pad(cursor, alignment) for t, off in zip(tensors, offsets): body += _pack_tensor_info(t, off) # Pad from end of tensor info section to start of data blob. body_len = len(body) - data_blob_start = _ggml_pad(body_len) + data_blob_start = _ggml_pad(body_len, alignment) body += b"\x00" * (data_blob_start - body_len) + if layout is not None: + layout["data_offset"] = data_blob_start # Emit each tensor's bytes at its declared offset within the data # blob. The data_blob_start above already aligned us to a 32-byte @@ -1557,7 +1564,11 @@ def _ecapa_tdnn_gguf(codes: list[str], names: list[str], aliases: list[str], q8_ SILERO_HIDDEN = 128 -def _silero_vad_gguf() -> bytes: +def _silero_vad_gguf( + alignment: int = GGUF_DEFAULT_ALIGNMENT, + extra_kvs: list[bytes] | None = None, + layout: dict | None = None, +) -> bytes: rng = _random.Random(1) tensors: list[Tensor] = [] @@ -1604,11 +1615,39 @@ def uniform(name: str, ne: list[int], fan_in: int) -> None: _pack_kv_array_int32("stt.silero_vad.encoder_channels", SILERO_CHANNELS), _pack_kv_array_int32("stt.silero_vad.encoder_strides", SILERO_STRIDES), _pack_kv_uint32("stt.silero_vad.lstm_hidden", SILERO_HIDDEN), - ], + ] + + (extra_kvs or []), tensors, + alignment, + layout, ) +def _silero_vad_misaligned_gguf() -> bytes: + """The toy silero_vad model as a valid GGUF that cannot be memory-mapped. + + general.alignment = 16 and a padding KV sized so the data section starts + 16 bytes past a 32-byte boundary: every tensor lands misaligned for + ggml's CPU buffers (32-byte TENSOR_ALIGNMENT, which + ggml_backend_cpu_buffer_from_ptr asserts). The loader must fall back to + copying the weights and produce the same probabilities as + arch_silero_vad.gguf. + """ + for pad in range(64): + layout: dict = {} + data = _silero_vad_gguf( + alignment=16, + extra_kvs=[ + _pack_kv_uint32("general.alignment", 16), + _pack_kv_string("test.pad", "x" * pad), + ], + layout=layout, + ) + if layout["data_offset"] % 32 == 16: + return data + raise RuntimeError("no padding gives a 16-mod-32 data offset") + + def _write(path: Path, data: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) @@ -2042,6 +2081,8 @@ def emit_fixtures(out_dir: Path) -> None: # silero_vad (VAD role): the real geometry with seeded weights. _write(out_dir / "arch_silero_vad.gguf", _silero_vad_gguf()) + # The same model with its tensors misaligned for memory mapping. + _write(out_dir / "arch_silero_vad_misaligned.gguf", _silero_vad_misaligned_gguf()) def main(argv: list[str]) -> int: diff --git a/tests/weights_mmap_smoke.cpp b/tests/weights_mmap_smoke.cpp new file mode 100644 index 000000000..b7dbfa189 --- /dev/null +++ b/tests/weights_mmap_smoke.cpp @@ -0,0 +1,464 @@ +// weights_mmap_smoke.cpp - memory-mapped model weights +// (transcribe_model_load_params::no_mmap, TRANSCRIBE_NO_MMAP). +// +// Fixture-driven, runs in CI. Covers: +// - every fixture family that loads on the CPU maps its weights by default +// and copies them with no_mmap or TRANSCRIBE_NO_MMAP (read off the load +// log). Mapped weights are read-only, so a family that writes into its +// GGUF weights at load (as a fusion done in place would) crashes here; +// - mapped and copied models give bit-identical results (silero VAD +// probabilities; ecapa langid candidates, including the Q8_0 fixture +// whose widened tensors are copied while the rest are mapped); +// - a caller built before no_mmap existed (shorter struct_size) gets the +// default; +// - a valid GGUF whose tensors are misaligned for the device loads by +// copying, with the same results, instead of aborting; +// - a truncated GGUF fails to load cleanly; +// - two models mapped from one file coexist and free in either order; +// - a model under a non-ASCII path maps (Windows wide-path conversion); +// - with Metal available, the same holds for Metal-mapped weights. + +#include "transcribe.h" +#include "transcribe/langid.h" +#include "transcribe/vad.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +std::string fixture(const char * name) { + return std::string(TRANSCRIBE_TEST_FIXTURES_DIR) + "/" + name; +} + +// ---- log capture ------------------------------------------------------------ + +std::mutex g_log_mu; +std::vector g_log; + +void capture_log(transcribe_log_level level, const char * msg, void * /*userdata*/) { + if (msg == nullptr || level == TRANSCRIBE_LOG_LEVEL_DEBUG || level == TRANSCRIBE_LOG_LEVEL_CONT) { + return; + } + std::lock_guard lock(g_log_mu); + g_log.emplace_back(msg); +} + +void clear_log() { + std::lock_guard lock(g_log_mu); + g_log.clear(); +} + +bool log_has(const char * needle) { + std::lock_guard lock(g_log_mu); + for (const auto & line : g_log) { + if (line.find(needle) != std::string::npos) { + return true; + } + } + return false; +} + +void set_env(const char * name, const char * value) { +#if defined(_WIN32) + _putenv_s(name, value != nullptr ? value : ""); +#else + if (value != nullptr) { + setenv(name, value, 1); + } else { + unsetenv(name); + } +#endif +} + +// ---- loading ---------------------------------------------------------------- + +enum class Mode { + Default, // params defaults: map where possible + NoMmapParam, // no_mmap = true + NoMmapEnv, // defaults + TRANSCRIBE_NO_MMAP=1 + OldStructSize, // struct_size of a caller built before no_mmap existed +}; + +const char * mode_name(Mode mode) { + switch (mode) { + case Mode::Default: + return "default"; + case Mode::NoMmapParam: + return "no_mmap"; + case Mode::NoMmapEnv: + return "TRANSCRIBE_NO_MMAP"; + case Mode::OldStructSize: + return "old struct_size"; + } + return "?"; +} + +bool expect_mapped(Mode mode) { + return mode == Mode::Default || mode == Mode::OldStructSize; +} + +// Load `path` and clear+fill the log with what the load said. +transcribe_model * load(const std::string & path, + Mode mode, + transcribe_backend_request backend = TRANSCRIBE_BACKEND_CPU, + transcribe_status * st_out = nullptr) { + transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = backend; + if (mode == Mode::NoMmapParam) { + mp.no_mmap = true; + } + if (mode == Mode::OldStructSize) { + // A garbage byte where the field would be must not be read. + mp.no_mmap = true; + mp.struct_size = offsetof(transcribe_model_load_params, no_mmap); + } + if (mode == Mode::NoMmapEnv) { + set_env("TRANSCRIBE_NO_MMAP", "1"); + } + clear_log(); + transcribe_model * m = nullptr; + const transcribe_status st = transcribe_model_load_file(path.c_str(), &mp, &m); + if (mode == Mode::NoMmapEnv) { + set_env("TRANSCRIBE_NO_MMAP", nullptr); + } + if (st_out != nullptr) { + *st_out = st; + } + return m; +} + +// Load and check the mapping decision the log reports. +transcribe_model * load_checked(const std::string & path, + Mode mode, + const char * tag, + transcribe_backend_request backend = TRANSCRIBE_BACKEND_CPU) { + transcribe_status st = TRANSCRIBE_OK; + transcribe_model * m = load(path, mode, backend, &st); + if (st != TRANSCRIBE_OK || m == nullptr) { + std::fprintf(stderr, "FAIL load %s (%s): %s\n", path.c_str(), mode_name(mode), transcribe_status_string(st)); + ++g_failures; + return nullptr; + } + const std::string mapped = std::string(tag) + ": weights memory-mapped"; + const std::string copied = std::string(tag) + ": weights copied"; + if (expect_mapped(mode)) { + if (!log_has(mapped.c_str())) { + std::fprintf(stderr, "FAIL %s (%s): expected \"%s\" in the load log\n", path.c_str(), mode_name(mode), + mapped.c_str()); + ++g_failures; + } + } else { + if (!log_has(copied.c_str()) || !log_has("(memory mapping disabled)")) { + std::fprintf(stderr, "FAIL %s (%s): expected \"%s ... (memory mapping disabled)\" in the load log\n", + path.c_str(), mode_name(mode), copied.c_str()); + ++g_failures; + } + } + return m; +} + +// ---- workloads ---------------------------------------------------------------- + +// Deterministic test signal: bursts of a chirp over low noise. +std::vector signal(int n) { + std::vector v(static_cast(n)); + uint32_t seed = 12345; + for (int i = 0; i < n; ++i) { + seed = seed * 1664525u + 1013904223u; + const float noise = (static_cast(seed >> 8) / 16777216.0f - 0.5f) * 0.01f; + const float t = static_cast(i) / 16000.0f; + const bool burst = (i / 8000) % 3 != 0; + v[static_cast(i)] = noise + (burst ? 0.3f * std::sin(6.2831853f * (200.0f + 400.0f * t) * t) : 0.0f); + } + return v; +} + +std::vector vad_probs(transcribe_model * m) { + std::vector out; + if (m == nullptr) { + return out; + } + transcribe_vad_session_params sp; + transcribe_vad_session_params_init(&sp); + transcribe_vad_session * s = nullptr; + CHECK(transcribe_vad_session_init(m, &sp, &s) == TRANSCRIBE_OK); + if (s == nullptr) { + return out; + } + const std::vector pcm = signal(3 * 16000); + CHECK(transcribe_vad_run(s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + transcribe_vad_result r; + transcribe_vad_result_init(&r); + CHECK(transcribe_vad_get_result(s, &r) == TRANSCRIBE_OK); + const float * p = transcribe_vad_probs(s); + if (p != nullptr && r.n_probs > 0) { + out.assign(p, p + r.n_probs); + } + transcribe_vad_session_free(s); + CHECK(!out.empty()); + return out; +} + +struct LangidOut { + std::vector index; + std::vector p; + std::vector logit; +}; + +LangidOut langid(transcribe_model * m) { + LangidOut out; + if (m == nullptr) { + return out; + } + transcribe_langid_session_params sp; + transcribe_langid_session_params_init(&sp); + sp.n_threads = 2; + transcribe_langid_session * s = nullptr; + CHECK(transcribe_langid_session_init(m, &sp, &s) == TRANSCRIBE_OK); + if (s == nullptr) { + return out; + } + const std::vector pcm = signal(2 * 16000); + CHECK(transcribe_langid_run(s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + transcribe_langid_result r; + transcribe_langid_result_init(&r); + CHECK(transcribe_langid_get_result(s, &r) == TRANSCRIBE_OK); + for (int i = 0; i < r.n_candidates; ++i) { + transcribe_langid_candidate c; + transcribe_langid_candidate_init(&c); + CHECK(transcribe_langid_get_candidate(s, i, &c) == TRANSCRIBE_OK); + out.index.push_back(c.index); + out.p.push_back(c.p); + out.logit.push_back(c.logit); + } + transcribe_langid_session_free(s); + CHECK(!out.index.empty()); + return out; +} + +bool same_floats(const std::vector & a, const std::vector & b) { + return a.size() == b.size() && (a.empty() || std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0); +} + +bool same_langid(const LangidOut & a, const LangidOut & b) { + return a.index == b.index && same_floats(a.p, b.p) && same_floats(a.logit, b.logit); +} + +// ---- tests -------------------------------------------------------------------- + +const Mode k_modes[] = { Mode::Default, Mode::NoMmapParam, Mode::NoMmapEnv, Mode::OldStructSize }; + +// Load-only families: mapping must not disturb load (and nothing may write +// into a mapped weight while loading). +void test_load_only_families() { + struct Fixture { + const char * file; + const char * tag; + }; + + const Fixture fixtures[] = { + { "arch_cohere_minimal.gguf", "cohere" }, + { "arch_qwen3_asr_minimal.gguf", "qwen3_asr" }, + { "arch_granite5_ctc_minimal.gguf", "granite5_ctc" }, + }; + for (const Fixture & f : fixtures) { + for (Mode mode : k_modes) { + transcribe_model * m = load_checked(fixture(f.file), mode, f.tag); + transcribe_model_free(m); + } + } +} + +void test_silero_equivalence() { + std::vector baseline; + for (Mode mode : k_modes) { + transcribe_model * m = load_checked(fixture("arch_silero_vad.gguf"), mode, "silero_vad"); + std::vector probs = vad_probs(m); + if (baseline.empty()) { + baseline = probs; + } else if (!same_floats(probs, baseline)) { + std::fprintf(stderr, "FAIL silero_vad (%s): probabilities differ from the mapped run\n", mode_name(mode)); + ++g_failures; + } + transcribe_model_free(m); + } +} + +void test_ecapa_equivalence() { + // minimal: every tensor mapped. q8_0: the Q8_0 tensors are widened to + // F16 at load, so they are copied while the rest stay mapped. + for (const char * file : { "arch_ecapa_tdnn_minimal.gguf", "arch_ecapa_tdnn_q8_0.gguf" }) { + LangidOut baseline; + bool have = false; + for (Mode mode : k_modes) { + transcribe_model * m = load_checked(fixture(file), mode, "ecapa_tdnn"); + if (expect_mapped(mode) && std::strstr(file, "q8_0") != nullptr) { + CHECK(log_has("copied (converted at load)")); + } + const LangidOut out = langid(m); + if (!have) { + baseline = out; + have = true; + } else if (!same_langid(out, baseline)) { + std::fprintf(stderr, "FAIL %s (%s): langid differs from the mapped run\n", file, mode_name(mode)); + ++g_failures; + } + transcribe_model_free(m); + } + } +} + +// Valid GGUF, tensors misaligned for the device: copied, same results, no +// abort (ggml_backend_cpu_buffer_from_ptr asserts on an unaligned base). +void test_misaligned_file() { + transcribe_model * ref = load(fixture("arch_silero_vad.gguf"), Mode::NoMmapParam); + const auto want = vad_probs(ref); + transcribe_model_free(ref); + + transcribe_status st = TRANSCRIBE_OK; + transcribe_model * m = load(fixture("arch_silero_vad_misaligned.gguf"), Mode::Default, TRANSCRIBE_BACKEND_CPU, &st); + CHECK(st == TRANSCRIBE_OK && m != nullptr); + CHECK(log_has("silero_vad: weights copied")); + CHECK(log_has("misaligned")); + CHECK(same_floats(vad_probs(m), want)); + transcribe_model_free(m); +} + +std::string temp_path(const char * stem) { + std::error_code ec; + auto dir = std::filesystem::temp_directory_path(ec); + if (ec) { + dir = std::filesystem::path("."); + } + return (dir / (std::string("transcribe_weights_mmap_") + stem + "_" + std::to_string(std::rand()) + ".gguf")) + .string(); +} + +// A file cut off inside the data section fails to load, in both modes. +void test_truncated_file() { + std::ifstream in(fixture("arch_silero_vad.gguf"), std::ios::binary); + std::vector bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + CHECK(bytes.size() > 4096); + const std::string path = temp_path("truncated"); + { + std::ofstream out(path, std::ios::binary); + out.write(bytes.data(), static_cast(bytes.size() / 2)); + } + for (Mode mode : { Mode::Default, Mode::NoMmapParam }) { + transcribe_status st = TRANSCRIBE_OK; + transcribe_model * m = load(path, mode, TRANSCRIBE_BACKEND_CPU, &st); + CHECK(st != TRANSCRIBE_OK); + CHECK(m == nullptr); + transcribe_model_free(m); + } + std::error_code ec; + std::filesystem::remove(path, ec); +} + +// Two models mapped from one file are independent; free order is free. +void test_two_models_one_file() { + const std::string path = fixture("arch_silero_vad.gguf"); + transcribe_model * a = load_checked(path, Mode::Default, "silero_vad"); + transcribe_model * b = load_checked(path, Mode::Default, "silero_vad"); + const auto pa = vad_probs(a); + const auto pb = vad_probs(b); + CHECK(same_floats(pa, pb)); + transcribe_model_free(a); + CHECK(same_floats(vad_probs(b), pb)); + transcribe_model_free(b); +} + +// A non-ASCII (UTF-8) path maps too: on Windows the mapping opens the file +// through the UTF-8 -> wide conversion, and a failure there would silently +// fall back to copying. The directory name is spelled in escaped UTF-8 bytes +// so the source stays ASCII. +void test_utf8_path() { + namespace fs = std::filesystem; + const char * k_name = "mmap-caf\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac"; + std::error_code ec; + fs::path dir = fs::temp_directory_path(ec); + if (ec) { + dir = fs::path("."); + } + dir /= fs::u8path(k_name); + fs::remove_all(dir, ec); + fs::create_directories(dir, ec); + CHECK(!ec); + const fs::path dst = dir / "arch_silero_vad.gguf"; + fs::copy_file(fs::u8path(fixture("arch_silero_vad.gguf")), dst, fs::copy_options::overwrite_existing, ec); + CHECK(!ec); + + transcribe_model * m = load_checked(dst.u8string(), Mode::Default, "silero_vad"); + CHECK(!vad_probs(m).empty()); + transcribe_model_free(m); + fs::remove_all(dir, ec); +} + +// Metal-mapped weights (Apple silicon): same decision log, same results as +// copied Metal weights. +void test_metal() { + if (!transcribe_backend_available(TRANSCRIBE_BACKEND_METAL)) { + std::fprintf(stderr, "weights_mmap_smoke: no Metal device; skipping the Metal case\n"); + return; + } + LangidOut baseline; + bool have = false; + for (Mode mode : { Mode::Default, Mode::NoMmapParam }) { + transcribe_model * m = + load_checked(fixture("arch_ecapa_tdnn_minimal.gguf"), mode, "ecapa_tdnn", TRANSCRIBE_BACKEND_METAL); + const LangidOut out = langid(m); + if (!have) { + baseline = out; + have = true; + } else { + CHECK(same_langid(out, baseline)); + } + transcribe_model_free(m); + } +} + +} // namespace + +int main() { + set_env("TRANSCRIBE_NO_MMAP", nullptr); + transcribe_log_set(capture_log, nullptr); + + test_load_only_families(); + test_silero_equivalence(); + test_ecapa_equivalence(); + test_misaligned_file(); + test_truncated_file(); + test_two_models_one_file(); + test_utf8_path(); + test_metal(); + + transcribe_log_set(nullptr, nullptr); + if (g_failures != 0) { + std::fprintf(stderr, "weights_mmap_smoke: %d failure(s)\n", g_failures); + return 1; + } + std::printf("weights_mmap_smoke: ok\n"); + return 0; +} diff --git a/tools/transcribe-bench/main.cpp b/tools/transcribe-bench/main.cpp index d4194c4b7..e9af4298c 100644 --- a/tools/transcribe-bench/main.cpp +++ b/tools/transcribe-bench/main.cpp @@ -47,7 +47,8 @@ struct bench_args { int n_threads = 0; bool quiet = false; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int device_index = -1; // --device N: -1 = auto, >=0 = exact device + int device_index = -1; // --device N: -1 = auto, >=0 = exact device + bool no_mmap = false; // --no-mmap: copy weights instead of mapping the file // Passed through to transcribe_run_params::spec_k_drafts. -1 = family // default, 0 = spec decode off, > 0 = explicit draft length. Silently // ignored by families without supports_spec_decode. Set by @@ -73,6 +74,8 @@ void print_usage(const char * argv0) { " (BLAS/AMX) when the build includes them.\n" " --device N exact device index from transcribe-cli --list-devices,\n" " including 0 (default: automatic selection)\n" + " --no-mmap copy the weights into memory instead of memory-mapping\n" + " the model file (TRANSCRIBE_NO_MMAP=1 does the same)\n" " --spec-k-drafts N speculative-decode draft length on the offline\n" " path: -1 = family default, 0 = off, > 0 = K.\n" " Ignored by families without spec support.\n" @@ -200,6 +203,8 @@ bool parse_args(int argc, char ** argv, bench_args & out) { if (!parse_backend_kind(v, out.backend)) { return false; } + } else if (a == "--no-mmap") { + out.no_mmap = true; } else if (a == "--device") { auto v = need_val(i, "--device"); if (!v) { @@ -332,6 +337,7 @@ int main(int argc, char ** argv) { transcribe_model_load_params_init(&mp); mp.backend = args.backend; mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + mp.no_mmap = args.no_mmap; if (args.device_index >= 0 && mp.device == nullptr) { std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); return EXIT_FAILURE; From 2944d179612e62ec81f8e97129cb095790681ce2 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 18 Oct 2026 03:17:51 -0700 Subject: [PATCH 5/7] bindings: no_mmap (regen; abihash f99ad4169374b8f3) Regenerate the Python/TypeScript FFI (+ include/transcribe.abihash) and the Rust sys bindings for transcribe_model_load_params.no_mmap, pin the Swift ABI hash, and expose the option: Python Model(no_mmap=) and transcribe(), Rust ModelOptions::no_mmap, TypeScript ModelOptions.noMmap, Swift ModelOptions(noMmap:). READMEs note the memory-mapping behavior and the TRANSCRIBE_NO_MMAP override. --- bindings/python/README.md | 7 +++++++ .../python/src/transcribe_cpp/__init__.py | 19 +++++++++++++++---- .../python/src/transcribe_cpp/_generated.py | 6 +++--- bindings/rust/sys/src/transcribe_sys.rs | 9 ++++++--- bindings/rust/transcribe-cpp/README.md | 9 +++++++++ bindings/rust/transcribe-cpp/src/model.rs | 9 +++++++++ .../transcribe-cpp/tests/device_select.rs | 2 ++ bindings/swift/README.md | 7 +++++++ .../swift/Sources/TranscribeCpp/ABIHash.swift | 2 +- .../swift/Sources/TranscribeCpp/Model.swift | 1 + .../swift/Sources/TranscribeCpp/Options.swift | 10 +++++++++- bindings/typescript/README.md | 7 +++++++ bindings/typescript/src/_generated.ts | 6 +++--- bindings/typescript/src/index.ts | 1 + bindings/typescript/src/types.ts | 7 +++++++ include/transcribe.abihash | 2 +- 16 files changed, 88 insertions(+), 16 deletions(-) diff --git a/bindings/python/README.md b/bindings/python/README.md index ccd53d887..f41cb8113 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -150,9 +150,16 @@ with transcribe_cpp.Model("model.gguf", device=device) as model: print(model.device) ``` +Weights are memory-mapped from the model file where the device can use host +memory (CPU, Metal on Apple silicon), so they live in reclaimable page cache +rather than process memory. Don't truncate or rewrite a model file in place +while it is loaded; write a new file and rename it over the old one. +`Model(no_mmap=True)` copies the weights instead. + | Variable | Effect | |---|---| | `TRANSCRIBE_BACKEND` | overrides the `"auto"` default; explicit `backend=` still wins | +| `TRANSCRIBE_NO_MMAP` | `1` copies weights instead of memory-mapping them, like `no_mmap=True` | | `TRANSCRIBE_NATIVE_PROVIDER` | forces an installed native provider package, for example `cu12` | | `TRANSCRIBE_LIBRARY` | loads exactly this shared library | diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 4b406a812..3527470ee 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -1057,10 +1057,18 @@ class Model: the escape hatch for machines whose best-ranked device misbehaves (e.g. CI's paravirtualized Metal, a broken GPU driver) without touching code. An explicit ``backend=`` argument always wins over the environment. + + The weights are memory-mapped from the model file where the device can + use host memory (the CPU everywhere, Metal on Apple silicon): they stay + in reclaimable page cache instead of process memory. Do not truncate or + rewrite the file in place while the model is open (write a new file and + rename it over the old one). ``no_mmap=True``, or the + ``TRANSCRIBE_NO_MMAP=1`` environment variable, copies them instead. """ def __init__(self, path: str | os.PathLike, *, - backend: Backend = "auto", device: BackendDevice | None = None): + backend: Backend = "auto", device: BackendDevice | None = None, + no_mmap: bool = False): # Live sessions, tracked weakly: close() must free them before the # model, because transcribe_model_free is only valid once every # derived session is gone (use-after-free otherwise). Created before @@ -1078,6 +1086,7 @@ def __init__(self, path: str | os.PathLike, *, if not isinstance(device, BackendDevice) or device._handle is None: raise TypeError("device must be a BackendDevice returned by backends()") params.device = device._handle + params.no_mmap = bool(no_mmap) handle = ctypes.c_void_p() status = _lib.transcribe_model_load_file( @@ -2260,6 +2269,7 @@ def transcribe( *, backend: Backend = "auto", device: BackendDevice | None = None, + no_mmap: bool = False, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0, @@ -2282,8 +2292,9 @@ def transcribe( *model* may be a path (loaded and freed within this call) or an existing Model (reused and left open). Loading a model is not free, so to transcribe many clips keep a Model and call ``model.session().run(...)`` yourself; this - helper is for the one-shot case. ``backend`` / ``device`` apply only when - *model* is a path — they are ignored when an already-loaded Model is passed. + helper is for the one-shot case. ``backend`` / ``device`` / ``no_mmap`` + apply only when *model* is a path — they are ignored when an already-loaded + Model is passed. ``family`` / ``spec_k_drafts`` and the prompting inputs (``vocabulary``, ``prompt``, ``prefix``) pass through to :meth:`Session.run`. """ @@ -2298,6 +2309,6 @@ def transcribe( with model.session(**session_opts) as session: return session.run(pcm, **run_opts) - with Model(model, backend=backend, device=device) as owned: + with Model(model, backend=backend, device=device, no_mmap=no_mmap) as owned: with owned.session(**session_opts) as session: return session.run(pcm, **run_opts) diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 9c0d7f7b6..844dfa8a4 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "98540564cd329b85" +PUBLIC_HEADER_HASH = "f99ad4169374b8f3" # === enum constants === TRANSCRIBE_OK = 0 @@ -228,7 +228,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_ext._fields_ = [("size", _c.c_uint64), ("kind", _c.c_uint32)] transcribe_backend_init_params._fields_ = [("struct_size", _c.c_uint64), ("artifact_dir", _c.c_char_p), ("allowed_backends", _c.c_uint32)] transcribe_device_info._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] -transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("device", _c.c_void_p)] +transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("device", _c.c_void_p), ("no_mmap", _c.c_bool)] transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)] transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("diarize", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32), ("vocabulary", _c.POINTER(_c.c_char_p)), ("n_vocabulary", _c.c_int32), ("prompt", _c.c_char_p), ("prefix", _c.c_char_p)] transcribe_capabilities._fields_ = [("struct_size", _c.c_uint64), ("native_sample_rate", _c.c_int32), ("n_languages", _c.c_int), ("languages", _c.POINTER(_c.c_char_p)), ("max_timestamp_kind", _c.c_int), ("supports_language_detect", _c.c_bool), ("supports_translate", _c.c_bool), ("supports_streaming", _c.c_bool), ("supports_spec_decode", _c.c_bool), ("max_audio_ms", _c.c_int64), ("n_translate_target_languages", _c.c_int), ("translate_target_languages", _c.POINTER(_c.c_char_p))] @@ -301,7 +301,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_ext': {'size': 16, 'align': 8, 'offsets': {'size': 0, 'kind': 8}}, 'transcribe_backend_init_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'artifact_dir': 8, 'allowed_backends': 16}}, 'transcribe_device_info': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, - 'transcribe_model_load_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'device': 16}}, + 'transcribe_model_load_params': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'device': 16, 'no_mmap': 24}}, 'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}}, 'transcribe_run_params': {'size': 104, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'diarize': 24, 'language': 32, 'target_language': 40, 'keep_special_tags': 48, 'family': 56, 'spec_k_drafts': 64, 'vocabulary': 72, 'n_vocabulary': 80, 'prompt': 88, 'prefix': 96}}, 'transcribe_capabilities': {'size': 56, 'align': 8, 'offsets': {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48}}, diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index 961123cf9..db7d0d9fd 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 98540564cd329b85 +// Pinned to include/transcribe.abihash = f99ad4169374b8f3 /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "98540564cd329b85"; +pub const PUBLIC_HEADER_HASH: &str = "f99ad4169374b8f3"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -348,11 +348,12 @@ pub struct transcribe_model_load_params { pub struct_size: u64, pub backend: transcribe_backend_request, pub device: transcribe_device_t, + pub no_mmap: bool, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of transcribe_model_load_params"] - [::std::mem::size_of::() - 24usize]; + [::std::mem::size_of::() - 32usize]; ["Alignment of transcribe_model_load_params"] [::std::mem::align_of::() - 8usize]; ["Offset of field: transcribe_model_load_params::struct_size"] @@ -361,6 +362,8 @@ const _: () = { [::std::mem::offset_of!(transcribe_model_load_params, backend) - 8usize]; ["Offset of field: transcribe_model_load_params::device"] [::std::mem::offset_of!(transcribe_model_load_params, device) - 16usize]; + ["Offset of field: transcribe_model_load_params::no_mmap"] + [::std::mem::offset_of!(transcribe_model_load_params, no_mmap) - 24usize]; }; unsafe extern "C" { pub fn transcribe_model_load_params_init(params: *mut transcribe_model_load_params); diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index a1de72948..77dfd10af 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -177,6 +177,15 @@ finish `init_backends()` or `init_backends_default()` before any thread enumerates devices, queries backend availability, or loads a model; native registry mutation is a startup-only operation and must not race those calls. +## Memory-mapped weights + +Where the device can use host memory (the CPU, Metal on Apple silicon), model +weights are memory-mapped from the file and served from the page cache, which +the OS can reclaim, instead of being copied into process memory. Don't +truncate or rewrite a model file in place while it is loaded; write a new file +and rename it over the old one. Set `ModelOptions::no_mmap` (or the +`TRANSCRIBE_NO_MMAP=1` environment variable) to copy the weights instead. + ## Packaging a distributable (`shared` / `dynamic-backends`) With the **default static** build there is nothing to do — the native code is diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index fac1b0231..a406afa33 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -41,6 +41,13 @@ pub struct ModelOptions { /// Exact process-local compute device. `None` applies the backend's /// automatic policy; `Some` selects that device or fails without fallback. pub device: Option, + /// Copy the weights into memory instead of memory-mapping the model file. + /// Default `false`: where the device can use host memory (CPU, Metal on + /// Apple silicon) the weights are served from the file's page cache, which + /// the OS can reclaim. Don't truncate or rewrite a mapped model file in + /// place; write a new file and rename it over the old one. The + /// `TRANSCRIBE_NO_MMAP=1` environment variable forces `true`. + pub no_mmap: bool, } impl Default for ModelOptions { @@ -48,6 +55,7 @@ impl Default for ModelOptions { ModelOptions { backend: Backend::Auto, device: None, + no_mmap: false, } } } @@ -154,6 +162,7 @@ impl Model { .device .as_ref() .map_or(std::ptr::null_mut(), |device| device.handle); + params.no_mmap = options.no_mmap; let mut out: *mut sys::transcribe_model = std::ptr::null_mut(); let status = unsafe { sys::transcribe_model_load_file(c_path.as_ptr(), ¶ms, &mut out) }; diff --git a/bindings/rust/transcribe-cpp/tests/device_select.rs b/bindings/rust/transcribe-cpp/tests/device_select.rs index ca4447e40..773230395 100644 --- a/bindings/rust/transcribe-cpp/tests/device_select.rs +++ b/bindings/rust/transcribe-cpp/tests/device_select.rs @@ -21,6 +21,7 @@ fn every_primary_device_can_be_selected_exactly() { &ModelOptions { backend: Backend::Auto, device: Some(device.clone()), + ..Default::default() }, ); match result { @@ -53,6 +54,7 @@ fn explicit_backend_must_match_device() { &ModelOptions { backend: Backend::Cpu, device: Some(gpu), + ..Default::default() }, ) .unwrap_err(); diff --git a/bindings/swift/README.md b/bindings/swift/README.md index b71c31af1..be2459a6c 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -168,6 +168,13 @@ Request a backend policy with `ModelOptions(backend:)`; probe availability with `Transcribe.devices()` to `ModelOptions(device:)`; exact selection never falls back to another primary device. +On the CPU and on Metal, model weights are memory-mapped from the file: they +are served from the page cache, which the OS can reclaim, and do not count +toward the app's memory footprint (the figure jetsam enforces on iOS). Don't +truncate or rewrite a model file in place while it is loaded; write a new file +and rename it over the old one. `ModelOptions(noMmap: true)` copies the weights +instead. + ## Concurrency and lifetime - `Model` is shareable. In 0.x, compute is serialized per model, so concurrent diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index 972c881ed..6995475a9 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "98540564cd329b85" + public static let pinnedHeaderHash = "f99ad4169374b8f3" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index b3f4c0894..6c5340fb2 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -56,6 +56,7 @@ public final class Model: @unchecked Sendable { transcribe_model_load_params_init(¶ms) params.backend = options.backend.cValue params.device = options.device?.handle + params.no_mmap = options.noMmap var out: OpaquePointer? let status = transcribe_model_load_file(path, ¶ms, &out) try TranscribeError.check(status, context: "loading \(path)") diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index 9477942fb..bff4e5998 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -113,9 +113,17 @@ public struct ModelOptions: Sendable { public var backend: Backend /// Exact process-local device. `nil` applies the backend's automatic policy. public var device: Device? - public init(backend: Backend = .auto, device: Device? = nil) { + /// Copy the weights into memory instead of memory-mapping the model file. + /// Default `false`: on the CPU and on Metal the weights are served from the + /// file's page cache, which the OS can reclaim and which does not count + /// toward the app's memory footprint. Don't truncate or rewrite a mapped + /// model file in place; write a new file and rename it over the old one. + /// The `TRANSCRIBE_NO_MMAP=1` environment variable forces `true`. + public var noMmap: Bool + public init(backend: Backend = .auto, device: Device? = nil, noMmap: Bool = false) { self.backend = backend self.device = device + self.noMmap = noMmap } } diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 00ff84855..1e0121669 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -226,6 +226,13 @@ const exact = await TranscribeModel.load("model.gguf", { device: cpu }); loader/driver degrades silently to CPU; an explicit backend that cannot be satisfied raises a `BackendError`. +Where the device can use host memory (CPU, Metal on Apple silicon), weights are +memory-mapped from the model file and served from the page cache, which the OS +can reclaim, instead of being copied into process memory. Don't truncate or +rewrite a model file in place while it is loaded; write a new file and rename +it over the old one. `{ noMmap: true }` (or `TRANSCRIBE_NO_MMAP=1`) copies the +weights instead. + ## Thread-safety Your JavaScript runs on one thread, but `run`/`feed`/`finalize` dispatch the diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index e14a8479f..683d416d1 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "98540564cd329b85"; +export const PUBLIC_HEADER_HASH = "f99ad4169374b8f3"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -154,7 +154,7 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_ext': { size: 16, align: 8, offsets: {'size': 0, 'kind': 8} }, 'transcribe_backend_init_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'artifact_dir': 8, 'allowed_backends': 16} }, 'transcribe_device_info': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, - 'transcribe_model_load_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'device': 16} }, + 'transcribe_model_load_params': { size: 32, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'device': 16, 'no_mmap': 24} }, 'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} }, 'transcribe_run_params': { size: 104, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'diarize': 24, 'language': 32, 'target_language': 40, 'keep_special_tags': 48, 'family': 56, 'spec_k_drafts': 64, 'vocabulary': 72, 'n_vocabulary': 80, 'prompt': 88, 'prefix': 96} }, 'transcribe_capabilities': { size: 56, align: 8, offsets: {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48} }, @@ -227,7 +227,7 @@ export function defineTypes(koffi: any): Record { T['transcribe_ext'] = koffi.struct({ size: 'uint64_t', kind: 'uint32_t' }); T['transcribe_backend_init_params'] = koffi.struct({ struct_size: 'uint64_t', artifact_dir: 'char *', allowed_backends: 'uint32_t' }); T['transcribe_device_info'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); - T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', device: 'void *' }); + T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', device: 'void *', no_mmap: 'bool' }); T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' }); T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', diarize: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t', vocabulary: 'void *', n_vocabulary: 'int32_t', prompt: 'char *', prefix: 'char *' }); T['transcribe_capabilities'] = koffi.struct({ struct_size: 'uint64_t', native_sample_rate: 'int32_t', n_languages: 'int', languages: 'void *', max_timestamp_kind: 'int', supports_language_detect: 'bool', supports_translate: 'bool', supports_streaming: 'bool', supports_spec_decode: 'bool', max_audio_ms: 'int64_t', n_translate_target_languages: 'int', translate_target_languages: 'void *' }); diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index bc4ecc66c..00fe33837 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -1759,6 +1759,7 @@ export class TranscribeModel { } p.device = handle; } + if (opts.noMmap) p.no_mmap = true; const out: any[] = [null]; const st = await callAsync(n.F.modelLoadFile, path, p, out); diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 97f6b1235..83ca0d8d1 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -145,6 +145,13 @@ export interface ModelOptions { /** Exact device returned by {@link getAvailableBackends}. Omit for the * backend's automatic policy. Exact selection never falls back. */ device?: BackendInfo; + /** Copy the weights into memory instead of memory-mapping the model file. + * Default false: where the device can use host memory (CPU, Metal on Apple + * silicon) the weights are served from the file's page cache, which the OS + * can reclaim. Don't truncate or rewrite a mapped model file in place; + * write a new file and rename it over the old one. The + * `TRANSCRIBE_NO_MMAP=1` environment variable forces true. */ + noMmap?: boolean; } export interface SessionOptions { diff --git a/include/transcribe.abihash b/include/transcribe.abihash index 8b621ba4a..152e9d3f4 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -98540564cd329b85 +f99ad4169374b8f3 From 673b20d7d4ac3a5491ae0032e00257747a80f662 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 19 Oct 2026 00:25:31 -0700 Subject: [PATCH 6/7] load: stream copied weights in bounded chunks stream_tensor_data read each tensor whole into a staging vector grown to the largest tensor, then handed it to ggml_backend_tensor_set in one call. On every load that copies weights (GPU backends, no_mmap, tensors converted at load) that held the largest tensor in anonymous memory a second time: 243 MiB for Qwen3-ASR-1.7B's token embedding. On device memory the host cannot map, Vulkan also stages each call whole through a per-device buffer it keeps for the life of the process, sized to the largest write. Read host-buffer (CPU) tensors straight into t->data, with no staging copy, and upload everything else in 16 MiB chunks via ggml_backend_tensor_set's offset. That bounds both staging buffers. Every weight buffer is a default buffer type, all of which accept partial writes. Measured on a Ryzen 7840U (RADV iGPU), interleaved A/B, identical transcripts: Vulkan qwen3-asr-1.7b peak anon 311 -> 86 MiB, RSS 364 -> 138 MiB Vulkan whisper-turbo peak anon 100 -> 42 MiB, load 491 -> 447 ms CPU no_mmap qwen3-asr-1.7b load 1316 -> 1023 ms Mapped CPU loads are unchanged; Metal (M4) transcripts are unchanged. --- src/transcribe-load-common.cpp | 48 +++++++++++++++++++++++++--------- src/transcribe-load-common.h | 7 ++--- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/transcribe-load-common.cpp b/src/transcribe-load-common.cpp index 2edee65f1..149844286 100644 --- a/src/transcribe-load-common.cpp +++ b/src/transcribe-load-common.cpp @@ -724,6 +724,13 @@ transcribe_status stream_tensor_data(const std::string & path, return TRANSCRIBE_ERR_GGUF; } + // Uploads go through a bounded staging buffer: ggml_backend_tensor_set + // stages each call whole, and for device memory the host cannot map, + // Vulkan grows a per-device staging buffer, kept for the life of the + // process, to the largest single write (243 MiB for Qwen3-ASR-1.7B's token + // embedding), on top of ours. + constexpr size_t k_stage_chunk = size_t(16) << 20; + const size_t data_offset = gguf_get_data_offset(gguf_data); std::vector staging; @@ -747,21 +754,36 @@ transcribe_status stream_tensor_data(const std::string & path, return TRANSCRIBE_ERR_GGUF; } - if (staging.size() < nbytes) { - staging.resize(nbytes); - } - fin.read(reinterpret_cast(staging.data()), static_cast(nbytes)); - if (!fin) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: short read for tensor \"%s\" (%zu bytes)", error_tag, t->name, - nbytes); - return TRANSCRIBE_ERR_GGUF; + // Host buffers (the CPU) take the file bytes directly: no staging + // copy. Everything else (Metal, Vulkan, CUDA, ...) is uploaded in + // chunks through ggml_backend_tensor_set, which needs a buffer type + // that accepts partial writes: every default one does (ggml's CPU + // repack buffers would not, and assert). + if (t->buffer != nullptr && ggml_backend_buffer_is_host(t->buffer)) { + fin.read(static_cast(t->data), static_cast(nbytes)); + if (!fin) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: short read for tensor \"%s\" (%zu bytes)", error_tag, t->name, + nbytes); + return TRANSCRIBE_ERR_GGUF; + } + continue; } - // ggml_backend_tensor_set is the right call regardless of - // backend: on host buffers (CPU + Metal unified memory on - // Apple Silicon) it's a memcpy; on discrete GPUs it does - // the upload. - ggml_backend_tensor_set(t, staging.data(), 0, nbytes); + const size_t chunk = std::min(nbytes, k_stage_chunk); + if (staging.size() < chunk) { + staging.resize(chunk); + } + for (size_t done = 0; done < nbytes;) { + const size_t n = std::min(nbytes - done, staging.size()); + fin.read(reinterpret_cast(staging.data()), static_cast(n)); + if (!fin) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: short read for tensor \"%s\" (%zu bytes)", error_tag, t->name, + nbytes); + return TRANSCRIBE_ERR_GGUF; + } + ggml_backend_tensor_set(t, staging.data(), done, n); + done += n; + } } return TRANSCRIBE_OK; diff --git a/src/transcribe-load-common.h b/src/transcribe-load-common.h index 4cb065e04..86002502d 100644 --- a/src/transcribe-load-common.h +++ b/src/transcribe-load-common.h @@ -175,9 +175,10 @@ transcribe_status allocate_copied_weights(ggml_context * ctx_meta, WeightStorage & out); // Stream tensor data from the GGUF data section on disk into -// already-allocated backend buffer slots via ggml_backend_tensor_set. -// Tensors that `skip` maps are left alone (they already hold the file -// bytes); with skip == nullptr every tensor in ctx_meta is streamed. +// already-allocated backend buffer slots: read straight into host buffers, +// uploaded through ggml_backend_tensor_set in bounded (16 MiB) chunks +// otherwise. Tensors that `skip` maps are left alone (they already hold the +// file bytes); with skip == nullptr every tensor in ctx_meta is streamed. // // path: filesystem path to the GGUF file, reopened fresh via // std::ifstream. Stable for the life of the call. From 6c0f80b22d73a92cd056b01a177059d03a247033 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 19 Oct 2026 05:58:53 -0700 Subject: [PATCH 7/7] cohere: rely on the weight binding in the Q-bias fusion; drop MappedFile::valid fuse_encoder_q_bias re-checked what weights.cpp already guarantees: q_bias_fusable skipped a block unless attn_q_b and pos_bias_u/v were F32 with d_model elements, and an early return skipped the whole fusion when d_model was not head_dim * n_heads. GET_F32 binds all three as required F32 tensors of exact shape, and read_cohere_hparams rejects d_model not divisible by n_heads, so neither check could fire; if the binding ever broke they would have quietly skipped the fold instead of surfacing it. Fuse every block unconditionally. MappedFile::valid() had no callers. Cohere 03-2026 and arabic-07-2026 (Q8_0, Q4_K_M) give identical transcripts on CPU (mapped and no_mmap) and Vulkan; all 48 blocks fuse. --- src/arch/cohere/model.cpp | 47 +++++---------------------------------- src/transcribe-mmap.h | 2 -- 2 files changed, 6 insertions(+), 43 deletions(-) diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index c38009f0d..e7f9e0544 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -341,19 +341,6 @@ transcribe_status fuse_batch_norm(CohereModel & m) { return TRANSCRIBE_OK; } -// A block whose Q-bias fusion inputs are all present as F32 [d_model] -// (weights.cpp binds them as F32 already; re-checked because the fusion reads -// raw floats). -bool q_bias_fusable(const CohereBlock & b, int64_t d_model) { - const ggml_tensor * ts[] = { b.attn_q_b, b.attn_pos_u, b.attn_pos_v }; - for (const ggml_tensor * t : ts) { - if (t == nullptr || t->type != GGML_TYPE_F32 || ggml_nelements(t) != d_model) { - return false; - } - } - return true; -} - // Fold each encoder layer's Q bias into its pos_bias_u / pos_bias_v at // load time. In rel_pos_mhsa: // q_u = (W_q x + q_b) + pos_u = W_q x + (q_b + pos_u) @@ -363,34 +350,18 @@ bool q_bias_fusable(const CohereBlock & b, int64_t d_model) { // q_bias_fused_buffer (the GGUF-backed pos_bias tensors may be a read-only // memory map, and weights are never written after load); the block's // pos_u/pos_v slots are repointed at them and attn_q_b is nulled so the -// builder skips the add. +// builder skips the add. weights.cpp binds all three as F32 (q_b [d_model], +// pos_u/v [head_dim, n_heads]) and read_cohere_hparams rejects d_model not +// divisible by n_heads, so every block fuses. transcribe_status fuse_encoder_q_bias(CohereModel & m) { const size_t n_blocks = m.weights.blocks.size(); if (n_blocks == 0) { return TRANSCRIBE_OK; } - const int64_t d_model = m.hparams.enc_d_model; - const int64_t n_heads = m.hparams.enc_n_heads; - const int64_t head_dim = n_heads > 0 ? d_model / n_heads : 0; - - if (head_dim * n_heads != d_model) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "cohere: d_model (%lld) != head_dim*n_heads; " - "skipping encoder Q-bias fusion", - (long long) d_model); - return TRANSCRIBE_OK; - } + const int64_t d_model = m.hparams.enc_d_model; - size_t n_fusable = 0; - for (const auto & b : m.weights.blocks) { - n_fusable += q_bias_fusable(b, d_model) ? 1 : 0; - } - if (n_fusable == 0) { - return TRANSCRIBE_OK; - } - - const size_t ctx_size = n_fusable * 2 * ggml_tensor_overhead() + 256; + const size_t ctx_size = n_blocks * 2 * ggml_tensor_overhead() + 256; ggml_init_params params = { ctx_size, nullptr, true }; m.q_bias_fused_ctx = ggml_init(params); if (m.q_bias_fused_ctx == nullptr) { @@ -402,9 +373,6 @@ transcribe_status fuse_encoder_q_bias(CohereModel & m) { std::vector fused_v(n_blocks, nullptr); for (size_t i = 0; i < n_blocks; ++i) { const auto & b = m.weights.blocks[i]; - if (!q_bias_fusable(b, d_model)) { - continue; - } fused_u[i] = ggml_new_tensor(m.q_bias_fused_ctx, GGML_TYPE_F32, ggml_n_dims(b.attn_pos_u), b.attn_pos_u->ne); fused_v[i] = ggml_new_tensor(m.q_bias_fused_ctx, GGML_TYPE_F32, ggml_n_dims(b.attn_pos_v), b.attn_pos_v->ne); ggml_set_name(fused_u[i], ggml_get_name(b.attn_pos_u)); @@ -424,9 +392,6 @@ transcribe_status fuse_encoder_q_bias(CohereModel & m) { for (size_t i = 0; i < n_blocks; ++i) { auto & b = m.weights.blocks[i]; - if (fused_u[i] == nullptr) { - continue; - } ggml_backend_tensor_get(b.attn_q_b, q_bias.data(), 0, nbytes); ggml_backend_tensor_get(b.attn_pos_u, pos_u.data(), 0, nbytes); @@ -448,7 +413,7 @@ transcribe_status fuse_encoder_q_bias(CohereModel & m) { b.attn_q_b = nullptr; } - log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "cohere: fused Q bias into pos_u/pos_v for %zu encoder blocks", n_fusable); + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "cohere: fused Q bias into pos_u/pos_v for %zu encoder blocks", n_blocks); return TRANSCRIBE_OK; } diff --git a/src/transcribe-mmap.h b/src/transcribe-mmap.h index 46dd96244..7944b2fea 100644 --- a/src/transcribe-mmap.h +++ b/src/transcribe-mmap.h @@ -50,8 +50,6 @@ class MappedFile { // Unmap. Every tensor pointing into the mapping must be dead by now. void reset() noexcept; - bool valid() const noexcept { return addr_ != nullptr; } - // Page-aligned base of the mapping (MapViewOfFile: allocation-granularity // aligned). Non-const because ggml's host-pointer buffers take void *; // the pages are still read-only.