diff --git a/.claude/skills/porting-4-cpp/SKILL.md b/.claude/skills/porting-4-cpp/SKILL.md index 8df400179..175c1c224 100644 --- a/.claude/skills/porting-4-cpp/SKILL.md +++ b/.claude/skills/porting-4-cpp/SKILL.md @@ -133,6 +133,16 @@ Pick the closest existing family by architecture pattern: not the whole file. Load the closest family's `src/arch//` as a working example. +Load weights the way every family does: `gguf_init_from_file` into +`ctx_meta`, bind the weight struct, `load_common::init_backends`, then +`load_common::load_weights` into a `load_common::WeightStorage` member that +the destructor resets before freeing the backends. Weights are memory-mapped +read-only wherever the device allows it, so never write a `ctx_meta` tensor +after load: fusions and promotions create new tensors in their own buffer and +repoint the weight slot (see `fuse_batch_norm`, cohere's +`fuse_encoder_q_bias`). A stray write crashes the process on CPU and Metal; +compare a default run against `--no-mmap` to confirm the outputs match. + Wire the new family into the arch dispatch in `src/` per the existing family wiring. Build iteratively: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2eb6e1cc2..94a194eba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -128,6 +128,10 @@ General rules: - Do not add new ggml operators, `ggml_type` values, or backend behavior without CPU validation, benchmark or accuracy evidence where relevant, and an upstream plan for ggml/llama.cpp when appropriate. +- Treat GGUF-backed weight tensors (a family's `ctx_meta`) as read-only after + load: they are memory-mapped from the file wherever the device allows it, so + a write crashes. Load-time transforms (fusions, promotions, packing) write + new tensors into their own buffer and repoint the weight slot. - Do not mix functional changes with broad reformatting or unrelated cleanup. - Keep comments concise. Explain non-obvious invariants, ABI contracts, tensor layout, numerical choices, and portability traps. Do not preserve local task diff --git a/README.md b/README.md index 098a27be4..c9237a39a 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,14 @@ Input must be 16 kHz mono WAV. Use `ffmpeg` or `sox` to convert other formats: ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav ``` +On the CPU and on Metal (Apple silicon), model weights are memory-mapped from +the GGUF file rather than copied into process memory: they live in the page +cache, which the OS can reclaim and share between processes, and loading is +faster. Don't truncate or rewrite a model file in place while a program has it +loaded; write a new file and rename it over the old one. `--no-mmap` +(`transcribe_model_load_params::no_mmap`, or `TRANSCRIBE_NO_MMAP=1`) copies the +weights instead. + ## Bindings Official bindings wrap the C API for other languages: diff --git a/bindings/python/README.md b/bindings/python/README.md index ccd53d887..f41cb8113 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -150,9 +150,16 @@ with transcribe_cpp.Model("model.gguf", device=device) as model: print(model.device) ``` +Weights are memory-mapped from the model file where the device can use host +memory (CPU, Metal on Apple silicon), so they live in reclaimable page cache +rather than process memory. Don't truncate or rewrite a model file in place +while it is loaded; write a new file and rename it over the old one. +`Model(no_mmap=True)` copies the weights instead. + | Variable | Effect | |---|---| | `TRANSCRIBE_BACKEND` | overrides the `"auto"` default; explicit `backend=` still wins | +| `TRANSCRIBE_NO_MMAP` | `1` copies weights instead of memory-mapping them, like `no_mmap=True` | | `TRANSCRIBE_NATIVE_PROVIDER` | forces an installed native provider package, for example `cu12` | | `TRANSCRIBE_LIBRARY` | loads exactly this shared library | diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 4b406a812..3527470ee 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -1057,10 +1057,18 @@ class Model: the escape hatch for machines whose best-ranked device misbehaves (e.g. CI's paravirtualized Metal, a broken GPU driver) without touching code. An explicit ``backend=`` argument always wins over the environment. + + The weights are memory-mapped from the model file where the device can + use host memory (the CPU everywhere, Metal on Apple silicon): they stay + in reclaimable page cache instead of process memory. Do not truncate or + rewrite the file in place while the model is open (write a new file and + rename it over the old one). ``no_mmap=True``, or the + ``TRANSCRIBE_NO_MMAP=1`` environment variable, copies them instead. """ def __init__(self, path: str | os.PathLike, *, - backend: Backend = "auto", device: BackendDevice | None = None): + backend: Backend = "auto", device: BackendDevice | None = None, + no_mmap: bool = False): # Live sessions, tracked weakly: close() must free them before the # model, because transcribe_model_free is only valid once every # derived session is gone (use-after-free otherwise). Created before @@ -1078,6 +1086,7 @@ def __init__(self, path: str | os.PathLike, *, if not isinstance(device, BackendDevice) or device._handle is None: raise TypeError("device must be a BackendDevice returned by backends()") params.device = device._handle + params.no_mmap = bool(no_mmap) handle = ctypes.c_void_p() status = _lib.transcribe_model_load_file( @@ -2260,6 +2269,7 @@ def transcribe( *, backend: Backend = "auto", device: BackendDevice | None = None, + no_mmap: bool = False, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0, @@ -2282,8 +2292,9 @@ def transcribe( *model* may be a path (loaded and freed within this call) or an existing Model (reused and left open). Loading a model is not free, so to transcribe many clips keep a Model and call ``model.session().run(...)`` yourself; this - helper is for the one-shot case. ``backend`` / ``device`` apply only when - *model* is a path — they are ignored when an already-loaded Model is passed. + helper is for the one-shot case. ``backend`` / ``device`` / ``no_mmap`` + apply only when *model* is a path — they are ignored when an already-loaded + Model is passed. ``family`` / ``spec_k_drafts`` and the prompting inputs (``vocabulary``, ``prompt``, ``prefix``) pass through to :meth:`Session.run`. """ @@ -2298,6 +2309,6 @@ def transcribe( with model.session(**session_opts) as session: return session.run(pcm, **run_opts) - with Model(model, backend=backend, device=device) as owned: + with Model(model, backend=backend, device=device, no_mmap=no_mmap) as owned: with owned.session(**session_opts) as session: return session.run(pcm, **run_opts) diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index 9c0d7f7b6..844dfa8a4 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "98540564cd329b85" +PUBLIC_HEADER_HASH = "f99ad4169374b8f3" # === enum constants === TRANSCRIBE_OK = 0 @@ -228,7 +228,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_ext._fields_ = [("size", _c.c_uint64), ("kind", _c.c_uint32)] transcribe_backend_init_params._fields_ = [("struct_size", _c.c_uint64), ("artifact_dir", _c.c_char_p), ("allowed_backends", _c.c_uint32)] transcribe_device_info._fields_ = [("struct_size", _c.c_uint64), ("name", _c.c_char_p), ("description", _c.c_char_p), ("kind", _c.c_char_p), ("device_id", _c.c_char_p), ("memory_total", _c.c_uint64), ("memory_free", _c.c_uint64), ("device_type", _c.c_int)] -transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("device", _c.c_void_p)] +transcribe_model_load_params._fields_ = [("struct_size", _c.c_uint64), ("backend", _c.c_int), ("device", _c.c_void_p), ("no_mmap", _c.c_bool)] transcribe_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int), ("kv_type", _c.c_int), ("n_ctx", _c.c_int32)] transcribe_run_params._fields_ = [("struct_size", _c.c_uint64), ("task", _c.c_int), ("timestamps", _c.c_int), ("pnc", _c.c_int), ("itn", _c.c_int), ("diarize", _c.c_int), ("language", _c.c_char_p), ("target_language", _c.c_char_p), ("keep_special_tags", _c.c_bool), ("family", _c.POINTER(transcribe_ext)), ("spec_k_drafts", _c.c_int32), ("vocabulary", _c.POINTER(_c.c_char_p)), ("n_vocabulary", _c.c_int32), ("prompt", _c.c_char_p), ("prefix", _c.c_char_p)] transcribe_capabilities._fields_ = [("struct_size", _c.c_uint64), ("native_sample_rate", _c.c_int32), ("n_languages", _c.c_int), ("languages", _c.POINTER(_c.c_char_p)), ("max_timestamp_kind", _c.c_int), ("supports_language_detect", _c.c_bool), ("supports_translate", _c.c_bool), ("supports_streaming", _c.c_bool), ("supports_spec_decode", _c.c_bool), ("max_audio_ms", _c.c_int64), ("n_translate_target_languages", _c.c_int), ("translate_target_languages", _c.POINTER(_c.c_char_p))] @@ -301,7 +301,7 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_ext': {'size': 16, 'align': 8, 'offsets': {'size': 0, 'kind': 8}}, 'transcribe_backend_init_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'artifact_dir': 8, 'allowed_backends': 16}}, 'transcribe_device_info': {'size': 64, 'align': 8, 'offsets': {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56}}, - 'transcribe_model_load_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'device': 16}}, + 'transcribe_model_load_params': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 'backend': 8, 'device': 16, 'no_mmap': 24}}, 'transcribe_session_params': {'size': 24, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16}}, 'transcribe_run_params': {'size': 104, 'align': 8, 'offsets': {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'diarize': 24, 'language': 32, 'target_language': 40, 'keep_special_tags': 48, 'family': 56, 'spec_k_drafts': 64, 'vocabulary': 72, 'n_vocabulary': 80, 'prompt': 88, 'prefix': 96}}, 'transcribe_capabilities': {'size': 56, 'align': 8, 'offsets': {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48}}, diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index 961123cf9..db7d0d9fd 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 98540564cd329b85 +// Pinned to include/transcribe.abihash = f99ad4169374b8f3 /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "98540564cd329b85"; +pub const PUBLIC_HEADER_HASH: &str = "f99ad4169374b8f3"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -348,11 +348,12 @@ pub struct transcribe_model_load_params { pub struct_size: u64, pub backend: transcribe_backend_request, pub device: transcribe_device_t, + pub no_mmap: bool, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of transcribe_model_load_params"] - [::std::mem::size_of::() - 24usize]; + [::std::mem::size_of::() - 32usize]; ["Alignment of transcribe_model_load_params"] [::std::mem::align_of::() - 8usize]; ["Offset of field: transcribe_model_load_params::struct_size"] @@ -361,6 +362,8 @@ const _: () = { [::std::mem::offset_of!(transcribe_model_load_params, backend) - 8usize]; ["Offset of field: transcribe_model_load_params::device"] [::std::mem::offset_of!(transcribe_model_load_params, device) - 16usize]; + ["Offset of field: transcribe_model_load_params::no_mmap"] + [::std::mem::offset_of!(transcribe_model_load_params, no_mmap) - 24usize]; }; unsafe extern "C" { pub fn transcribe_model_load_params_init(params: *mut transcribe_model_load_params); diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index a1de72948..77dfd10af 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -177,6 +177,15 @@ finish `init_backends()` or `init_backends_default()` before any thread enumerates devices, queries backend availability, or loads a model; native registry mutation is a startup-only operation and must not race those calls. +## Memory-mapped weights + +Where the device can use host memory (the CPU, Metal on Apple silicon), model +weights are memory-mapped from the file and served from the page cache, which +the OS can reclaim, instead of being copied into process memory. Don't +truncate or rewrite a model file in place while it is loaded; write a new file +and rename it over the old one. Set `ModelOptions::no_mmap` (or the +`TRANSCRIBE_NO_MMAP=1` environment variable) to copy the weights instead. + ## Packaging a distributable (`shared` / `dynamic-backends`) With the **default static** build there is nothing to do — the native code is diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index fac1b0231..a406afa33 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -41,6 +41,13 @@ pub struct ModelOptions { /// Exact process-local compute device. `None` applies the backend's /// automatic policy; `Some` selects that device or fails without fallback. pub device: Option, + /// Copy the weights into memory instead of memory-mapping the model file. + /// Default `false`: where the device can use host memory (CPU, Metal on + /// Apple silicon) the weights are served from the file's page cache, which + /// the OS can reclaim. Don't truncate or rewrite a mapped model file in + /// place; write a new file and rename it over the old one. The + /// `TRANSCRIBE_NO_MMAP=1` environment variable forces `true`. + pub no_mmap: bool, } impl Default for ModelOptions { @@ -48,6 +55,7 @@ impl Default for ModelOptions { ModelOptions { backend: Backend::Auto, device: None, + no_mmap: false, } } } @@ -154,6 +162,7 @@ impl Model { .device .as_ref() .map_or(std::ptr::null_mut(), |device| device.handle); + params.no_mmap = options.no_mmap; let mut out: *mut sys::transcribe_model = std::ptr::null_mut(); let status = unsafe { sys::transcribe_model_load_file(c_path.as_ptr(), ¶ms, &mut out) }; diff --git a/bindings/rust/transcribe-cpp/tests/device_select.rs b/bindings/rust/transcribe-cpp/tests/device_select.rs index ca4447e40..773230395 100644 --- a/bindings/rust/transcribe-cpp/tests/device_select.rs +++ b/bindings/rust/transcribe-cpp/tests/device_select.rs @@ -21,6 +21,7 @@ fn every_primary_device_can_be_selected_exactly() { &ModelOptions { backend: Backend::Auto, device: Some(device.clone()), + ..Default::default() }, ); match result { @@ -53,6 +54,7 @@ fn explicit_backend_must_match_device() { &ModelOptions { backend: Backend::Cpu, device: Some(gpu), + ..Default::default() }, ) .unwrap_err(); diff --git a/bindings/swift/README.md b/bindings/swift/README.md index b71c31af1..be2459a6c 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -168,6 +168,13 @@ Request a backend policy with `ModelOptions(backend:)`; probe availability with `Transcribe.devices()` to `ModelOptions(device:)`; exact selection never falls back to another primary device. +On the CPU and on Metal, model weights are memory-mapped from the file: they +are served from the page cache, which the OS can reclaim, and do not count +toward the app's memory footprint (the figure jetsam enforces on iOS). Don't +truncate or rewrite a model file in place while it is loaded; write a new file +and rename it over the old one. `ModelOptions(noMmap: true)` copies the weights +instead. + ## Concurrency and lifetime - `Model` is shareable. In 0.x, compute is serialized per model, so concurrent diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index 972c881ed..6995475a9 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "98540564cd329b85" + public static let pinnedHeaderHash = "f99ad4169374b8f3" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index b3f4c0894..6c5340fb2 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -56,6 +56,7 @@ public final class Model: @unchecked Sendable { transcribe_model_load_params_init(¶ms) params.backend = options.backend.cValue params.device = options.device?.handle + params.no_mmap = options.noMmap var out: OpaquePointer? let status = transcribe_model_load_file(path, ¶ms, &out) try TranscribeError.check(status, context: "loading \(path)") diff --git a/bindings/swift/Sources/TranscribeCpp/Options.swift b/bindings/swift/Sources/TranscribeCpp/Options.swift index 9477942fb..bff4e5998 100644 --- a/bindings/swift/Sources/TranscribeCpp/Options.swift +++ b/bindings/swift/Sources/TranscribeCpp/Options.swift @@ -113,9 +113,17 @@ public struct ModelOptions: Sendable { public var backend: Backend /// Exact process-local device. `nil` applies the backend's automatic policy. public var device: Device? - public init(backend: Backend = .auto, device: Device? = nil) { + /// Copy the weights into memory instead of memory-mapping the model file. + /// Default `false`: on the CPU and on Metal the weights are served from the + /// file's page cache, which the OS can reclaim and which does not count + /// toward the app's memory footprint. Don't truncate or rewrite a mapped + /// model file in place; write a new file and rename it over the old one. + /// The `TRANSCRIBE_NO_MMAP=1` environment variable forces `true`. + public var noMmap: Bool + public init(backend: Backend = .auto, device: Device? = nil, noMmap: Bool = false) { self.backend = backend self.device = device + self.noMmap = noMmap } } diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 00ff84855..1e0121669 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -226,6 +226,13 @@ const exact = await TranscribeModel.load("model.gguf", { device: cpu }); loader/driver degrades silently to CPU; an explicit backend that cannot be satisfied raises a `BackendError`. +Where the device can use host memory (CPU, Metal on Apple silicon), weights are +memory-mapped from the model file and served from the page cache, which the OS +can reclaim, instead of being copied into process memory. Don't truncate or +rewrite a model file in place while it is loaded; write a new file and rename +it over the old one. `{ noMmap: true }` (or `TRANSCRIBE_NO_MMAP=1`) copies the +weights instead. + ## Thread-safety Your JavaScript runs on one thread, but `run`/`feed`/`finalize` dispatch the diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index e14a8479f..683d416d1 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "98540564cd329b85"; +export const PUBLIC_HEADER_HASH = "f99ad4169374b8f3"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -154,7 +154,7 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_ext': { size: 16, align: 8, offsets: {'size': 0, 'kind': 8} }, 'transcribe_backend_init_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'artifact_dir': 8, 'allowed_backends': 16} }, 'transcribe_device_info': { size: 64, align: 8, offsets: {'struct_size': 0, 'name': 8, 'description': 16, 'kind': 24, 'device_id': 32, 'memory_total': 40, 'memory_free': 48, 'device_type': 56} }, - 'transcribe_model_load_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'device': 16} }, + 'transcribe_model_load_params': { size: 32, align: 8, offsets: {'struct_size': 0, 'backend': 8, 'device': 16, 'no_mmap': 24} }, 'transcribe_session_params': { size: 24, align: 8, offsets: {'struct_size': 0, 'n_threads': 8, 'kv_type': 12, 'n_ctx': 16} }, 'transcribe_run_params': { size: 104, align: 8, offsets: {'struct_size': 0, 'task': 8, 'timestamps': 12, 'pnc': 16, 'itn': 20, 'diarize': 24, 'language': 32, 'target_language': 40, 'keep_special_tags': 48, 'family': 56, 'spec_k_drafts': 64, 'vocabulary': 72, 'n_vocabulary': 80, 'prompt': 88, 'prefix': 96} }, 'transcribe_capabilities': { size: 56, align: 8, offsets: {'struct_size': 0, 'native_sample_rate': 8, 'n_languages': 12, 'languages': 16, 'max_timestamp_kind': 24, 'supports_language_detect': 28, 'supports_translate': 29, 'supports_streaming': 30, 'supports_spec_decode': 31, 'max_audio_ms': 32, 'n_translate_target_languages': 40, 'translate_target_languages': 48} }, @@ -227,7 +227,7 @@ export function defineTypes(koffi: any): Record { T['transcribe_ext'] = koffi.struct({ size: 'uint64_t', kind: 'uint32_t' }); T['transcribe_backend_init_params'] = koffi.struct({ struct_size: 'uint64_t', artifact_dir: 'char *', allowed_backends: 'uint32_t' }); T['transcribe_device_info'] = koffi.struct({ struct_size: 'uint64_t', name: 'char *', description: 'char *', kind: 'char *', device_id: 'char *', memory_total: 'uint64_t', memory_free: 'uint64_t', device_type: 'int' }); - T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', device: 'void *' }); + T['transcribe_model_load_params'] = koffi.struct({ struct_size: 'uint64_t', backend: 'int', device: 'void *', no_mmap: 'bool' }); T['transcribe_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int', kv_type: 'int', n_ctx: 'int32_t' }); T['transcribe_run_params'] = koffi.struct({ struct_size: 'uint64_t', task: 'int', timestamps: 'int', pnc: 'int', itn: 'int', diarize: 'int', language: 'char *', target_language: 'char *', keep_special_tags: 'bool', family: 'void *', spec_k_drafts: 'int32_t', vocabulary: 'void *', n_vocabulary: 'int32_t', prompt: 'char *', prefix: 'char *' }); T['transcribe_capabilities'] = koffi.struct({ struct_size: 'uint64_t', native_sample_rate: 'int32_t', n_languages: 'int', languages: 'void *', max_timestamp_kind: 'int', supports_language_detect: 'bool', supports_translate: 'bool', supports_streaming: 'bool', supports_spec_decode: 'bool', max_audio_ms: 'int64_t', n_translate_target_languages: 'int', translate_target_languages: 'void *' }); diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index bc4ecc66c..00fe33837 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -1759,6 +1759,7 @@ export class TranscribeModel { } p.device = handle; } + if (opts.noMmap) p.no_mmap = true; const out: any[] = [null]; const st = await callAsync(n.F.modelLoadFile, path, p, out); diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index 97f6b1235..83ca0d8d1 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -145,6 +145,13 @@ export interface ModelOptions { /** Exact device returned by {@link getAvailableBackends}. Omit for the * backend's automatic policy. Exact selection never falls back. */ device?: BackendInfo; + /** Copy the weights into memory instead of memory-mapping the model file. + * Default false: where the device can use host memory (CPU, Metal on Apple + * silicon) the weights are served from the file's page cache, which the OS + * can reclaim. Don't truncate or rewrite a mapped model file in place; + * write a new file and rename it over the old one. The + * `TRANSCRIBE_NO_MMAP=1` environment variable forces true. */ + noMmap?: boolean; } export interface SessionOptions { diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 4e1ae2237..4bb0336da 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -25,6 +25,7 @@ of tests. | Variable | Effect | | --- | --- | | `TRANSCRIBE_BACKENDS=` | Restrict which backends may register: a comma-separated list of `cpu`, `metal`, `vulkan`, `cuda`, `rocm`, `other`, `all` (case-insensitive). An excluded backend never runs any code (no module load, no driver init). It can only narrow the mask a host passes to `transcribe_init_backends_ex()`, applies even if the host never calls it, and CPU is always kept. Unknown names are dropped and logged as an error naming what is still allowed, so a typo narrows rather than widens (`vulcan` means CPU-only). Unset or empty means `all`. Read once per process. | +| `TRANSCRIBE_NO_MMAP` | Copy model weights into memory instead of memory-mapping the model file, for every model load in the process (same as `transcribe_model_load_params::no_mmap`, CLI `--no-mmap`). Read at each load. | | `TRANSCRIBE_NO_FLASH` | Disable flash attention on encoder and decoder (forces the manual F32 path). | | `TRANSCRIBE_FORCE_FLASH` | Force flash attention on. Wins over `TRANSCRIBE_NO_FLASH` if both are set. | | `TRANSCRIBE_CONV_DIRECT_DW` / `TRANSCRIBE_CONV_NO_DIRECT_DW` | Force the depthwise-conv dispatch to the direct `conv_2d_dw` path / the im2col path, overriding the per-family backend default. | diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp index 071ba338b..3f8e49bfb 100644 --- a/examples/cli/asr.cpp +++ b/examples/cli/asr.cpp @@ -275,6 +275,7 @@ int run_asr_batch(const cli_args & args, std::ofstream * output) { transcribe_model_load_params_init(&mp); mp.backend = args.backend; mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + mp.no_mmap = args.no_mmap; if (args.device_index >= 0 && mp.device == nullptr) { std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); return EXIT_FAILURE; diff --git a/examples/cli/cli.h b/examples/cli/cli.h index a61f671da..96cbf5a90 100644 --- a/examples/cli/cli.h +++ b/examples/cli/cli.h @@ -36,7 +36,8 @@ struct cli_args { int n_ctx = 0; // 0 = model's true max; >0 lowers the cap transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + bool no_mmap = false; // --no-mmap: copy weights instead of mapping the file transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; // Generic prompting (transcribe_run_params::vocabulary / prompt / prefix). diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index aecb2e4de..ef7d4aa4b 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -66,6 +66,8 @@ void print_usage(const char * argv0) { " (default: auto)\n" " --device N exact device index from --list-devices, including 0\n" " (default: automatic device selection)\n" + " --no-mmap copy the weights into memory instead of memory-mapping\n" + " the model file (TRANSCRIBE_NO_MMAP=1 does the same)\n" " --timestamps TYPE timestamps: auto, none, segment, word, token (default: auto)\n" " --batch FILE batch mode: FILE has one wav path per line\n" " --batch-jsonl output one JSON line per file (for batch)\n" @@ -326,6 +328,8 @@ bool parse_args(int argc, char ** argv, cli_args & out) { std::fprintf(stderr, "error: --backend must be auto, cpu, cpu_accel, metal, vulkan, cuda, or rocm\n"); return false; } + } else if (a == "--no-mmap") { + out.no_mmap = true; } else if (a == "--device") { const char * v = take_value(a.c_str()); if (!v) { @@ -610,6 +614,7 @@ int run_file(const cli_args & args, std::ofstream * output) { transcribe_model_load_params_init(&mp); mp.backend = args.backend; mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + mp.no_mmap = args.no_mmap; if (args.device_index >= 0 && mp.device == nullptr) { std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); return EXIT_FAILURE; diff --git a/ggml/UPSTREAM b/ggml/UPSTREAM index 6bcede247..923e32dff 100644 --- a/ggml/UPSTREAM +++ b/ggml/UPSTREAM @@ -3,6 +3,7 @@ sha: d7cb574130e6f01ad25b3289685489200febcd74 patches: patches/ggml/0001-fix-threadpool-oversubscription.patch patches/ggml/0002-backend-reg-filter.patch + patches/ggml/0003-metal-mapped-buffer-null.patch This directory is generated from the upstream ggml tree at the SHA above, minus .github/, with the listed downstream patches applied in order. Do not edit it by diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index 344882e04..9e4d970e0 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -744,6 +744,11 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size); + if (res == NULL) { + // transcribe.cpp patch: report the failed mapping instead of wrapping + // a NULL context in a buffer that crashes on first use. + return NULL; + } const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev); diff --git a/include/transcribe.abihash b/include/transcribe.abihash index 8b621ba4a..152e9d3f4 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -98540564cd329b85 +f99ad4169374b8f3 diff --git a/include/transcribe.h b/include/transcribe.h index 313c96e23..e690fe24c 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -1047,11 +1047,36 @@ TRANSCRIBE_API transcribe_device_t transcribe_model_device(const struct transcri * * Handles are process-local. Persist device_id (when available), then * enumerate and resolve a fresh handle in each process. + * + * no_mmap: false (the default) memory-maps the model file and serves the + * weights straight from it wherever the primary device can use host + * memory: the CPU on every platform, and Metal on Apple silicon. + * The weights then live in clean, file-backed pages that the OS can + * reclaim, share between models loaded from the same file, and does + * not count as the process's anonymous memory (what the Android + * low-memory killer and iOS/macOS jetsam charge). Loading is faster + * too; a first run on a cold page cache reads the weights from disk. + * Devices with their own memory (Vulkan, CUDA, ROCm) always get a + * copy, and weights a family converts at load are copied as well. + * true copies every weight into backend memory and closes the file, + * the behavior before memory mapping. + * + * While a memory-mapped model is loaded its file must not be + * truncated or rewritten in place: a later access to a changed page + * kills the process (SIGBUS; EXCEPTION_IN_PAGE_ERROR on Windows). + * Update a model file by writing a new file and renaming it over the + * old one. On Windows the file cannot be opened for writing or + * truncated until the model is freed. Use no_mmap for models on + * removable or network storage that may disappear. + * + * Setting the TRANSCRIBE_NO_MMAP environment variable (to anything + * but "0") forces no_mmap for every load in the process. */ struct transcribe_model_load_params { uint64_t struct_size; transcribe_backend_request backend; transcribe_device_t device; + bool no_mmap; }; TRANSCRIBE_API void transcribe_model_load_params_init(struct transcribe_model_load_params * params); diff --git a/patches/ggml/0003-metal-mapped-buffer-null.patch b/patches/ggml/0003-metal-mapped-buffer-null.patch new file mode 100644 index 000000000..175c73d55 --- /dev/null +++ b/patches/ggml/0003-metal-mapped-buffer-null.patch @@ -0,0 +1,16 @@ +diff --git a/src/ggml-metal/ggml-metal.cpp b/src/ggml-metal/ggml-metal.cpp +index 344882e0..9e4d970e 100644 +--- a/src/ggml-metal/ggml-metal.cpp ++++ b/src/ggml-metal/ggml-metal.cpp +@@ -744,6 +744,11 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen + ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; + + ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size); ++ if (res == NULL) { ++ // transcribe.cpp patch: report the failed mapping instead of wrapping ++ // a NULL context in a buffer that crashes on first use. ++ return NULL; ++ } + + const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev); + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15c7685b8..68606dcc8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -31,6 +31,7 @@ add_library(transcribe transcribe-flash-policy.cpp transcribe-backend.cpp transcribe-load-common.cpp + transcribe-mmap.cpp transcribe-weights-util.cpp transcribe-bin-loader.cpp transcribe-batch-util.cpp diff --git a/src/arch/canary/canary.h b/src/arch/canary/canary.h index 27bac0115..7b0e9e7e9 100644 --- a/src/arch/canary/canary.h +++ b/src/arch/canary/canary.h @@ -9,6 +9,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -103,8 +104,8 @@ struct CanaryModel final : public transcribe_model { CanaryWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused BN parameters (same as parakeet/cohere). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/canary/model.cpp b/src/arch/canary/model.cpp index dbdbf488d..cdaf3ac64 100644 --- a/src/arch/canary/model.cpp +++ b/src/arch/canary/model.cpp @@ -198,10 +198,7 @@ CanaryModel::~CanaryModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -502,17 +499,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "canary"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "canary", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/canary_qwen/canary_qwen.h b/src/arch/canary_qwen/canary_qwen.h index 89ed2c87e..2674cae8f 100644 --- a/src/arch/canary_qwen/canary_qwen.h +++ b/src/arch/canary_qwen/canary_qwen.h @@ -19,6 +19,7 @@ #include "causal_lm/causal_lm.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -57,8 +58,8 @@ struct CanaryQwenModel final : public transcribe_model { CanaryQwenWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // BatchNorm fusion in the conformer conv module: fused (scale, bias) // computed at load time; running_mean/var unused at inference. diff --git a/src/arch/canary_qwen/model.cpp b/src/arch/canary_qwen/model.cpp index 95698cbb0..5b00b4068 100644 --- a/src/arch/canary_qwen/model.cpp +++ b/src/arch/canary_qwen/model.cpp @@ -70,10 +70,7 @@ CanaryQwenModel::~CanaryQwenModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); if (bn_fused_buffer != nullptr) { safe_buffer_free(bn_fused_buffer); bn_fused_buffer = nullptr; @@ -648,16 +645,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "canary_qwen: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "canary_qwen"); + if (const transcribe_status st = load_common::load_weights(loader.path(), gguf_data, m->ctx_meta, m->plan, params, + "canary_qwen", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/cohere/cohere.h b/src/arch/cohere/cohere.h index 0a7e98604..09f1449e3 100644 --- a/src/arch/cohere/cohere.h +++ b/src/arch/cohere/cohere.h @@ -9,6 +9,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -131,13 +132,18 @@ struct CohereModel final : public transcribe_model { // Runtime backend plan — see transcribe-backend.h. scheduler_list // holds the backend handles plus a classified primary kind. - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused BN parameters (same as Parakeet). ggml_context * bn_fused_ctx = nullptr; ggml_backend_buffer_t bn_fused_buffer = nullptr; + // pos_bias_u/v with the encoder Q bias folded in (fuse_encoder_q_bias); + // block attn_pos_u/v slots point here once the fusion has run. + ggml_context * q_bias_fused_ctx = nullptr; + ggml_backend_buffer_t q_bias_fused_buffer = nullptr; + // On CPU primary backend, the conformer 1×1 pointwise conv weights // are dequantized F16->F32 at load time (Zen 2 has no native F16 // compute). Tensors live here; CohereBlock slots point at them. diff --git a/src/arch/cohere/model.cpp b/src/arch/cohere/model.cpp index fc9cc8a02..e7f9e0544 100644 --- a/src/arch/cohere/model.cpp +++ b/src/arch/cohere/model.cpp @@ -187,6 +187,14 @@ bool kv_cache_init_batched(CohereKvCache & cache, } CohereModel::~CohereModel() { + if (q_bias_fused_ctx != nullptr) { + ggml_free(q_bias_fused_ctx); + q_bias_fused_ctx = nullptr; + } + if (q_bias_fused_buffer != nullptr) { + safe_buffer_free(q_bias_fused_buffer); + q_bias_fused_buffer = nullptr; + } if (bn_fused_ctx != nullptr) { ggml_free(bn_fused_ctx); bn_fused_ctx = nullptr; @@ -207,10 +215,7 @@ CohereModel::~CohereModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -341,36 +346,52 @@ transcribe_status fuse_batch_norm(CohereModel & m) { // q_u = (W_q x + q_b) + pos_u = W_q x + (q_b + pos_u) // q_v = (W_q x + q_b) + pos_v = W_q x + (q_b + pos_v) // so pre-adding q_b into pos_u/pos_v is identical and drops the explicit -// `q = q + q_b` graph op. We null attn_q_b after so the builder skips the add. +// `q = q + q_b` graph op. The sums go into new tensors in q_bias_fused_ctx / +// q_bias_fused_buffer (the GGUF-backed pos_bias tensors may be a read-only +// memory map, and weights are never written after load); the block's +// pos_u/pos_v slots are repointed at them and attn_q_b is nulled so the +// builder skips the add. weights.cpp binds all three as F32 (q_b [d_model], +// pos_u/v [head_dim, n_heads]) and read_cohere_hparams rejects d_model not +// divisible by n_heads, so every block fuses. transcribe_status fuse_encoder_q_bias(CohereModel & m) { const size_t n_blocks = m.weights.blocks.size(); if (n_blocks == 0) { return TRANSCRIBE_OK; } - const int64_t d_model = m.hparams.enc_d_model; - const int64_t n_heads = m.hparams.enc_n_heads; - const int64_t head_dim = n_heads > 0 ? d_model / n_heads : 0; + const int64_t d_model = m.hparams.enc_d_model; - if (head_dim * n_heads != d_model) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "cohere: d_model (%lld) != head_dim*n_heads; " - "skipping encoder Q-bias fusion", - (long long) d_model); - return TRANSCRIBE_OK; + const size_t ctx_size = n_blocks * 2 * ggml_tensor_overhead() + 256; + ggml_init_params params = { ctx_size, nullptr, true }; + m.q_bias_fused_ctx = ggml_init(params); + if (m.q_bias_fused_ctx == nullptr) { + return TRANSCRIBE_ERR_OOM; + } + + // Same shape and name as the tensors they replace, on the same backend. + std::vector fused_u(n_blocks, nullptr); + std::vector fused_v(n_blocks, nullptr); + for (size_t i = 0; i < n_blocks; ++i) { + const auto & b = m.weights.blocks[i]; + fused_u[i] = ggml_new_tensor(m.q_bias_fused_ctx, GGML_TYPE_F32, ggml_n_dims(b.attn_pos_u), b.attn_pos_u->ne); + fused_v[i] = ggml_new_tensor(m.q_bias_fused_ctx, GGML_TYPE_F32, ggml_n_dims(b.attn_pos_v), b.attn_pos_v->ne); + ggml_set_name(fused_u[i], ggml_get_name(b.attn_pos_u)); + ggml_set_name(fused_v[i], ggml_get_name(b.attn_pos_v)); } + m.q_bias_fused_buffer = ggml_backend_alloc_ctx_tensors(m.q_bias_fused_ctx, m.plan.primary); + if (m.q_bias_fused_buffer == nullptr) { + return TRANSCRIBE_ERR_OOM; + } + ggml_backend_buffer_set_usage(m.q_bias_fused_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + const size_t nbytes = static_cast(d_model) * sizeof(float); std::vector q_bias(d_model); std::vector pos_u(d_model); std::vector pos_v(d_model); - size_t fused = 0; for (size_t i = 0; i < n_blocks; ++i) { auto & b = m.weights.blocks[i]; - if (b.attn_q_b == nullptr || b.attn_pos_u == nullptr || b.attn_pos_v == nullptr) { - continue; - } ggml_backend_tensor_get(b.attn_q_b, q_bias.data(), 0, nbytes); ggml_backend_tensor_get(b.attn_pos_u, pos_u.data(), 0, nbytes); @@ -383,18 +404,16 @@ transcribe_status fuse_encoder_q_bias(CohereModel & m) { pos_v[j] += q_bias[j]; } - ggml_backend_tensor_set(b.attn_pos_u, pos_u.data(), 0, nbytes); - ggml_backend_tensor_set(b.attn_pos_v, pos_v.data(), 0, nbytes); + ggml_backend_tensor_set(fused_u[i], pos_u.data(), 0, nbytes); + ggml_backend_tensor_set(fused_v[i], pos_v.data(), 0, nbytes); - // Drop the reference so the graph builder skips the add. - // Tensor memory stays allocated inside ctx_meta. - b.attn_q_b = nullptr; - ++fused; + // Repoint the slots; drop q_b so the graph builder skips the add. + b.attn_pos_u = fused_u[i]; + b.attn_pos_v = fused_v[i]; + b.attn_q_b = nullptr; } - if (fused > 0) { - log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "cohere: fused Q bias into pos_u/pos_v for %zu encoder blocks", fused); - } + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "cohere: fused Q bias into pos_u/pos_v for %zu encoder blocks", n_blocks); return TRANSCRIBE_OK; } @@ -572,19 +591,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "cohere: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - // Stream tensor data from the GGUF file into the backend buffer - // slots. See transcribe-load-common.h for the shared loop. - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "cohere"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "cohere", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/ecapa_tdnn/ecapa_tdnn.h b/src/arch/ecapa_tdnn/ecapa_tdnn.h index f282ad5f0..00fba0014 100644 --- a/src/arch/ecapa_tdnn/ecapa_tdnn.h +++ b/src/arch/ecapa_tdnn/ecapa_tdnn.h @@ -7,6 +7,7 @@ #include "transcribe-backend.h" #include "transcribe-langid.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "weights.h" @@ -30,10 +31,10 @@ struct Model final : public transcribe_model { Weights weights; // GGUF metadata context: owns the ggml_tensor structs the weight slots - // above point at. Their data lives in backend_buffer. - ggml_context * ctx_meta = nullptr; - ggml_backend_buffer_t backend_buffer = nullptr; - BackendPlan plan; + // above point at. Their data lives in weight_storage. + ggml_context * ctx_meta = nullptr; + load_common::WeightStorage weight_storage; + BackendPlan plan; // Load-time derived weights (Weights::blk0_w_im2col): their own metadata // context and backend buffer, since ctx_meta is sized for the file. diff --git a/src/arch/ecapa_tdnn/model.cpp b/src/arch/ecapa_tdnn/model.cpp index f20293665..0b8451099 100644 --- a/src/arch/ecapa_tdnn/model.cpp +++ b/src/arch/ecapa_tdnn/model.cpp @@ -115,8 +115,13 @@ int widen_q8_0_weights(ggml_context * ctx_meta) { // load_common::stream_tensor_data, plus the Q8_0 -> F16 widening: a tensor // whose file type is Q8_0 and whose bound type is F16 is dequantized with // ggml's own Q8_0 reference and rounded to F16 on the way in. Every other -// tensor must be bound with its file type and is copied as-is. -transcribe_status stream_weights(const std::string & path, const gguf_context * gguf, ggml_context * ctx_meta) { +// tensor must be bound with its file type and is copied as-is, unless +// `storage` memory-mapped it (allocate_weights maps only tensors whose +// file bytes are usable as-is, so never a widened one). +transcribe_status stream_weights(const std::string & path, + const gguf_context * gguf, + ggml_context * ctx_meta, + const load_common::WeightStorage & storage) { std::ifstream fin(path_from_utf8(path), std::ios::binary); if (!fin) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: failed to reopen %s for tensor data", kTag, path.c_str()); @@ -130,6 +135,9 @@ transcribe_status stream_weights(const std::string & path, const gguf_context * std::vector f16; for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (storage.is_mapped(t)) { + continue; // already backed by the file bytes + } const int64_t idx = gguf_find_tensor(gguf, t->name); if (idx < 0) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: tensor \"%s\" not in gguf data", kTag, t->name); @@ -310,14 +318,14 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - m->backend_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (m->backend_buffer == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", kTag); - return TRANSCRIBE_ERR_OOM; + // Tensors widened above differ from their file bytes, so allocate_weights + // copies them and maps the rest. + if (auto st = load_common::allocate_weights(loader.path(), guard.ctx, m->ctx_meta, m->plan, params, kTag, + m->weight_storage); + st != TRANSCRIBE_OK) { + return st; } - ggml_backend_buffer_set_usage(m->backend_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = stream_weights(loader.path(), guard.ctx, m->ctx_meta); st != TRANSCRIBE_OK) { + if (auto st = stream_weights(loader.path(), guard.ctx, m->ctx_meta, m->weight_storage); st != TRANSCRIBE_OK) { return st; } if (auto st = build_derived_weights(*m); st != TRANSCRIBE_OK) { @@ -519,9 +527,7 @@ Model::~Model() { if (ctx_meta != nullptr) { ggml_free(ctx_meta); } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - } + weight_storage.reset(); if (ctx_derived != nullptr) { ggml_free(ctx_derived); } diff --git a/src/arch/funasr_nano/funasr_nano.h b/src/arch/funasr_nano/funasr_nano.h index 31f765203..97ea4d515 100644 --- a/src/arch/funasr_nano/funasr_nano.h +++ b/src/arch/funasr_nano/funasr_nano.h @@ -8,6 +8,7 @@ #include "causal_lm/causal_lm.h" #include "transcribe-backend.h" #include "transcribe-kaldi-fbank.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -54,7 +55,7 @@ struct FunAsrNanoModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; // Constructed once at load() time; const-after-construction. Kaldi diff --git a/src/arch/funasr_nano/model.cpp b/src/arch/funasr_nano/model.cpp index 8dfa567af..25af20815 100644 --- a/src/arch/funasr_nano/model.cpp +++ b/src/arch/funasr_nano/model.cpp @@ -57,10 +57,7 @@ FunAsrNanoModel::~FunAsrNanoModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -419,16 +416,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "funasr_nano: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "funasr_nano"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "funasr_nano", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/gigaam/gigaam.h b/src/arch/gigaam/gigaam.h index 8ceaaae16..3815b8395 100644 --- a/src/arch/gigaam/gigaam.h +++ b/src/arch/gigaam/gigaam.h @@ -12,6 +12,7 @@ #include "decoder.h" #include "mel.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -45,8 +46,8 @@ struct GigaamModel final : public transcribe_model { GigaamWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Family-specific mel frontend (HTK mel, power=2, log-clamp, // center=False, periodic Hann). See arch/gigaam/mel.{h,cpp}. diff --git a/src/arch/gigaam/model.cpp b/src/arch/gigaam/model.cpp index 42c4720ec..31195055f 100644 --- a/src/arch/gigaam/model.cpp +++ b/src/arch/gigaam/model.cpp @@ -73,10 +73,7 @@ GigaamModel::~GigaamModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -150,16 +147,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "gigaam: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "gigaam"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "gigaam", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite/granite.h b/src/arch/granite/granite.h index f5c6c1301..1e01166dd 100644 --- a/src/arch/granite/granite.h +++ b/src/arch/granite/granite.h @@ -27,6 +27,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -71,8 +72,8 @@ struct GraniteModel final : public transcribe_model { GraniteWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Pre-fused BatchNorm scale + bias for every encoder block. Lives // in a separate ggml context + backend buffer so we can write the diff --git a/src/arch/granite/model.cpp b/src/arch/granite/model.cpp index 357a85b38..7005c537b 100644 --- a/src/arch/granite/model.cpp +++ b/src/arch/granite/model.cpp @@ -61,10 +61,7 @@ GraniteModel::~GraniteModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -420,17 +417,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "granite"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "granite", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite5_ctc/granite5_ctc.h b/src/arch/granite5_ctc/granite5_ctc.h index 072faa410..124b2b8b5 100644 --- a/src/arch/granite5_ctc/granite5_ctc.h +++ b/src/arch/granite5_ctc/granite5_ctc.h @@ -15,6 +15,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -42,8 +43,8 @@ struct Granite5CtcModel final : public transcribe_model { Granite5CtcWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Per-block fused BatchNorm scale/bias, precomputed at load // (model.cpp fuse_batch_norm). Lives in its own context/buffer so diff --git a/src/arch/granite5_ctc/model.cpp b/src/arch/granite5_ctc/model.cpp index ae4724dc0..7565077a1 100644 --- a/src/arch/granite5_ctc/model.cpp +++ b/src/arch/granite5_ctc/model.cpp @@ -46,10 +46,7 @@ Granite5CtcModel::~Granite5CtcModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); if (bn_fused_buffer != nullptr) { safe_buffer_free(bn_fused_buffer); bn_fused_buffer = nullptr; @@ -220,16 +217,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite5_ctc: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "granite5_ctc"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "granite5_ctc", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/granite_nar/granite_nar.h b/src/arch/granite_nar/granite_nar.h index 3e0a84bf6..500b86494 100644 --- a/src/arch/granite_nar/granite_nar.h +++ b/src/arch/granite_nar/granite_nar.h @@ -13,6 +13,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -40,8 +41,8 @@ struct GraniteNarModel final : public transcribe_model { GraniteNarWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Pre-fused BN (parakeet-style). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/granite_nar/model.cpp b/src/arch/granite_nar/model.cpp index 65c981a3a..0b02687a0 100644 --- a/src/arch/granite_nar/model.cpp +++ b/src/arch/granite_nar/model.cpp @@ -68,10 +68,7 @@ GraniteNarModel::~GraniteNarModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -340,17 +337,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "granite_nar: alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "granite_nar"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "granite_nar", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/medasr/medasr.h b/src/arch/medasr/medasr.h index 3d11881cf..0967d0016 100644 --- a/src/arch/medasr/medasr.h +++ b/src/arch/medasr/medasr.h @@ -6,6 +6,7 @@ #pragma once #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -35,8 +36,8 @@ struct MedAsrModel final : public transcribe_model { MedAsrWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; std::optional mel; diff --git a/src/arch/medasr/model.cpp b/src/arch/medasr/model.cpp index 2a1454c54..dff7be46b 100644 --- a/src/arch/medasr/model.cpp +++ b/src/arch/medasr/model.cpp @@ -61,10 +61,7 @@ MedAsrModel::~MedAsrModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -188,16 +185,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "medasr: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "medasr"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "medasr", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moonshine/model.cpp b/src/arch/moonshine/model.cpp index 2d93c23ec..00f61a647 100644 --- a/src/arch/moonshine/model.cpp +++ b/src/arch/moonshine/model.cpp @@ -53,10 +53,7 @@ MoonshineModel::~MoonshineModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -256,16 +253,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "moonshine: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "moonshine"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "moonshine", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moonshine/moonshine.h b/src/arch/moonshine/moonshine.h index e11044e65..b60d04ab3 100644 --- a/src/arch/moonshine/moonshine.h +++ b/src/arch/moonshine/moonshine.h @@ -14,6 +14,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -113,8 +114,8 @@ struct MoonshineModel final : public transcribe_model { MoonshineWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; MoonshineModel() = default; ~MoonshineModel() override; diff --git a/src/arch/moonshine_streaming/model.cpp b/src/arch/moonshine_streaming/model.cpp index 2c66045ca..7a6f150c3 100644 --- a/src/arch/moonshine_streaming/model.cpp +++ b/src/arch/moonshine_streaming/model.cpp @@ -69,10 +69,7 @@ MoonshineStreamingModel::~MoonshineStreamingModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -269,17 +266,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "moonshine_streaming: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "moonshine_streaming"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "moonshine_streaming", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moonshine_streaming/moonshine_streaming.h b/src/arch/moonshine_streaming/moonshine_streaming.h index 8cf76e9cb..cf08d28b9 100644 --- a/src/arch/moonshine_streaming/moonshine_streaming.h +++ b/src/arch/moonshine_streaming/moonshine_streaming.h @@ -13,6 +13,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -109,8 +110,8 @@ struct MoonshineStreamingModel final : public transcribe_model { MoonshineStreamingWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; MoonshineStreamingModel() = default; ~MoonshineStreamingModel() override; diff --git a/src/arch/moss/model.cpp b/src/arch/moss/model.cpp index 6357a2f4b..f704cbdfe 100644 --- a/src/arch/moss/model.cpp +++ b/src/arch/moss/model.cpp @@ -109,10 +109,7 @@ MossModel::~MossModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -393,17 +390,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "moss: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "moss"); + if (const transcribe_status st = transcribe::load_common::load_weights(loader.path(), gguf_data, m->ctx_meta, + m->plan, params, "moss", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/moss/moss.h b/src/arch/moss/moss.h index 9eef5aecd..01eba4f3a 100644 --- a/src/arch/moss/moss.h +++ b/src/arch/moss/moss.h @@ -16,6 +16,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -69,7 +70,7 @@ struct MossModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/parakeet/model.cpp b/src/arch/parakeet/model.cpp index a2f75eddb..a877781ea 100644 --- a/src/arch/parakeet/model.cpp +++ b/src/arch/parakeet/model.cpp @@ -1,11 +1,12 @@ // arch/parakeet/model.cpp - Parakeet family handler. // // load() binds a runtime backend, opens the GGUF with no_alloc=true so -// ggml builds the tensor catalog without touching the data section, -// allocates a backend buffer for every tensor, and streams the data -// section into it; afterward every borrowed ggml_tensor* in `weights` -// points at backend memory. The persistent model state is ctx_meta + -// backend_buffer + backends, freed in that order in the destructor. +// ggml builds the tensor catalog without touching the data section, then +// gives every tensor storage via load_common::load_weights (memory-mapped +// from the file where the device allows it, copied otherwise); afterward +// every borrowed ggml_tensor* in `weights` points at backend memory. The +// persistent model state is ctx_meta + weight_storage + backends, freed in +// that order in the destructor. // run() computes the mel front-end, runs the encoder graph, then // host-decodes (predictor + joint + TDT). @@ -95,8 +96,8 @@ void ParakeetSession::on_scratch_released() noexcept { } ParakeetModel::~ParakeetModel() { - // Teardown order: ctx_meta → backend_buffer → plan backends. The - // buffer must be freed before the backends (it holds a backend ref); + // Teardown order: ctx_meta → weight_storage → plan backends. The + // buffers must be freed before the backends (they hold a backend ref); // backends freed in reverse init order. if (bn_fused_ctx != nullptr) { ggml_free(bn_fused_ctx); @@ -129,10 +130,7 @@ ParakeetModel::~ParakeetModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -538,22 +536,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - // Allocate a backend buffer for every tensor in ctx_meta on the - // primary backend; the weight bytes are streamed in below. - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "parakeet: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - // Stream tensor data from the GGUF into the backend buffer slots - // (shared loop in transcribe-load-common.h; works on host-memory - // backends and discrete GPUs). - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "parakeet"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "parakeet", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/parakeet/parakeet.h b/src/arch/parakeet/parakeet.h index cf3ed9589..30ae27439 100644 --- a/src/arch/parakeet/parakeet.h +++ b/src/arch/parakeet/parakeet.h @@ -10,6 +10,7 @@ #include "../sortformer/sortformer.h" // embedded diarizer (multitalker bundle) #include "decoder.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -104,8 +105,8 @@ struct ParakeetModel final : public transcribe_model { // load_common::init_backends (see transcribe-backend.h). plan's // scheduler_list owns the backends; the destructor frees them in // reverse order. - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused BN parameters live in a separate ggml context + buffer, // computed at load time from the raw BN tensors. Freed in dtor. diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index d56645fab..66a7868be 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -54,10 +54,7 @@ QwenAsrModel::~QwenAsrModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -248,17 +245,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "qwen3_asr: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "qwen3_asr"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "qwen3_asr", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/qwen3_asr/qwen3_asr.h b/src/arch/qwen3_asr/qwen3_asr.h index 3bf8f4a0e..73b5634ba 100644 --- a/src/arch/qwen3_asr/qwen3_asr.h +++ b/src/arch/qwen3_asr/qwen3_asr.h @@ -10,6 +10,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -77,7 +78,7 @@ struct QwenAsrModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/sensevoice/model.cpp b/src/arch/sensevoice/model.cpp index 99c09c31f..5867a5069 100644 --- a/src/arch/sensevoice/model.cpp +++ b/src/arch/sensevoice/model.cpp @@ -50,10 +50,7 @@ SenseVoiceModel::~SenseVoiceModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -145,16 +142,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "sensevoice: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "sensevoice"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "sensevoice", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/sensevoice/sensevoice.h b/src/arch/sensevoice/sensevoice.h index 09564c8e5..bfdce0282 100644 --- a/src/arch/sensevoice/sensevoice.h +++ b/src/arch/sensevoice/sensevoice.h @@ -8,6 +8,7 @@ #include "transcribe-backend.h" #include "transcribe-kaldi-fbank.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-session.h" #include "transcribe-tokenizer.h" @@ -33,8 +34,8 @@ struct SenseVoiceModel final : public transcribe_model { SenseVoiceWeights weights; ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Constructed once at load() time; const-after-construction so // every context that derives from this model can share the cached diff --git a/src/arch/silero_vad/model.cpp b/src/arch/silero_vad/model.cpp index 3635e6de1..e2458d3dd 100644 --- a/src/arch/silero_vad/model.cpp +++ b/src/arch/silero_vad/model.cpp @@ -62,8 +62,13 @@ struct GgufGuard { GgufGuard & operator=(const GgufGuard &) = delete; }; -// Shared only by the two Silero file adapters. -transcribe_status init_storage(Model & m, const transcribe_model_load_params * params) { +// Shared only by the two Silero file adapters. `gguf` is null for the .bin +// adapter, whose weights are converted on the host and so always copied; the +// GGUF adapter memory-maps them where it can (load_common::allocate_weights). +transcribe_status init_storage(Model & m, + const transcribe_model_load_params * params, + const std::string & path, + const gguf_context * gguf) { // AUTO resolves to the strict CPU backend. The model is tiny and the CPU // path matches the reference to ~1e-6 in probability, while ggml-metal's // F32 mul_mm stages operands as half (about 2e-3). A specific GPU request @@ -81,13 +86,10 @@ transcribe_status init_storage(Model & m, const transcribe_model_load_params * p m.backend = ggml_backend_name(m.plan.primary); m.primary_backend = m.plan.primary; - m.backend_buffer = ggml_backend_alloc_ctx_tensors(m.ctx_meta, m.plan.primary); - if (m.backend_buffer == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", kTag); - return TRANSCRIBE_ERR_OOM; + if (gguf != nullptr) { + return load_common::allocate_weights(path, gguf, m.ctx_meta, m.plan, params, kTag, m.weight_storage); } - ggml_backend_buffer_set_usage(m.backend_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - return TRANSCRIBE_OK; + return load_common::allocate_copied_weights(m.ctx_meta, m.plan, kTag, m.weight_storage); } transcribe_status load(Loader & loader, const transcribe_model_load_params * params, transcribe_model ** out_model) { @@ -117,10 +119,11 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } - if (auto st = init_storage(*m, params); st != TRANSCRIBE_OK) { + if (auto st = init_storage(*m, params, loader.path(), guard.ctx); st != TRANSCRIBE_OK) { return st; } - if (auto st = load_common::stream_tensor_data(loader.path(), guard.ctx, m->ctx_meta, kTag); st != TRANSCRIBE_OK) { + if (auto st = load_common::stream_tensor_data(loader.path(), guard.ctx, m->ctx_meta, kTag, &m->weight_storage); + st != TRANSCRIBE_OK) { return st; } if (auto st = build_derived_weights(*m); st != TRANSCRIBE_OK) { @@ -428,7 +431,7 @@ transcribe_status load_from_bin(const char * path, if (auto st = bind_weights(m->ctx_meta, hp, m->weights); st != TRANSCRIBE_OK) { return st; } - if (auto st = init_storage(*m, params); st != TRANSCRIBE_OK) { + if (auto st = init_storage(*m, params, std::string(), nullptr); st != TRANSCRIBE_OK) { return st; } for (size_t i = 0; i < k_bin_count; ++i) { @@ -483,9 +486,7 @@ Model::~Model() { if (ctx_meta != nullptr) { ggml_free(ctx_meta); } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - } + weight_storage.reset(); if (ctx_derived != nullptr) { ggml_free(ctx_derived); } diff --git a/src/arch/silero_vad/silero_vad.h b/src/arch/silero_vad/silero_vad.h index 33ccdbe6a..911ce1f74 100644 --- a/src/arch/silero_vad/silero_vad.h +++ b/src/arch/silero_vad/silero_vad.h @@ -19,6 +19,7 @@ #pragma once #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-model.h" #include "transcribe-vad.h" @@ -102,11 +103,11 @@ struct Model final : public transcribe_model { HParams hparams; Weights weights; - ggml_context * ctx_meta = nullptr; // owns the file's tensor structs - ggml_backend_buffer_t backend_buffer = nullptr; - ggml_context * ctx_derived = nullptr; - ggml_backend_buffer_t derived_buffer = nullptr; - BackendPlan plan; + ggml_context * ctx_meta = nullptr; // owns the file's tensor structs + load_common::WeightStorage weight_storage; + ggml_context * ctx_derived = nullptr; + ggml_backend_buffer_t derived_buffer = nullptr; + BackendPlan plan; HostDecoder host; diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index dc8a5ca7d..09eb44557 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -53,9 +53,7 @@ SortformerModel::~SortformerModel() { if (ctx_meta != nullptr) { ggml_free(ctx_meta); } - if (backend_buffer != nullptr) { - transcribe::safe_buffer_free(backend_buffer); - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { transcribe::safe_backend_free(*it); } @@ -590,17 +588,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "sortformer: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "sortformer"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "sortformer", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/sortformer/sortformer.h b/src/arch/sortformer/sortformer.h index dd1bebeba..a9cb6a20e 100644 --- a/src/arch/sortformer/sortformer.h +++ b/src/arch/sortformer/sortformer.h @@ -15,6 +15,7 @@ #include "../parakeet/weights.h" // ParakeetHParams / ParakeetWeights (conformer) #include "transcribe-backend.h" #include "transcribe-diarize.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe/sortformer.h" // public preset enum + diarize ext @@ -52,9 +53,9 @@ struct SortformerModel final : public transcribe_model { // Sortformer-specific weights (encoder_proj + transformer + diar head). SortformerWeights weights; - ggml_context * ctx_meta = nullptr; - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + ggml_context * ctx_meta = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Fused conformer BN params (computed at load from the raw BN tensors). ggml_context * bn_fused_ctx = nullptr; diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index ae6575e19..577f8e883 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -62,10 +62,7 @@ VoxtralModel::~VoxtralModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -504,17 +501,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "voxtral"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "voxtral", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/voxtral/voxtral.h b/src/arch/voxtral/voxtral.h index 63adadee3..89876cc08 100644 --- a/src/arch/voxtral/voxtral.h +++ b/src/arch/voxtral/voxtral.h @@ -13,6 +13,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -60,7 +61,7 @@ struct VoxtralModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/voxtral_realtime/model.cpp b/src/arch/voxtral_realtime/model.cpp index eefcd228b..13bd06523 100644 --- a/src/arch/voxtral_realtime/model.cpp +++ b/src/arch/voxtral_realtime/model.cpp @@ -69,10 +69,7 @@ Model::~Model() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); packed_gate_up.free(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); @@ -263,17 +260,8 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral_realtime: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (auto st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "voxtral_realtime"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "voxtral_realtime", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/voxtral_realtime/voxtral_realtime.h b/src/arch/voxtral_realtime/voxtral_realtime.h index 88a1ef49b..3a035eddd 100644 --- a/src/arch/voxtral_realtime/voxtral_realtime.h +++ b/src/arch/voxtral_realtime/voxtral_realtime.h @@ -10,6 +10,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -49,7 +50,7 @@ struct Model final : public transcribe_model { ggml_context * ctx_meta = nullptr; transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::load_common::WeightStorage weight_storage; transcribe::causal_lm::PackedGateUpHandles packed_gate_up; std::optional mel; diff --git a/src/arch/whisper/bin_load.cpp b/src/arch/whisper/bin_load.cpp index 638fa6270..88e12278c 100644 --- a/src/arch/whisper/bin_load.cpp +++ b/src/arch/whisper/bin_load.cpp @@ -617,13 +617,13 @@ transcribe_status load_from_bin(const char * path m->primary_backend = m->plan.primary; // ---- Allocate backend buffer ---- - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (buf == nullptr) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", kTag); - return TRANSCRIBE_ERR_OOM; + // Copied, never memory-mapped: the .bin payload is not laid out for ggml + // (unaligned offsets, renamed/reshaped tensors) and the frontend tensors + // below are written after allocation. + if (auto st = transcribe::load_common::allocate_copied_weights(m->ctx_meta, m->plan, kTag, m->weight_storage); + st != TRANSCRIBE_OK) { + return st; } - m->backend_buffer = buf; - ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // ---- Stream tensor bytes from the .bin ---- if (auto st = transcribe::bin_loader::stream_tensor_data_from_bin(bm.path, stream_slots, "whisper"); diff --git a/src/arch/whisper/model.cpp b/src/arch/whisper/model.cpp index bb14c16ca..435fb24b5 100644 --- a/src/arch/whisper/model.cpp +++ b/src/arch/whisper/model.cpp @@ -58,10 +58,7 @@ WhisperModel::~WhisperModel() { ggml_free(ctx_meta); ctx_meta = nullptr; } - if (backend_buffer != nullptr) { - safe_buffer_free(backend_buffer); - backend_buffer = nullptr; - } + weight_storage.reset(); for (auto it = plan.scheduler_list.rbegin(); it != plan.scheduler_list.rend(); ++it) { safe_backend_free(*it); } @@ -480,17 +477,8 @@ transcribe_status whisper_load(Loader & loader, m->backend = ggml_backend_name(m->plan.primary); m->primary_backend = m->plan.primary; - ggml_backend_buffer_t weights_buffer = ggml_backend_alloc_ctx_tensors(m->ctx_meta, m->plan.primary); - if (weights_buffer == nullptr) { - gguf_free(gguf_data); - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "whisper: ggml_backend_alloc_ctx_tensors failed"); - return TRANSCRIBE_ERR_OOM; - } - m->backend_buffer = weights_buffer; - ggml_backend_buffer_set_usage(weights_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - if (const transcribe_status st = - transcribe::load_common::stream_tensor_data(loader.path(), gguf_data, m->ctx_meta, "whisper"); + if (const transcribe_status st = transcribe::load_common::load_weights( + loader.path(), gguf_data, m->ctx_meta, m->plan, params, "whisper", m->weight_storage); st != TRANSCRIBE_OK) { gguf_free(gguf_data); return st; diff --git a/src/arch/whisper/whisper.h b/src/arch/whisper/whisper.h index 184fb9eb2..bbdad49f3 100644 --- a/src/arch/whisper/whisper.h +++ b/src/arch/whisper/whisper.h @@ -7,6 +7,7 @@ #include "ggml-backend.h" #include "ggml.h" #include "transcribe-backend.h" +#include "transcribe-load-common.h" #include "transcribe-mel.h" #include "transcribe-model.h" #include "transcribe-session.h" @@ -268,8 +269,8 @@ struct WhisperModel final : public transcribe_model { ggml_context * ctx_meta = nullptr; // Runtime backend plan. See transcribe-backend.h. - transcribe::BackendPlan plan; - ggml_backend_buffer_t backend_buffer = nullptr; + transcribe::BackendPlan plan; + transcribe::load_common::WeightStorage weight_storage; // Language token ids keyed by BCP-47 short code (from general.languages). // Resolves a params.language string to the <|lang_xx|> token id (whisper diff --git a/src/transcribe-load-common.cpp b/src/transcribe-load-common.cpp index 0f17b856b..149844286 100644 --- a/src/transcribe-load-common.cpp +++ b/src/transcribe-load-common.cpp @@ -15,9 +15,11 @@ # include "ggml-metal.h" #endif #include "transcribe-backend.h" +#include "transcribe-env.h" #include "transcribe-log.h" #include "transcribe-path.h" +#include #include #include #include @@ -440,10 +442,278 @@ transcribe_status init_backends(transcribe_backend_request requested, return TRANSCRIBE_ERR_INVALID_ARG; } -transcribe_status stream_tensor_data(const std::string & path, - const gguf_context * gguf_data, - ggml_context * ctx_meta, - const char * error_tag) { +// --------------------------------------------------------------------------- +// Weight storage: memory-mapped where the device can use host memory, +// copied into a backend buffer otherwise. +// --------------------------------------------------------------------------- + +void WeightStorage::reset() noexcept { + // Buffers before the mapping. A host-pointer buffer only wraps the pages + // (its free never touches them), but it must not outlive them. + safe_buffer_free(mapped_buffer_); + mapped_buffer_ = nullptr; + safe_buffer_free(copy_buffer_); + copy_buffer_ = nullptr; + file_.reset(); +} + +bool WeightStorage::is_mapped(const ggml_tensor * t) const noexcept { + return t != nullptr && mapped_buffer_ != nullptr && t->buffer == mapped_buffer_; +} + +bool mmap_allowed(const transcribe_model_load_params * params) { + if (params != nullptr && params->no_mmap) { + return false; + } + return !env::flag("TRANSCRIBE_NO_MMAP"); +} + +namespace { + +constexpr double k_mib = 1024.0 * 1024.0; + +// The primary device can keep weights in a wrapped host allocation. ggml says +// so through buffer_from_host_ptr (CPU, Metal) and mmap_support (false for +// integrated CUDA/Vulkan GPUs). Pre-Apple7 Metal GPUs are also excluded: they +// include the discrete AMD GPUs of Intel Macs, whose weights belong in VRAM +// rather than in shared system memory. +bool device_can_map(const BackendPlan & plan, ggml_backend_dev_t dev) { + if (dev == nullptr) { + return false; + } + ggml_backend_dev_props props{}; + ggml_backend_dev_get_props(dev, &props); + if (!props.caps.buffer_from_host_ptr || !props.caps.mmap_support) { + return false; + } + if (plan.primary_kind == BackendKind::Metal && metal_backend_lacks_simdgroup_mm(plan.primary, dev)) { + return false; + } + return true; +} + +struct MapCounts { + size_t n_mapped = 0; + size_t mapped_bytes = 0; + size_t n_layout = 0; // in-memory type/size differs from the file (converted at load) + size_t n_misaligned = 0; // file offset not aligned for the device +}; + +// Map every tensor of ctx_meta whose file bytes are usable as-is. When nothing +// can be mapped, `file` and `out_buffer` are left empty, `why` says why, and +// the result is still TRANSCRIBE_OK: the caller copies everything instead. +transcribe_status map_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + ggml_backend_dev_t dev, + const char * error_tag, + MappedFile & file, + ggml_backend_buffer_t & out_buffer, + MapCounts & counts, + std::string & why) { + if (!file.open(path, &why)) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: cannot memory-map %s (%s); copying weights instead", error_tag, + path.c_str(), why.c_str()); + why = "memory mapping failed"; + return TRANSCRIBE_OK; + } + + const size_t file_size = file.size(); + const size_t data_offset = gguf_get_data_offset(gguf_data); + char * base = static_cast(file.addr()); + + struct Candidate { + ggml_tensor * t; + char * addr; + }; + + std::vector candidates; + size_t max_tensor_size = 0; + if (data_offset <= file_size) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (t->data != nullptr || t->buffer != nullptr || t->view_src != nullptr) { + continue; // already has storage + } + const int64_t idx = gguf_find_tensor(gguf_data, ggml_get_name(t)); + if (idx < 0) { + continue; // stream_tensor_data reports it + } + const size_t nbytes = ggml_nbytes(t); + if (gguf_get_tensor_type(gguf_data, idx) != t->type || gguf_get_tensor_size(gguf_data, idx) != nbytes || + !ggml_is_contiguous(t)) { + ++counts.n_layout; + continue; + } + const size_t toffset = gguf_get_tensor_offset(gguf_data, idx); + if (toffset > file_size - data_offset || nbytes > file_size - data_offset - toffset) { + continue; // runs past EOF: the stream reports the truncation + } + candidates.push_back({ t, base + data_offset + toffset }); + max_tensor_size = std::max(max_tensor_size, nbytes); + } + } + if (candidates.empty()) { + file.reset(); + why = "no tensor is stored as-is in the file"; + return TRANSCRIBE_OK; + } + + // Wrap the whole page-aligned mapping, not just the data section: ggml's + // CPU host-pointer buffer asserts 32-byte alignment of its base, which a + // GGUF with general.alignment < 32 would not give data_offset. + ggml_backend_buffer_t buffer = ggml_backend_dev_buffer_from_host_ptr(dev, base, file_size, max_tensor_size); + if (buffer == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: %s cannot wrap the memory-mapped file; copying weights instead", + error_tag, ggml_backend_dev_name(dev)); + file.reset(); + why = "device could not wrap the mapping"; + return TRANSCRIBE_OK; + } + + const size_t alignment = ggml_backend_buffer_get_alignment(buffer); + for (const Candidate & c : candidates) { + const size_t nbytes = ggml_nbytes(c.t); + if (alignment > 1 && reinterpret_cast(c.addr) % alignment != 0) { + ++counts.n_misaligned; + continue; + } + if (ggml_backend_buffer_get_alloc_size(buffer, c.t) != nbytes) { + ++counts.n_layout; + continue; + } + if (ggml_backend_tensor_alloc(buffer, c.t, c.addr) != GGML_STATUS_SUCCESS) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: %s rejected memory-mapped tensor \"%s\"", error_tag, + ggml_backend_dev_name(dev), ggml_get_name(c.t)); + out_buffer = buffer; // owned by the storage from here; tensors may point into it + return TRANSCRIBE_ERR_BACKEND; + } + ++counts.n_mapped; + counts.mapped_bytes += nbytes; + } + if (counts.n_mapped == 0) { + safe_buffer_free(buffer); + file.reset(); + why = counts.n_misaligned > 0 ? "tensors misaligned in the file" : "no tensor is stored as-is in the file"; + return TRANSCRIBE_OK; + } + + ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + out_buffer = buffer; + return TRANSCRIBE_OK; +} + +} // namespace + +transcribe_status allocate_copied_weights(ggml_context * ctx_meta, + const BackendPlan & plan, + const char * error_tag, + WeightStorage & out) { + bool any_unallocated = false; + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (t->data == nullptr && t->view_src == nullptr) { + any_unallocated = true; + break; + } + } + if (!any_unallocated) { + return TRANSCRIBE_OK; + } + if (out.copy_buffer_ != nullptr || plan.primary == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: weight storage already allocated or no primary backend", error_tag); + return TRANSCRIBE_ERR_BACKEND; + } + + // Allocates only the tensors that have no storage yet. + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx_meta, plan.primary); + if (buffer == nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: ggml_backend_alloc_ctx_tensors failed", error_tag); + return TRANSCRIBE_ERR_OOM; + } + ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + out.copy_buffer_ = buffer; + return TRANSCRIBE_OK; +} + +transcribe_status allocate_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out) { + if (plan.primary == nullptr || gguf_data == nullptr || ctx_meta == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + ggml_backend_dev_t dev = ggml_backend_get_device(plan.primary); + + std::string why; + MapCounts counts; + if (!mmap_allowed(params)) { + why = "memory mapping disabled"; + } else if (!MappedFile::supported()) { + why = "memory mapping unsupported on this platform"; + } else if (!device_can_map(plan, dev)) { + why = "device keeps weights in its own memory"; + } else if (const transcribe_status st = + map_weights(path, gguf_data, ctx_meta, dev, error_tag, out.file_, out.mapped_buffer_, counts, why); + st != TRANSCRIBE_OK) { + return st; + } + + if (const transcribe_status st = allocate_copied_weights(ctx_meta, plan, error_tag, out); st != TRANSCRIBE_OK) { + return st; + } + + size_t n_copied = 0, copied_bytes = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (out.copy_buffer_ != nullptr && t->buffer == out.copy_buffer_) { + ++n_copied; + copied_bytes += ggml_nbytes(t); + } + } + if (counts.n_mapped == 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: weights copied: %.1f MiB in %s (%s)", error_tag, copied_bytes / k_mib, + out.copy_buffer_ != nullptr ? ggml_backend_buffer_name(out.copy_buffer_) : "-", why.c_str()); + } else if (n_copied == 0) { + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s: weights memory-mapped: %.1f MiB in %s", error_tag, + counts.mapped_bytes / k_mib, ggml_backend_buffer_name(out.mapped_buffer_)); + } else { + char reasons[96]; + if (counts.n_layout > 0 && counts.n_misaligned > 0) { + std::snprintf(reasons, sizeof(reasons), "%zu converted at load, %zu misaligned in the file", + counts.n_layout, counts.n_misaligned); + } else if (counts.n_misaligned > 0) { + std::snprintf(reasons, sizeof(reasons), "misaligned in the file"); + } else { + std::snprintf(reasons, sizeof(reasons), "converted at load"); + } + log_msg(TRANSCRIBE_LOG_LEVEL_INFO, + "%s: weights memory-mapped: %.1f MiB in %s; %zu tensors (%.1f MiB) copied (%s)", error_tag, + counts.mapped_bytes / k_mib, ggml_backend_buffer_name(out.mapped_buffer_), n_copied, + copied_bytes / k_mib, reasons); + } + return TRANSCRIBE_OK; +} + +transcribe_status load_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out) { + if (const transcribe_status st = allocate_weights(path, gguf_data, ctx_meta, plan, params, error_tag, out); + st != TRANSCRIBE_OK) { + return st; + } + return stream_tensor_data(path, gguf_data, ctx_meta, error_tag, &out); +} + +transcribe_status stream_tensor_data(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const char * error_tag, + const WeightStorage * skip) { // std::ifstream rather than going through gguf's loader a third // time: ifstream::seekg takes a streamoff (signed 64-bit on every // platform we target), so multi-GB tensor offsets work without @@ -454,10 +724,20 @@ transcribe_status stream_tensor_data(const std::string & path, return TRANSCRIBE_ERR_GGUF; } + // Uploads go through a bounded staging buffer: ggml_backend_tensor_set + // stages each call whole, and for device memory the host cannot map, + // Vulkan grows a per-device staging buffer, kept for the life of the + // process, to the largest single write (243 MiB for Qwen3-ASR-1.7B's token + // embedding), on top of ours. + constexpr size_t k_stage_chunk = size_t(16) << 20; + const size_t data_offset = gguf_get_data_offset(gguf_data); std::vector staging; for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta); t != nullptr; t = ggml_get_next_tensor(ctx_meta, t)) { + if (skip != nullptr && skip->is_mapped(t)) { + continue; // already backed by the file bytes + } const int64_t idx = gguf_find_tensor(gguf_data, t->name); if (idx < 0) { log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: tensor \"%s\" not in gguf data", error_tag, t->name); @@ -474,21 +754,36 @@ transcribe_status stream_tensor_data(const std::string & path, return TRANSCRIBE_ERR_GGUF; } - if (staging.size() < nbytes) { - staging.resize(nbytes); - } - fin.read(reinterpret_cast(staging.data()), static_cast(nbytes)); - if (!fin) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: short read for tensor \"%s\" (%zu bytes)", error_tag, t->name, - nbytes); - return TRANSCRIBE_ERR_GGUF; + // Host buffers (the CPU) take the file bytes directly: no staging + // copy. Everything else (Metal, Vulkan, CUDA, ...) is uploaded in + // chunks through ggml_backend_tensor_set, which needs a buffer type + // that accepts partial writes: every default one does (ggml's CPU + // repack buffers would not, and assert). + if (t->buffer != nullptr && ggml_backend_buffer_is_host(t->buffer)) { + fin.read(static_cast(t->data), static_cast(nbytes)); + if (!fin) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: short read for tensor \"%s\" (%zu bytes)", error_tag, t->name, + nbytes); + return TRANSCRIBE_ERR_GGUF; + } + continue; } - // ggml_backend_tensor_set is the right call regardless of - // backend: on host buffers (CPU + Metal unified memory on - // Apple Silicon) it's a memcpy; on discrete GPUs it does - // the upload. - ggml_backend_tensor_set(t, staging.data(), 0, nbytes); + const size_t chunk = std::min(nbytes, k_stage_chunk); + if (staging.size() < chunk) { + staging.resize(chunk); + } + for (size_t done = 0; done < nbytes;) { + const size_t n = std::min(nbytes - done, staging.size()); + fin.read(reinterpret_cast(staging.data()), static_cast(n)); + if (!fin) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: short read for tensor \"%s\" (%zu bytes)", error_tag, t->name, + nbytes); + return TRANSCRIBE_ERR_GGUF; + } + ggml_backend_tensor_set(t, staging.data(), done, n); + done += n; + } } return TRANSCRIBE_OK; diff --git a/src/transcribe-load-common.h b/src/transcribe-load-common.h index 4d3eb7aa7..86002502d 100644 --- a/src/transcribe-load-common.h +++ b/src/transcribe-load-common.h @@ -2,14 +2,16 @@ // // INTERNAL, C++17. Not part of the public ABI. // -// Every per-family load() walks the same three steps after building the +// Every per-family load() walks the same two steps after building the // GGUF catalog: // -// 1. Resolve a BackendPlan from transcribe_model_load_params::backend. -// 2. Allocate a backend buffer for every tensor in ctx_meta on the plan's -// primary backend. -// 3. Stream each tensor's bytes from the GGUF data section into its -// backend slot via ggml_backend_tensor_set. +// 1. Resolve a BackendPlan from transcribe_model_load_params::backend +// (init_backends). +// 2. Give every tensor in ctx_meta storage on the plan's primary backend +// and fill it from the GGUF data section (load_weights). Where the +// device can use host memory directly the tensors point into a +// read-only memory map of the file; otherwise their bytes are copied +// into a backend buffer. // // These functions hoist that scaffolding; per-family load() still owns the // model, params validation, weights catalog, fusion, and destructor wiring. @@ -17,6 +19,7 @@ #pragma once #include "transcribe-backend.h" +#include "transcribe-mmap.h" #include "transcribe.h" #include @@ -79,11 +82,103 @@ transcribe_status init_backends(transcribe_backend_request requested, const char * error_tag, BackendPlan & out); -// Stream every tensor in `ctx_meta` from the GGUF data section on -// disk into its already-allocated backend buffer slot via -// ggml_backend_tensor_set. The backend buffer must already be -// allocated (typically via ggml_backend_alloc_ctx_tensors) before -// calling this. +// Storage behind a model's GGUF-backed weight tensors (its ctx_meta). +// +// Mapped tensors point straight into a read-only MappedFile of the GGUF, +// wrapped as a backend buffer (`mapped_buffer`). Every other tensor lives in +// an ordinary backend buffer filled from the file (`copy_buffer`). With +// mapping off, unavailable, or impossible for every tensor, copy_buffer +// holds them all: the classic path. +// +// Mapped weights are READ-ONLY: writing one (ggml_backend_tensor_set, an +// in-place graph op, a load-time fix-up) crashes. Load-time transforms must +// write their results into separate tensors in their own buffer, as +// fuse_batch_norm and promote_conv_pw_f16_to_f32_on_cpu do. A family should +// never write a ctx_meta tensor after load anyway: weights are shared by +// every session on the model. +class WeightStorage { + public: + WeightStorage() = default; + + ~WeightStorage() { reset(); } + + WeightStorage(const WeightStorage &) = delete; + WeightStorage & operator=(const WeightStorage &) = delete; + WeightStorage(WeightStorage &&) = delete; + WeightStorage & operator=(WeightStorage &&) = delete; + + // Free both buffers, then unmap. Call it where the family used to free + // its weight buffer (before the plan's backends); the destructor calls it + // again harmlessly. + void reset() noexcept; + + // True when `t`'s data lives in the file mapping. + bool is_mapped(const ggml_tensor * t) const noexcept; + + private: + friend transcribe_status allocate_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out); + friend transcribe_status allocate_copied_weights(ggml_context * ctx_meta, + const BackendPlan & plan, + const char * error_tag, + WeightStorage & out); + + MappedFile file_; + ggml_backend_buffer_t mapped_buffer_ = nullptr; + ggml_backend_buffer_t copy_buffer_ = nullptr; +}; + +// Whether a load may memory-map weights: false when params->no_mmap is set +// or the TRANSCRIBE_NO_MMAP environment flag is on. `params` may be NULL +// (defaults); otherwise it must be the full-size view that +// transcribe_model_load_file passes to the family loaders. +bool mmap_allowed(const transcribe_model_load_params * params); + +// Give every tensor in `ctx_meta` storage on plan.primary, without filling +// the copied ones. +// +// When mmap_allowed(params), the platform can map files, and the primary +// device can wrap host memory as a buffer (ggml caps buffer_from_host_ptr + +// mmap_support: the CPU everywhere and Apple-silicon Metal), the GGUF at +// `path` is mapped read-only and every tensor whose file bytes are usable +// as-is (same type and size as in the file, inside it, and aligned for the +// device) points into it. The rest, and everything on other devices (Vulkan, +// CUDA, ROCm upload into device memory), goes into an ordinary buffer. +// Mapping failures are not errors: they log and fall back to copying. +// +// Follow with stream_tensor_data(..., &out) (or a family streamer that skips +// out.is_mapped(t)) to fill the copied tensors. load_weights does both. +// +// Returns TRANSCRIBE_OK, TRANSCRIBE_ERR_OOM when the copy buffer cannot be +// allocated, or TRANSCRIBE_ERR_BACKEND when the device rejects a mapped +// tensor. On error `out` may hold partial storage; the caller discards the +// whole model, whose destructor resets it. +transcribe_status allocate_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out); + +// Allocate every still-unallocated tensor in `ctx_meta` in an ordinary +// buffer on plan.primary (no mapping). For loaders whose weights are not +// GGUF file bytes (legacy whisper.cpp / Silero .bin files). +transcribe_status allocate_copied_weights(ggml_context * ctx_meta, + const BackendPlan & plan, + const char * error_tag, + WeightStorage & out); + +// Stream tensor data from the GGUF data section on disk into +// already-allocated backend buffer slots: read straight into host buffers, +// uploaded through ggml_backend_tensor_set in bounded (16 MiB) chunks +// otherwise. Tensors that `skip` maps are left alone (they already hold the +// file bytes); with skip == nullptr every tensor in ctx_meta is streamed. // // path: filesystem path to the GGUF file, reopened fresh via // std::ifstream. Stable for the life of the call. @@ -98,10 +193,22 @@ transcribe_status init_backends(transcribe_backend_request requested, // I/O or tensor-not-found failure. On failure ctx_meta's tensors // may be partially populated; the caller should discard the // whole model state on error. -transcribe_status stream_tensor_data(const std::string & path, - const gguf_context * gguf_data, - ggml_context * ctx_meta, - const char * error_tag); +transcribe_status stream_tensor_data(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const char * error_tag, + const WeightStorage * skip = nullptr); + +// allocate_weights + stream_tensor_data: the weight load every GGUF family +// does after init_backends. Same contract as allocate_weights, plus +// TRANSCRIBE_ERR_GGUF from the stream. +transcribe_status load_weights(const std::string & path, + const gguf_context * gguf_data, + ggml_context * ctx_meta, + const BackendPlan & plan, + const transcribe_model_load_params * params, + const char * error_tag, + WeightStorage & out); // A single F16 → F32 promotion target: `dst_slot` is a pointer into a // family weights struct (e.g. &block.conv_pw1_w) that currently holds diff --git a/src/transcribe-mmap.cpp b/src/transcribe-mmap.cpp new file mode 100644 index 000000000..c13fa9baa --- /dev/null +++ b/src/transcribe-mmap.cpp @@ -0,0 +1,248 @@ +// transcribe-mmap.cpp - read-only memory map of a model file. +// +// See transcribe-mmap.h. The mapping follows llama.cpp's llama_mmap +// (MAP_SHARED + PROT_READ; CreateFileMapping + MapViewOfFile). Once mapped, +// the whole file is read in the background: F_RDADVISE on Apple, +// POSIX_FADV_SEQUENTIAL + POSIX_FADV_WILLNEED on Linux/Android, +// PrefetchVirtualMemory on Windows. On a cold page cache that keeps the first +// run as fast as the copying load's sequential read (MADV_WILLNEED was +// measurably slower on macOS). Never MADV_RANDOM: every family reads +// essentially all of its weights each run, so readahead is what keeps a cold +// start fast. No MAP_POPULATE: it blocks the load on the read and fills every +// page-table entry up front, which costs time even on a warm cache. + +#include "transcribe-mmap.h" + +#include "transcribe-path.h" + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# define TRANSCRIBE_MMAP_WIN32 1 +#elif defined(__unix__) || defined(__APPLE__) +# include +# include +# include +# include +# define TRANSCRIBE_MMAP_POSIX 1 +#endif + +namespace transcribe { + +namespace { + +void set_why(std::string * why, const char * what, const char * detail) { + if (why != nullptr) { + *why = what; + if (detail != nullptr && detail[0] != '\0') { + *why += ": "; + *why += detail; + } + } +} + +#if defined(TRANSCRIBE_MMAP_WIN32) +void set_why_win32(std::string * why, const char * what) { + char buf[32]; + std::snprintf(buf, sizeof(buf), "error %lu", static_cast(GetLastError())); + set_why(why, what, buf); +} + +// PrefetchVirtualMemory is Windows 8+; resolve it at runtime so the library +// still loads on anything older (where prefetching is a no-op). +struct PrefetchRange { + void * VirtualAddress; + SIZE_T NumberOfBytes; +}; + +typedef BOOL(WINAPI * PrefetchFn)(HANDLE, ULONG_PTR, PrefetchRange *, ULONG); + +PrefetchFn resolve_prefetch() { + HMODULE k32 = GetModuleHandleW(L"kernel32.dll"); + if (k32 == nullptr) { + return nullptr; + } + return reinterpret_cast(reinterpret_cast(GetProcAddress(k32, "PrefetchVirtualMemory"))); +} + +// Start reading [addr, addr + size) in the background. +void prefetch_view(void * addr, size_t size) noexcept { + static const PrefetchFn prefetch_fn = resolve_prefetch(); + if (prefetch_fn == nullptr) { + return; + } + PrefetchRange range{ addr, size }; + (void) prefetch_fn(GetCurrentProcess(), 1, &range, 0); +} +#endif + +} // namespace + +bool MappedFile::supported() noexcept { +#if defined(TRANSCRIBE_MMAP_WIN32) || defined(TRANSCRIBE_MMAP_POSIX) + return true; +#else + return false; +#endif +} + +#if defined(TRANSCRIBE_MMAP_POSIX) + +bool MappedFile::open(const std::string & path, std::string * why) { + reset(); + + const int fd = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) { + set_why(why, "open failed", std::strerror(errno)); + return false; + } + + struct stat sb{}; + + if (::fstat(fd, &sb) != 0) { + set_why(why, "fstat failed", std::strerror(errno)); + ::close(fd); + return false; + } + if (!S_ISREG(sb.st_mode) || sb.st_size <= 0) { + set_why(why, "not a non-empty regular file", nullptr); + ::close(fd); + return false; + } + if (static_cast(sb.st_size) > static_cast(SIZE_MAX)) { + set_why(why, "file larger than the address space", nullptr); + ::close(fd); + return false; + } + const size_t size = static_cast(sb.st_size); + + void * addr = ::mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + set_why(why, "mmap failed", std::strerror(errno)); + ::close(fd); + return false; + } + + // Start reading the file in the background. Hints: failure is harmless. +# if defined(__APPLE__) + // radvisory::ra_count is an int, so advise in chunks below 2 GiB. + const off_t chunk = off_t(1) << 30; + for (off_t off = 0; off < static_cast(size); off += chunk) { + struct radvisory ra{}; + ra.ra_offset = off; + ra.ra_count = static_cast(std::min(chunk, static_cast(size) - off)); + (void) ::fcntl(fd, F_RDADVISE, &ra); + } +# elif defined(__linux__) + (void) ::posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void) ::posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); +# else + (void) ::madvise(addr, size, MADV_WILLNEED); +# endif + ::close(fd); // the mapping holds its own reference to the file + + addr_ = addr; + size_ = size; + return true; +} + +void MappedFile::reset() noexcept { + if (addr_ != nullptr) { + (void) ::munmap(addr_, size_); + } + addr_ = nullptr; + size_ = 0; +} + +#elif defined(TRANSCRIBE_MMAP_WIN32) + +bool MappedFile::open(const std::string & path, std::string * why) { + reset(); + + // FILE_SHARE_DELETE lets a model manager rename the file away while it is + // loaded; no FILE_SHARE_WRITE, so nobody can rewrite it underneath us. + const std::filesystem::path p = path_from_utf8(path); + HANDLE file = CreateFileW(p.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) { + set_why_win32(why, "CreateFileW failed"); + return false; + } + + LARGE_INTEGER file_size{}; + if (!GetFileSizeEx(file, &file_size)) { + set_why_win32(why, "GetFileSizeEx failed"); + CloseHandle(file); + return false; + } + if (file_size.QuadPart <= 0) { + set_why(why, "empty file", nullptr); + CloseHandle(file); + return false; + } + if (static_cast(file_size.QuadPart) > static_cast(SIZE_MAX)) { + set_why(why, "file larger than the address space", nullptr); + CloseHandle(file); + return false; + } + + HANDLE mapping = CreateFileMappingW(file, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (mapping == nullptr) { + set_why_win32(why, "CreateFileMappingW failed"); + CloseHandle(file); + return false; + } + // The section keeps the file referenced and the view keeps the section + // alive, so neither handle is needed once the view exists. + void * addr = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + if (addr == nullptr) { + set_why_win32(why, "MapViewOfFile failed"); + } + CloseHandle(mapping); + CloseHandle(file); + if (addr == nullptr) { + return false; + } + + addr_ = addr; + size_ = static_cast(file_size.QuadPart); + prefetch_view(addr_, size_); + return true; +} + +void MappedFile::reset() noexcept { + if (addr_ != nullptr) { + (void) UnmapViewOfFile(addr_); + } + addr_ = nullptr; + size_ = 0; +} + +#else + +bool MappedFile::open(const std::string & path, std::string * why) { + (void) path; + reset(); + set_why(why, "memory mapping is not supported on this platform", nullptr); + return false; +} + +void MappedFile::reset() noexcept { + addr_ = nullptr; + size_ = 0; +} + +#endif + +} // namespace transcribe diff --git a/src/transcribe-mmap.h b/src/transcribe-mmap.h new file mode 100644 index 000000000..7944b2fea --- /dev/null +++ b/src/transcribe-mmap.h @@ -0,0 +1,65 @@ +// transcribe-mmap.h - read-only memory map of a model file. +// +// INTERNAL, C++17. Not part of the public ABI. +// +// MappedFile maps a whole file read-only so model weights can be served +// straight from the page cache instead of being copied into anonymous memory. +// File-backed pages are clean and reclaimable: they do not count against the +// Android low-memory killer's anonymous budget or the iOS/macOS footprint, and +// several models loaded from one file share them. +// +// POSIX: open + mmap(PROT_READ, MAP_SHARED). +// Windows: CreateFileW + CreateFileMappingW(PAGE_READONLY) + +// MapViewOfFile(FILE_MAP_READ). +// +// The mapping is read-only: a write through it crashes. And it reflects the +// file: truncating or rewriting the file in place while it is mapped turns a +// later access into SIGBUS (POSIX) or EXCEPTION_IN_PAGE_ERROR (Windows). +// Replace a model file by writing a new one and renaming it over the old one. +// On Windows the mapped file cannot be opened for writing or truncated until +// the mapping is released. + +#pragma once + +#include +#include + +namespace transcribe { + +class MappedFile { + public: + MappedFile() = default; + + ~MappedFile() { reset(); } + + MappedFile(const MappedFile &) = delete; + MappedFile & operator=(const MappedFile &) = delete; + MappedFile(MappedFile &&) = delete; + MappedFile & operator=(MappedFile &&) = delete; + + // True when this build can map files at all. + static bool supported() noexcept; + + // Map the file at `path` (UTF-8) read-only and start reading it into the + // page cache in the background (the weights are read either way: mapped + // tensors fault in from it, copied ones are streamed from it). Replaces + // any previous mapping. On failure returns false, leaves the object empty + // and, when `why` is non-null, stores a short reason for the log. + bool open(const std::string & path, std::string * why); + + // Unmap. Every tensor pointing into the mapping must be dead by now. + void reset() noexcept; + + // Page-aligned base of the mapping (MapViewOfFile: allocation-granularity + // aligned). Non-const because ggml's host-pointer buffers take void *; + // the pages are still read-only. + void * addr() const noexcept { return addr_; } + + size_t size() const noexcept { return size_; } + + private: + void * addr_ = nullptr; + size_t size_ = 0; +}; + +} // namespace transcribe diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 0525619c3..88cda4cd1 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -1024,6 +1024,21 @@ static transcribe_status transcribe_model_load_file_impl(const char * if (const auto st = check_input_struct_size(params->struct_size, k_min_model_params_size); st != TRANSCRIBE_OK) { return st; } + // Full-size view for the family loaders: defaults first, then only the + // prefix the caller's struct_size covers, so every trailing field is + // readable (no_mmap reads false for a caller built before it existed). + // The bool is re-derived from its raw byte: a C caller can store any + // value there, and loading a non-0/1 byte as bool is UB. + struct transcribe_model_load_params params_view; + transcribe_model_load_params_init(¶ms_view); + std::memcpy(¶ms_view, params, + static_cast(std::min(params->struct_size, sizeof(params_view)))); + { + unsigned char no_mmap_raw = 0; + std::memcpy(&no_mmap_raw, ¶ms_view.no_mmap, sizeof(no_mmap_raw)); + params_view.no_mmap = no_mmap_raw != 0; + } + params = ¶ms_view; // device is validated where the registry is available, in // load_common::init_backends. NULL means automatic selection; a non-NULL diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8ce587f21..b29e6cd29 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -545,7 +545,8 @@ set(TRANSCRIBE_FIXTURE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_ecapa_tdnn_q8_0_as_f16.gguf ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_ecapa_tdnn_bad_hop0.gguf ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_ecapa_tdnn_bad_win_gt_fft.gguf - ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_silero_vad.gguf) + ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_silero_vad.gguf + ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/arch_silero_vad_misaligned.gguf) find_program(TRANSCRIBE_UV uv) @@ -819,6 +820,22 @@ if(TRANSCRIBE_UV) add_test(NAME transcribe_silero_vad_smoke COMMAND transcribe_silero_vad_smoke) endif() +# ----------------------------------------------------------------------------- +# Memory-mapped weights (no_mmap / TRANSCRIBE_NO_MMAP) across the fixture +# families: mapping decision, mapped-vs-copied equivalence, misaligned and +# truncated files, two models on one file, Metal when present. +# ----------------------------------------------------------------------------- + +if(TRANSCRIBE_UV) + add_executable(transcribe_weights_mmap_smoke weights_mmap_smoke.cpp) + target_link_libraries(transcribe_weights_mmap_smoke PRIVATE transcribe) + target_compile_definitions(transcribe_weights_mmap_smoke PRIVATE + "TRANSCRIBE_TEST_FIXTURES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/fixtures\"") + transcribe_apply_warnings(transcribe_weights_mmap_smoke) + add_dependencies(transcribe_weights_mmap_smoke fixtures) + add_test(NAME transcribe_weights_mmap_smoke COMMAND transcribe_weights_mmap_smoke) +endif() + # ----------------------------------------------------------------------------- # Real-model gated test (developer-local, never CI) # ----------------------------------------------------------------------------- diff --git a/tests/api_smoke.c b/tests/api_smoke.c index ed06e4403..d7419ecf6 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -237,6 +238,7 @@ static void test_init_macros(void) { CHECK(mp_macro.struct_size == sizeof(struct transcribe_model_load_params)); CHECK(mp_macro.backend == TRANSCRIBE_BACKEND_AUTO); CHECK(mp_macro.device == NULL); + CHECK(mp_macro.no_mmap == false); struct transcribe_session_params cp_macro; transcribe_session_params_init(&cp_macro); @@ -454,6 +456,16 @@ static void test_load_invalid(void) { TRANSCRIBE_ERR_FILE_NOT_FOUND); CHECK(m == NULL); + /* A caller built before no_mmap existed passes a shorter struct_size; + * it is accepted and the missing field reads as its default. */ + struct transcribe_model_load_params mp_old; + transcribe_model_load_params_init(&mp_old); + mp_old.struct_size = offsetof(struct transcribe_model_load_params, no_mmap); + m = (struct transcribe_model *) 0xdeadbeef; + CHECK(transcribe_model_load_file("/__transcribe_smoke_does_not_exist__.gguf", &mp_old, &m) == + TRANSCRIBE_ERR_FILE_NOT_FOUND); + CHECK(m == NULL); + /* Otherwise-valid call against a path that does not exist on disk * -> FILE_NOT_FOUND. */ m = (struct transcribe_model *) 0xdeadbeef; diff --git a/tests/fixtures/make_gguf_fixtures.py b/tests/fixtures/make_gguf_fixtures.py index f74ff1188..efb69d105 100644 --- a/tests/fixtures/make_gguf_fixtures.py +++ b/tests/fixtures/make_gguf_fixtures.py @@ -323,6 +323,8 @@ def _build_full_gguf( magic: bytes, kv_blobs: list[bytes], tensors: list[Tensor], + alignment: int = GGUF_DEFAULT_ALIGNMENT, + layout: dict | None = None, ) -> bytes: """Assemble a complete GGUF file with KV pairs AND tensor data. @@ -337,7 +339,10 @@ def _build_full_gguf( Each tensor's data offset within the data blob is aligned to GGUF_DEFAULT_ALIGNMENT. ggml_get_alignment defaults to 32 when general.alignment is absent (we don't write it), so emitting at the - same alignment matches what the loader expects. + same alignment matches what the loader expects. A caller that passes a + different `alignment` must also put a matching general.alignment KV in + kv_blobs. `layout`, when given, receives the file offset of the data + section under "data_offset". """ body = bytearray() body += magic @@ -355,18 +360,20 @@ def _build_full_gguf( offsets: list[int] = [] cursor = 0 for t in tensors: - cursor = _ggml_pad(cursor) + cursor = _ggml_pad(cursor, alignment) offsets.append(cursor) cursor += len(t.data) - data_blob_size = _ggml_pad(cursor) + data_blob_size = _ggml_pad(cursor, alignment) for t, off in zip(tensors, offsets): body += _pack_tensor_info(t, off) # Pad from end of tensor info section to start of data blob. body_len = len(body) - data_blob_start = _ggml_pad(body_len) + data_blob_start = _ggml_pad(body_len, alignment) body += b"\x00" * (data_blob_start - body_len) + if layout is not None: + layout["data_offset"] = data_blob_start # Emit each tensor's bytes at its declared offset within the data # blob. The data_blob_start above already aligned us to a 32-byte @@ -1557,7 +1564,11 @@ def _ecapa_tdnn_gguf(codes: list[str], names: list[str], aliases: list[str], q8_ SILERO_HIDDEN = 128 -def _silero_vad_gguf() -> bytes: +def _silero_vad_gguf( + alignment: int = GGUF_DEFAULT_ALIGNMENT, + extra_kvs: list[bytes] | None = None, + layout: dict | None = None, +) -> bytes: rng = _random.Random(1) tensors: list[Tensor] = [] @@ -1604,11 +1615,39 @@ def uniform(name: str, ne: list[int], fan_in: int) -> None: _pack_kv_array_int32("stt.silero_vad.encoder_channels", SILERO_CHANNELS), _pack_kv_array_int32("stt.silero_vad.encoder_strides", SILERO_STRIDES), _pack_kv_uint32("stt.silero_vad.lstm_hidden", SILERO_HIDDEN), - ], + ] + + (extra_kvs or []), tensors, + alignment, + layout, ) +def _silero_vad_misaligned_gguf() -> bytes: + """The toy silero_vad model as a valid GGUF that cannot be memory-mapped. + + general.alignment = 16 and a padding KV sized so the data section starts + 16 bytes past a 32-byte boundary: every tensor lands misaligned for + ggml's CPU buffers (32-byte TENSOR_ALIGNMENT, which + ggml_backend_cpu_buffer_from_ptr asserts). The loader must fall back to + copying the weights and produce the same probabilities as + arch_silero_vad.gguf. + """ + for pad in range(64): + layout: dict = {} + data = _silero_vad_gguf( + alignment=16, + extra_kvs=[ + _pack_kv_uint32("general.alignment", 16), + _pack_kv_string("test.pad", "x" * pad), + ], + layout=layout, + ) + if layout["data_offset"] % 32 == 16: + return data + raise RuntimeError("no padding gives a 16-mod-32 data offset") + + def _write(path: Path, data: bytes) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) @@ -2042,6 +2081,8 @@ def emit_fixtures(out_dir: Path) -> None: # silero_vad (VAD role): the real geometry with seeded weights. _write(out_dir / "arch_silero_vad.gguf", _silero_vad_gguf()) + # The same model with its tensors misaligned for memory mapping. + _write(out_dir / "arch_silero_vad_misaligned.gguf", _silero_vad_misaligned_gguf()) def main(argv: list[str]) -> int: diff --git a/tests/weights_mmap_smoke.cpp b/tests/weights_mmap_smoke.cpp new file mode 100644 index 000000000..b7dbfa189 --- /dev/null +++ b/tests/weights_mmap_smoke.cpp @@ -0,0 +1,464 @@ +// weights_mmap_smoke.cpp - memory-mapped model weights +// (transcribe_model_load_params::no_mmap, TRANSCRIBE_NO_MMAP). +// +// Fixture-driven, runs in CI. Covers: +// - every fixture family that loads on the CPU maps its weights by default +// and copies them with no_mmap or TRANSCRIBE_NO_MMAP (read off the load +// log). Mapped weights are read-only, so a family that writes into its +// GGUF weights at load (as a fusion done in place would) crashes here; +// - mapped and copied models give bit-identical results (silero VAD +// probabilities; ecapa langid candidates, including the Q8_0 fixture +// whose widened tensors are copied while the rest are mapped); +// - a caller built before no_mmap existed (shorter struct_size) gets the +// default; +// - a valid GGUF whose tensors are misaligned for the device loads by +// copying, with the same results, instead of aborting; +// - a truncated GGUF fails to load cleanly; +// - two models mapped from one file coexist and free in either order; +// - a model under a non-ASCII path maps (Windows wide-path conversion); +// - with Metal available, the same holds for Metal-mapped weights. + +#include "transcribe.h" +#include "transcribe/langid.h" +#include "transcribe/vad.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +std::string fixture(const char * name) { + return std::string(TRANSCRIBE_TEST_FIXTURES_DIR) + "/" + name; +} + +// ---- log capture ------------------------------------------------------------ + +std::mutex g_log_mu; +std::vector g_log; + +void capture_log(transcribe_log_level level, const char * msg, void * /*userdata*/) { + if (msg == nullptr || level == TRANSCRIBE_LOG_LEVEL_DEBUG || level == TRANSCRIBE_LOG_LEVEL_CONT) { + return; + } + std::lock_guard lock(g_log_mu); + g_log.emplace_back(msg); +} + +void clear_log() { + std::lock_guard lock(g_log_mu); + g_log.clear(); +} + +bool log_has(const char * needle) { + std::lock_guard lock(g_log_mu); + for (const auto & line : g_log) { + if (line.find(needle) != std::string::npos) { + return true; + } + } + return false; +} + +void set_env(const char * name, const char * value) { +#if defined(_WIN32) + _putenv_s(name, value != nullptr ? value : ""); +#else + if (value != nullptr) { + setenv(name, value, 1); + } else { + unsetenv(name); + } +#endif +} + +// ---- loading ---------------------------------------------------------------- + +enum class Mode { + Default, // params defaults: map where possible + NoMmapParam, // no_mmap = true + NoMmapEnv, // defaults + TRANSCRIBE_NO_MMAP=1 + OldStructSize, // struct_size of a caller built before no_mmap existed +}; + +const char * mode_name(Mode mode) { + switch (mode) { + case Mode::Default: + return "default"; + case Mode::NoMmapParam: + return "no_mmap"; + case Mode::NoMmapEnv: + return "TRANSCRIBE_NO_MMAP"; + case Mode::OldStructSize: + return "old struct_size"; + } + return "?"; +} + +bool expect_mapped(Mode mode) { + return mode == Mode::Default || mode == Mode::OldStructSize; +} + +// Load `path` and clear+fill the log with what the load said. +transcribe_model * load(const std::string & path, + Mode mode, + transcribe_backend_request backend = TRANSCRIBE_BACKEND_CPU, + transcribe_status * st_out = nullptr) { + transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = backend; + if (mode == Mode::NoMmapParam) { + mp.no_mmap = true; + } + if (mode == Mode::OldStructSize) { + // A garbage byte where the field would be must not be read. + mp.no_mmap = true; + mp.struct_size = offsetof(transcribe_model_load_params, no_mmap); + } + if (mode == Mode::NoMmapEnv) { + set_env("TRANSCRIBE_NO_MMAP", "1"); + } + clear_log(); + transcribe_model * m = nullptr; + const transcribe_status st = transcribe_model_load_file(path.c_str(), &mp, &m); + if (mode == Mode::NoMmapEnv) { + set_env("TRANSCRIBE_NO_MMAP", nullptr); + } + if (st_out != nullptr) { + *st_out = st; + } + return m; +} + +// Load and check the mapping decision the log reports. +transcribe_model * load_checked(const std::string & path, + Mode mode, + const char * tag, + transcribe_backend_request backend = TRANSCRIBE_BACKEND_CPU) { + transcribe_status st = TRANSCRIBE_OK; + transcribe_model * m = load(path, mode, backend, &st); + if (st != TRANSCRIBE_OK || m == nullptr) { + std::fprintf(stderr, "FAIL load %s (%s): %s\n", path.c_str(), mode_name(mode), transcribe_status_string(st)); + ++g_failures; + return nullptr; + } + const std::string mapped = std::string(tag) + ": weights memory-mapped"; + const std::string copied = std::string(tag) + ": weights copied"; + if (expect_mapped(mode)) { + if (!log_has(mapped.c_str())) { + std::fprintf(stderr, "FAIL %s (%s): expected \"%s\" in the load log\n", path.c_str(), mode_name(mode), + mapped.c_str()); + ++g_failures; + } + } else { + if (!log_has(copied.c_str()) || !log_has("(memory mapping disabled)")) { + std::fprintf(stderr, "FAIL %s (%s): expected \"%s ... (memory mapping disabled)\" in the load log\n", + path.c_str(), mode_name(mode), copied.c_str()); + ++g_failures; + } + } + return m; +} + +// ---- workloads ---------------------------------------------------------------- + +// Deterministic test signal: bursts of a chirp over low noise. +std::vector signal(int n) { + std::vector v(static_cast(n)); + uint32_t seed = 12345; + for (int i = 0; i < n; ++i) { + seed = seed * 1664525u + 1013904223u; + const float noise = (static_cast(seed >> 8) / 16777216.0f - 0.5f) * 0.01f; + const float t = static_cast(i) / 16000.0f; + const bool burst = (i / 8000) % 3 != 0; + v[static_cast(i)] = noise + (burst ? 0.3f * std::sin(6.2831853f * (200.0f + 400.0f * t) * t) : 0.0f); + } + return v; +} + +std::vector vad_probs(transcribe_model * m) { + std::vector out; + if (m == nullptr) { + return out; + } + transcribe_vad_session_params sp; + transcribe_vad_session_params_init(&sp); + transcribe_vad_session * s = nullptr; + CHECK(transcribe_vad_session_init(m, &sp, &s) == TRANSCRIBE_OK); + if (s == nullptr) { + return out; + } + const std::vector pcm = signal(3 * 16000); + CHECK(transcribe_vad_run(s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + transcribe_vad_result r; + transcribe_vad_result_init(&r); + CHECK(transcribe_vad_get_result(s, &r) == TRANSCRIBE_OK); + const float * p = transcribe_vad_probs(s); + if (p != nullptr && r.n_probs > 0) { + out.assign(p, p + r.n_probs); + } + transcribe_vad_session_free(s); + CHECK(!out.empty()); + return out; +} + +struct LangidOut { + std::vector index; + std::vector p; + std::vector logit; +}; + +LangidOut langid(transcribe_model * m) { + LangidOut out; + if (m == nullptr) { + return out; + } + transcribe_langid_session_params sp; + transcribe_langid_session_params_init(&sp); + sp.n_threads = 2; + transcribe_langid_session * s = nullptr; + CHECK(transcribe_langid_session_init(m, &sp, &s) == TRANSCRIBE_OK); + if (s == nullptr) { + return out; + } + const std::vector pcm = signal(2 * 16000); + CHECK(transcribe_langid_run(s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + transcribe_langid_result r; + transcribe_langid_result_init(&r); + CHECK(transcribe_langid_get_result(s, &r) == TRANSCRIBE_OK); + for (int i = 0; i < r.n_candidates; ++i) { + transcribe_langid_candidate c; + transcribe_langid_candidate_init(&c); + CHECK(transcribe_langid_get_candidate(s, i, &c) == TRANSCRIBE_OK); + out.index.push_back(c.index); + out.p.push_back(c.p); + out.logit.push_back(c.logit); + } + transcribe_langid_session_free(s); + CHECK(!out.index.empty()); + return out; +} + +bool same_floats(const std::vector & a, const std::vector & b) { + return a.size() == b.size() && (a.empty() || std::memcmp(a.data(), b.data(), a.size() * sizeof(float)) == 0); +} + +bool same_langid(const LangidOut & a, const LangidOut & b) { + return a.index == b.index && same_floats(a.p, b.p) && same_floats(a.logit, b.logit); +} + +// ---- tests -------------------------------------------------------------------- + +const Mode k_modes[] = { Mode::Default, Mode::NoMmapParam, Mode::NoMmapEnv, Mode::OldStructSize }; + +// Load-only families: mapping must not disturb load (and nothing may write +// into a mapped weight while loading). +void test_load_only_families() { + struct Fixture { + const char * file; + const char * tag; + }; + + const Fixture fixtures[] = { + { "arch_cohere_minimal.gguf", "cohere" }, + { "arch_qwen3_asr_minimal.gguf", "qwen3_asr" }, + { "arch_granite5_ctc_minimal.gguf", "granite5_ctc" }, + }; + for (const Fixture & f : fixtures) { + for (Mode mode : k_modes) { + transcribe_model * m = load_checked(fixture(f.file), mode, f.tag); + transcribe_model_free(m); + } + } +} + +void test_silero_equivalence() { + std::vector baseline; + for (Mode mode : k_modes) { + transcribe_model * m = load_checked(fixture("arch_silero_vad.gguf"), mode, "silero_vad"); + std::vector probs = vad_probs(m); + if (baseline.empty()) { + baseline = probs; + } else if (!same_floats(probs, baseline)) { + std::fprintf(stderr, "FAIL silero_vad (%s): probabilities differ from the mapped run\n", mode_name(mode)); + ++g_failures; + } + transcribe_model_free(m); + } +} + +void test_ecapa_equivalence() { + // minimal: every tensor mapped. q8_0: the Q8_0 tensors are widened to + // F16 at load, so they are copied while the rest stay mapped. + for (const char * file : { "arch_ecapa_tdnn_minimal.gguf", "arch_ecapa_tdnn_q8_0.gguf" }) { + LangidOut baseline; + bool have = false; + for (Mode mode : k_modes) { + transcribe_model * m = load_checked(fixture(file), mode, "ecapa_tdnn"); + if (expect_mapped(mode) && std::strstr(file, "q8_0") != nullptr) { + CHECK(log_has("copied (converted at load)")); + } + const LangidOut out = langid(m); + if (!have) { + baseline = out; + have = true; + } else if (!same_langid(out, baseline)) { + std::fprintf(stderr, "FAIL %s (%s): langid differs from the mapped run\n", file, mode_name(mode)); + ++g_failures; + } + transcribe_model_free(m); + } + } +} + +// Valid GGUF, tensors misaligned for the device: copied, same results, no +// abort (ggml_backend_cpu_buffer_from_ptr asserts on an unaligned base). +void test_misaligned_file() { + transcribe_model * ref = load(fixture("arch_silero_vad.gguf"), Mode::NoMmapParam); + const auto want = vad_probs(ref); + transcribe_model_free(ref); + + transcribe_status st = TRANSCRIBE_OK; + transcribe_model * m = load(fixture("arch_silero_vad_misaligned.gguf"), Mode::Default, TRANSCRIBE_BACKEND_CPU, &st); + CHECK(st == TRANSCRIBE_OK && m != nullptr); + CHECK(log_has("silero_vad: weights copied")); + CHECK(log_has("misaligned")); + CHECK(same_floats(vad_probs(m), want)); + transcribe_model_free(m); +} + +std::string temp_path(const char * stem) { + std::error_code ec; + auto dir = std::filesystem::temp_directory_path(ec); + if (ec) { + dir = std::filesystem::path("."); + } + return (dir / (std::string("transcribe_weights_mmap_") + stem + "_" + std::to_string(std::rand()) + ".gguf")) + .string(); +} + +// A file cut off inside the data section fails to load, in both modes. +void test_truncated_file() { + std::ifstream in(fixture("arch_silero_vad.gguf"), std::ios::binary); + std::vector bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + CHECK(bytes.size() > 4096); + const std::string path = temp_path("truncated"); + { + std::ofstream out(path, std::ios::binary); + out.write(bytes.data(), static_cast(bytes.size() / 2)); + } + for (Mode mode : { Mode::Default, Mode::NoMmapParam }) { + transcribe_status st = TRANSCRIBE_OK; + transcribe_model * m = load(path, mode, TRANSCRIBE_BACKEND_CPU, &st); + CHECK(st != TRANSCRIBE_OK); + CHECK(m == nullptr); + transcribe_model_free(m); + } + std::error_code ec; + std::filesystem::remove(path, ec); +} + +// Two models mapped from one file are independent; free order is free. +void test_two_models_one_file() { + const std::string path = fixture("arch_silero_vad.gguf"); + transcribe_model * a = load_checked(path, Mode::Default, "silero_vad"); + transcribe_model * b = load_checked(path, Mode::Default, "silero_vad"); + const auto pa = vad_probs(a); + const auto pb = vad_probs(b); + CHECK(same_floats(pa, pb)); + transcribe_model_free(a); + CHECK(same_floats(vad_probs(b), pb)); + transcribe_model_free(b); +} + +// A non-ASCII (UTF-8) path maps too: on Windows the mapping opens the file +// through the UTF-8 -> wide conversion, and a failure there would silently +// fall back to copying. The directory name is spelled in escaped UTF-8 bytes +// so the source stays ASCII. +void test_utf8_path() { + namespace fs = std::filesystem; + const char * k_name = "mmap-caf\xc3\xa9-\xe6\x97\xa5\xe6\x9c\xac"; + std::error_code ec; + fs::path dir = fs::temp_directory_path(ec); + if (ec) { + dir = fs::path("."); + } + dir /= fs::u8path(k_name); + fs::remove_all(dir, ec); + fs::create_directories(dir, ec); + CHECK(!ec); + const fs::path dst = dir / "arch_silero_vad.gguf"; + fs::copy_file(fs::u8path(fixture("arch_silero_vad.gguf")), dst, fs::copy_options::overwrite_existing, ec); + CHECK(!ec); + + transcribe_model * m = load_checked(dst.u8string(), Mode::Default, "silero_vad"); + CHECK(!vad_probs(m).empty()); + transcribe_model_free(m); + fs::remove_all(dir, ec); +} + +// Metal-mapped weights (Apple silicon): same decision log, same results as +// copied Metal weights. +void test_metal() { + if (!transcribe_backend_available(TRANSCRIBE_BACKEND_METAL)) { + std::fprintf(stderr, "weights_mmap_smoke: no Metal device; skipping the Metal case\n"); + return; + } + LangidOut baseline; + bool have = false; + for (Mode mode : { Mode::Default, Mode::NoMmapParam }) { + transcribe_model * m = + load_checked(fixture("arch_ecapa_tdnn_minimal.gguf"), mode, "ecapa_tdnn", TRANSCRIBE_BACKEND_METAL); + const LangidOut out = langid(m); + if (!have) { + baseline = out; + have = true; + } else { + CHECK(same_langid(out, baseline)); + } + transcribe_model_free(m); + } +} + +} // namespace + +int main() { + set_env("TRANSCRIBE_NO_MMAP", nullptr); + transcribe_log_set(capture_log, nullptr); + + test_load_only_families(); + test_silero_equivalence(); + test_ecapa_equivalence(); + test_misaligned_file(); + test_truncated_file(); + test_two_models_one_file(); + test_utf8_path(); + test_metal(); + + transcribe_log_set(nullptr, nullptr); + if (g_failures != 0) { + std::fprintf(stderr, "weights_mmap_smoke: %d failure(s)\n", g_failures); + return 1; + } + std::printf("weights_mmap_smoke: ok\n"); + return 0; +} diff --git a/tools/transcribe-bench/main.cpp b/tools/transcribe-bench/main.cpp index d4194c4b7..e9af4298c 100644 --- a/tools/transcribe-bench/main.cpp +++ b/tools/transcribe-bench/main.cpp @@ -47,7 +47,8 @@ struct bench_args { int n_threads = 0; bool quiet = false; transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int device_index = -1; // --device N: -1 = auto, >=0 = exact device + int device_index = -1; // --device N: -1 = auto, >=0 = exact device + bool no_mmap = false; // --no-mmap: copy weights instead of mapping the file // Passed through to transcribe_run_params::spec_k_drafts. -1 = family // default, 0 = spec decode off, > 0 = explicit draft length. Silently // ignored by families without supports_spec_decode. Set by @@ -73,6 +74,8 @@ void print_usage(const char * argv0) { " (BLAS/AMX) when the build includes them.\n" " --device N exact device index from transcribe-cli --list-devices,\n" " including 0 (default: automatic selection)\n" + " --no-mmap copy the weights into memory instead of memory-mapping\n" + " the model file (TRANSCRIBE_NO_MMAP=1 does the same)\n" " --spec-k-drafts N speculative-decode draft length on the offline\n" " path: -1 = family default, 0 = off, > 0 = K.\n" " Ignored by families without spec support.\n" @@ -200,6 +203,8 @@ bool parse_args(int argc, char ** argv, bench_args & out) { if (!parse_backend_kind(v, out.backend)) { return false; } + } else if (a == "--no-mmap") { + out.no_mmap = true; } else if (a == "--device") { auto v = need_val(i, "--device"); if (!v) { @@ -332,6 +337,7 @@ int main(int argc, char ** argv) { transcribe_model_load_params_init(&mp); mp.backend = args.backend; mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + mp.no_mmap = args.no_mmap; if (args.device_index >= 0 && mp.device == nullptr) { std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); return EXIT_FAILURE;