diff --git a/Makefile b/Makefile index a021214ab7b..de1a6912b96 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,7 @@ help: @echo " muse-glimmer-mlx - Build Muse Glimmer runners and worker with MLX backend" @echo " muse-glimmer-batching-mlx - Build batching Muse Glimmer runner and worker with MLX" @echo " muse-glimmer-batching-cuda - Build batching Muse Glimmer runner and worker with CUDA" + @echo " muse-glimmer-batching-native - Build batching Muse Glimmer runner and worker on the native ggml engine" @echo " qwen3_5_moe-cuda - Build Qwen3.5 MoE runner with CUDA backend" @echo " qwen3_5_moe-metal - Build Qwen3.5 MoE runner with Metal backend" @echo " qwen3_5_moe-mlx - Build Qwen3.5 MoE runner with MLX backend" @@ -562,6 +563,24 @@ muse-glimmer-batching-cuda: @echo " Runner: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_runner" @echo " Worker: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker" +.PHONY: muse-glimmer-batching-native +muse-glimmer-batching-native: + @echo "==> Building and installing ExecuTorch with the native ggml engine..." + cmake --preset llm-release -DEXECUTORCH_BUILD_NATIVE=ON \ + -DEXECUTORCH_BUILD_NATIVE_GGML=ON -DGGML_OPENMP=OFF + cmake --build cmake-out --target install \ + --parallel "$${CMAKE_BUILD_PARALLEL_LEVEL:-$$(sysctl -n hw.ncpu 2>/dev/null || nproc)}" + @echo "==> Configuring batching Muse Glimmer runners on the native engine..." + cd examples/models/muse-glimmer && cmake --preset muse-glimmer-native \ + -B "$(CURDIR)/cmake-out/examples/models/muse-glimmer-batching" + cmake --build "$(CURDIR)/cmake-out/examples/models/muse-glimmer-batching" \ + --target muse_glimmer_batching_runner muse_glimmer_batching_worker \ + --parallel "$${CMAKE_BUILD_PARALLEL_LEVEL:-$$(sysctl -n hw.ncpu 2>/dev/null || nproc)}" + @echo "" + @echo "Build complete!" + @echo " Runner: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_runner" + @echo " Worker: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker" + gemma4_31b-cuda: @echo "==> Building and installing ExecuTorch with CUDA..." cmake --workflow --preset llm-release-cuda diff --git a/examples/models/muse-glimmer/CMakeLists.txt b/examples/models/muse-glimmer/CMakeLists.txt index abcd30994d9..ffe03cca72c 100644 --- a/examples/models/muse-glimmer/CMakeLists.txt +++ b/examples/models/muse-glimmer/CMakeLists.txt @@ -50,7 +50,9 @@ list( extension_flat_tensor ) -# Backend: CUDA or MLX (exactly one required) +# Backend: CUDA or MLX for every runner. The batching runner can also serve on +# the native ggml engine, from an installation with +# EXECUTORCH_BUILD_NATIVE_GGML. if(EXECUTORCH_BUILD_CUDA) find_package(CUDAToolkit REQUIRED) list(APPEND link_libraries aoti_cuda_backend) @@ -60,13 +62,22 @@ elseif(TARGET mlxdelegate) list(APPEND link_libraries mlxdelegate mlx) executorch_target_link_options_shared_lib(mlxdelegate) add_compile_definitions(EXECUTORCH_BUILD_MLX) -else() - message(FATAL_ERROR "Set EXECUTORCH_BUILD_CUDA=ON or EXECUTORCH_BUILD_MLX=ON") +elseif(NOT TARGET native_ggml_engine) + message( + FATAL_ERROR + "Set EXECUTORCH_BUILD_CUDA=ON or EXECUTORCH_BUILD_MLX=ON, or use an ExecuTorch installation with EXECUTORCH_BUILD_NATIVE_GGML=ON" + ) +endif() +if(TARGET native_ggml_engine) + add_compile_definitions(EXECUTORCH_BUILD_NATIVE) endif() # Use a supplied or local stb checkout before fetching it. The resize header is # under deprecated/ in the pinned revision. -if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) +if(EXECUTORCH_BUILD_CUDA + OR TARGET mlxdelegate + OR TARGET native_ggml_engine +) if(NOT MUSE_GLIMMER_STB_INCLUDE_DIR AND EXISTS "${EXECUTORCH_ROOT}/third-party/stb/stb_image.h" ) @@ -98,100 +109,106 @@ endif() # Tokenizer (tiktoken via pytorch/tokenizers) list(APPEND link_libraries tokenizers::tokenizers) -add_executable( - solo_runner runtime/runners/solo.cpp runtime/engine/muse_glimmer_engine.cpp - runtime/engine/dflash_session.cpp -) +# The other runners and the legacy worker run on CUDA or MLX. if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) - target_sources( - solo_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp + add_executable( + solo_runner runtime/runners/solo.cpp runtime/engine/muse_glimmer_engine.cpp + runtime/engine/dflash_session.cpp ) -endif() -target_include_directories( - solo_runner PUBLIC ${_common_include_directories} ${_json_include} -) -target_link_libraries(solo_runner PUBLIC ${link_libraries}) + if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) + target_sources( + solo_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp + ) + endif() + target_include_directories( + solo_runner PUBLIC ${_common_include_directories} ${_json_include} + ) + target_link_libraries(solo_runner PUBLIC ${link_libraries}) -# Export symbols required by the dynamically loaded AOTI CUDA blob. -if(EXECUTORCH_BUILD_CUDA - AND NOT APPLE - AND NOT MSVC -) - target_link_options(solo_runner PRIVATE "LINKER:--export-dynamic") -endif() + # Export symbols required by the dynamically loaded AOTI CUDA blob. + if(EXECUTORCH_BUILD_CUDA + AND NOT APPLE + AND NOT MSVC + ) + target_link_options(solo_runner PRIVATE "LINKER:--export-dynamic") + endif() -add_executable( - dflash_runner - runtime/runners/dflash.cpp runtime/engine/muse_glimmer_engine.cpp - runtime/engine/dflash_session.cpp -) -if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) - target_sources( - dflash_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp + add_executable( + dflash_runner + runtime/runners/dflash.cpp runtime/engine/muse_glimmer_engine.cpp + runtime/engine/dflash_session.cpp ) -endif() -target_include_directories( - dflash_runner PUBLIC ${_common_include_directories} ${_json_include} -) -target_link_libraries(dflash_runner PUBLIC ${link_libraries}) + if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) + target_sources( + dflash_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp + ) + endif() + target_include_directories( + dflash_runner PUBLIC ${_common_include_directories} ${_json_include} + ) + target_link_libraries(dflash_runner PUBLIC ${link_libraries}) -add_executable(sampling_test tests/sampling_test.cpp) -target_include_directories(sampling_test PUBLIC ${_common_include_directories}) -target_link_libraries(sampling_test PUBLIC extension_tensor executorch) + add_executable(sampling_test tests/sampling_test.cpp) + target_include_directories( + sampling_test PUBLIC ${_common_include_directories} + ) + target_link_libraries(sampling_test PUBLIC extension_tensor executorch) -add_executable(dflash_timing_test tests/dflash_timing_test.cpp) -target_include_directories( - dflash_timing_test PUBLIC ${_common_include_directories} -) + add_executable(dflash_timing_test tests/dflash_timing_test.cpp) + target_include_directories( + dflash_timing_test PUBLIC ${_common_include_directories} + ) -# Worker used by the OpenAI-compatible serving harness. -add_executable( - muse_glimmer_worker - runtime/runners/muse_glimmer_worker.cpp - runtime/engine/muse_glimmer_engine.cpp runtime/engine/dflash_session.cpp -) -if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) - target_sources( - muse_glimmer_worker PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp + # Worker used by the OpenAI-compatible serving harness. + add_executable( + muse_glimmer_worker + runtime/runners/muse_glimmer_worker.cpp + runtime/engine/muse_glimmer_engine.cpp runtime/engine/dflash_session.cpp + ) + if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate) + target_sources( + muse_glimmer_worker + PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp + ) + endif() + target_include_directories( + muse_glimmer_worker PUBLIC ${_common_include_directories} ${_json_include} + ) + target_link_libraries( + muse_glimmer_worker PUBLIC ${link_libraries} Threads::Threads ) -endif() -target_include_directories( - muse_glimmer_worker PUBLIC ${_common_include_directories} ${_json_include} -) -target_link_libraries( - muse_glimmer_worker PUBLIC ${link_libraries} Threads::Threads -) -if(EXECUTORCH_BUILD_CUDA - AND NOT APPLE - AND NOT MSVC -) - target_link_options(dflash_runner PRIVATE "LINKER:--export-dynamic") - target_link_options(muse_glimmer_worker PRIVATE "LINKER:--export-dynamic") -endif() + if(EXECUTORCH_BUILD_CUDA + AND NOT APPLE + AND NOT MSVC + ) + target_link_options(dflash_runner PRIVATE "LINKER:--export-dynamic") + target_link_options(muse_glimmer_worker PRIVATE "LINKER:--export-dynamic") + endif() -if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - target_link_options_gc_sections(solo_runner) - target_link_options_gc_sections(dflash_runner) - target_link_options_gc_sections(muse_glimmer_worker) - if(NOT APPLE AND NOT MSVC) - # CUDA AOTI blobs resolve symbols from the host executable. - if(NOT EXECUTORCH_BUILD_CUDA) - target_link_options(solo_runner PRIVATE "LINKER:-s") - target_link_options(dflash_runner PRIVATE "LINKER:-s") - target_link_options(muse_glimmer_worker PRIVATE "LINKER:-s") + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + target_link_options_gc_sections(solo_runner) + target_link_options_gc_sections(dflash_runner) + target_link_options_gc_sections(muse_glimmer_worker) + if(NOT APPLE AND NOT MSVC) + # CUDA AOTI blobs resolve symbols from the host executable. + if(NOT EXECUTORCH_BUILD_CUDA) + target_link_options(solo_runner PRIVATE "LINKER:-s") + target_link_options(dflash_runner PRIVATE "LINKER:-s") + target_link_options(muse_glimmer_worker PRIVATE "LINKER:-s") + endif() endif() endif() -endif() -if(TARGET mlxdelegate) - executorch_target_copy_mlx_metallib(solo_runner) - executorch_target_copy_mlx_metallib(dflash_runner) - executorch_target_copy_mlx_metallib(muse_glimmer_worker) + if(TARGET mlxdelegate) + executorch_target_copy_mlx_metallib(solo_runner) + executorch_target_copy_mlx_metallib(dflash_runner) + executorch_target_copy_mlx_metallib(muse_glimmer_worker) + endif() endif() option(MUSE_GLIMMER_BUILD_BATCHING - "Build the batching runner and worker (MLX or CUDA)" OFF + "Build the batching runner and worker (MLX, CUDA or native)" OFF ) option(MUSE_GLIMMER_BUILD_BATCHING_TESTS "Build preparation/materialization tests without the batching executor" @@ -208,10 +225,12 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) set(_batching_backend cuda) elseif(EXECUTORCH_BUILD_MLX AND TARGET mlxdelegate) set(_batching_backend mlx) + elseif(TARGET native_ggml_engine) + set(_batching_backend native) else() message( FATAL_ERROR - "Muse Glimmer batching targets require EXECUTORCH_BUILD_CUDA=ON or EXECUTORCH_BUILD_MLX=ON with an MLX-enabled ExecuTorch installation" + "Muse Glimmer batching targets require EXECUTORCH_BUILD_CUDA=ON, EXECUTORCH_BUILD_MLX=ON with an MLX-enabled ExecuTorch installation, or an installation with EXECUTORCH_BUILD_NATIVE_GGML=ON" ) endif() foreach(_dependency extension_module extension_tensor tokenizers::tokenizers) @@ -244,10 +263,14 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) if(_batching_backend STREQUAL "cuda") set(_batching_executor runtime/cuda_dflash_executor.cpp) set(_batching_backend_library cuda_batching) - else() + elseif(_batching_backend STREQUAL "mlx") set(_batching_executor runtime/mlx_executor.cpp) set(_batching_backend_library mlx) endif() + # The factory offers the native engine wherever it is installed. + if(TARGET native_ggml_engine) + list(APPEND _batching_backend_library native_batching native_ggml_engine) + endif() set(_batching_dependencies extension_llm_batching extension_llm_cache @@ -321,7 +344,7 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) muse_glimmer_batching_worker PRIVATE "LINKER:--export-dynamic" ) endif() - else() + elseif(_batching_backend STREQUAL "mlx") executorch_target_copy_mlx_metallib(muse_glimmer_batching_runner) executorch_target_copy_mlx_metallib(muse_glimmer_batching_worker) endif() @@ -339,7 +362,7 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS) muse_glimmer_batching_smoke PRIVATE "LINKER:--export-dynamic" ) endif() - else() + elseif(_batching_backend STREQUAL "mlx") executorch_target_copy_mlx_metallib(muse_glimmer_batching_smoke) endif() endif() diff --git a/examples/models/muse-glimmer/CMakePresets.json b/examples/models/muse-glimmer/CMakePresets.json index 4cc407d4081..b80c95f3750 100644 --- a/examples/models/muse-glimmer/CMakePresets.json +++ b/examples/models/muse-glimmer/CMakePresets.json @@ -36,6 +36,15 @@ "lhs": "${hostSystemName}", "rhs": "Darwin" } + }, + { + "name": "muse-glimmer-native", + "displayName": "Muse Glimmer batching runner (native ggml engine)", + "inherits": ["muse-glimmer-base"], + "cacheVariables": { + "EXECUTORCH_BUILD_CUDA": "OFF", + "MUSE_GLIMMER_BUILD_BATCHING": "ON" + } } ], "buildPresets": [ diff --git a/examples/models/muse-glimmer/runtime/backend_factory.cpp b/examples/models/muse-glimmer/runtime/backend_factory.cpp index 0916ceb4fb0..8d1b6df86e0 100644 --- a/examples/models/muse-glimmer/runtime/backend_factory.cpp +++ b/examples/models/muse-glimmer/runtime/backend_factory.cpp @@ -9,6 +9,10 @@ #if defined(EXECUTORCH_BUILD_MLX) #include #endif +#if defined(EXECUTORCH_BUILD_NATIVE) +#include +#include +#endif #if defined(EXECUTORCH_BUILD_CUDA) #include #include @@ -25,6 +29,64 @@ runtime::Result create_muse_glimmer_backend( return MuseGlimmerMLXExecutor::create(config); } #endif +#if defined(EXECUTORCH_BUILD_NATIVE) + if (config.backend == "native") { + // Needs a package exported with export_hf_llm.py --offgraph_cache + // --logits_to_keep selected; text only. + ET_CHECK_OR_RETURN_ERROR( + !config.use_dflash, + NotSupported, + "Muse Glimmer native: DFlash is unsupported"); + ET_CHECK_OR_RETURN_ERROR( + (config.native_device == "cpu" || config.native_device == "gpu") && + (config.native_kv_dtype == "fp32" || + config.native_kv_dtype == "fp16"), + InvalidArgument, + "Muse Glimmer native: --native_device must be cpu or gpu and " + "--native_kv_dtype fp32 or fp16"); + ET_CHECK_OR_RETURN_ERROR( + config.data_path.empty() && config.pos_embed_path.empty(), + InvalidArgument, + "Muse Glimmer native: the .ptn holds the weights and serves text " + "only; omit --data_path and --pos_embed_path"); + ptn::register_ggml_cache_builders(); + const ptn::GgmlDevice device = config.native_device == "cpu" + ? ptn::GgmlDevice::Cpu + : ptn::GgmlDevice::Gpu; + ET_ASSIGN_OR_RETURN( + executor, + ptn::NativeExecutor::create( + {.model_path = config.model_path, + .make_host = + [device] { + return ptn::GgmlEngineHost::create({.device = device}); + }, + .cache_backend_id = ptn::kGgmlCacheBackendId, + .max_sessions = config.max_sessions, + .max_session_tokens = config.max_session_tokens, + .kv_dtype = config.native_kv_dtype == "fp16" + ? ptn::ScalarType::Half + : ptn::ScalarType::Float})); + const int32_t vocab = executor->vocab_size(); + ET_CHECK_OR_RETURN_ERROR( + vocab > 200092 && config.bos_id < static_cast(vocab), + InvalidProgram, + "Muse Glimmer native: vocabulary %d cannot hold its special tokens", + vocab); + auto spec = std::make_shared( + MuseGlimmerPreparationSpec{ + aten::ScalarType::Float, + /*hidden_dim=*/0, + executor->max_session_tokens(), + static_cast(executor->preferred_batch_tokens()), + vocab, + /*has_vision=*/false, + /*max_soft_tokens=*/0, + config.image_limits, + config.bos_id}); + return MuseGlimmerBackend{std::move(executor), std::move(spec)}; + } +#endif #if defined(EXECUTORCH_BUILD_CUDA) if (config.backend == "cuda") { namespace cb = ::executorch::backends::cuda::batching; diff --git a/examples/models/muse-glimmer/runtime/backend_factory.h b/examples/models/muse-glimmer/runtime/backend_factory.h index cb908851cc5..cee23f35e21 100644 --- a/examples/models/muse-glimmer/runtime/backend_factory.h +++ b/examples/models/muse-glimmer/runtime/backend_factory.h @@ -24,6 +24,8 @@ struct MuseGlimmerBackendConfig { batching::Token bos_id = 200000; bool use_dflash = false; int dflash_n_draft = 0; // zero uses the exported maximum minus one + std::string native_device = "gpu"; // native: ggml "cpu" or "gpu" + std::string native_kv_dtype = "fp32"; // native: "fp32" or "fp16" }; struct MuseGlimmerBackend { diff --git a/examples/models/muse-glimmer/runtime/prompt_preparation.cpp b/examples/models/muse-glimmer/runtime/prompt_preparation.cpp index cf04aef4072..d749a9735f7 100644 --- a/examples/models/muse-glimmer/runtime/prompt_preparation.cpp +++ b/examples/models/muse-glimmer/runtime/prompt_preparation.cpp @@ -15,7 +15,9 @@ #include #include -#if defined(EXECUTORCH_BUILD_MLX) +// CommonCrypto where Apple provides it; the portable digest elsewhere. +#if defined(__APPLE__) && !defined(EXECUTORCH_BUILD_CUDA) +#define MUSE_GLIMMER_COMMON_CRYPTO #include #endif @@ -49,8 +51,8 @@ bool valid_image( bytes == static_cast(pixels) * 3; } -#if defined(EXECUTORCH_BUILD_CUDA) -// FIPS 180-4 SHA-256 for CUDA builds without a platform crypto library. +#if !defined(MUSE_GLIMMER_COMMON_CRYPTO) +// FIPS 180-4 SHA-256 for builds without a platform crypto library. class Sha256 { public: void update(const void* data, size_t size) { @@ -149,7 +151,7 @@ class Sha256 { size_t buffered_ = 0; uint64_t length_ = 0; }; -#elif defined(EXECUTORCH_BUILD_MLX) +#else class Sha256 { public: Sha256() { @@ -176,8 +178,6 @@ class Sha256 { private: CC_SHA256_CTX context_; }; -#else -#error "Muse Glimmer preparation requires a CUDA or MLX build" #endif batching::ContentKey image_key( diff --git a/examples/models/muse-glimmer/runtime/runners/batching_bootstrap.cpp b/examples/models/muse-glimmer/runtime/runners/batching_bootstrap.cpp index afa9b93a9fd..7d99be9bf4c 100644 --- a/examples/models/muse-glimmer/runtime/runners/batching_bootstrap.cpp +++ b/examples/models/muse-glimmer/runtime/runners/batching_bootstrap.cpp @@ -22,9 +22,13 @@ DEFINE_string(data_path, "", "Optional external weights file"); DEFINE_string(pos_embed_path, "", "Optional vision positional table path"); #if defined(EXECUTORCH_BUILD_CUDA) DEFINE_string(backend, "cuda", "Executor backend"); -#else +#elif defined(EXECUTORCH_BUILD_MLX) DEFINE_string(backend, "mlx", "Executor backend"); +#else +DEFINE_string(backend, "native", "Executor backend"); #endif +DEFINE_string(native_device, "gpu", "Native backend ggml device: cpu or gpu"); +DEFINE_string(native_kv_dtype, "fp32", "Native backend KV cache: fp32 or fp16"); DEFINE_int32(max_sessions, 4, "Maximum logical sessions"); DEFINE_int32( max_session_tokens, @@ -111,6 +115,8 @@ create_muse_glimmer_batching_runtime() { backend.bos_id = FLAGS_bos_id; backend.use_dflash = FLAGS_dflash; backend.dflash_n_draft = FLAGS_dflash_n_draft; + backend.native_device = FLAGS_native_device; + backend.native_kv_dtype = FLAGS_native_kv_dtype; auto created = create_muse_glimmer_backend(backend); if (!created.ok()) throw std::runtime_error(