Skip to content

[Native] Serve Muse Glimmer on the native ggml engine #24956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
metascroy wants to merge 1 commit into
base: offgraph-kv-native-executor
Choose a base branch
from
Draft

[Native] Serve Muse Glimmer on the native ggml engine #24956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ help:
@echo " muse-glimmer-mlx - Build Muse Glimmer runners and worker with MLX backend"
@echo " muse-glimmer-batching-mlx - Build batching Muse Glimmer runner and worker with MLX"
@echo " muse-glimmer-batching-cuda - Build batching Muse Glimmer runner and worker with CUDA"
@echo " muse-glimmer-batching-native - Build batching Muse Glimmer runner and worker on the native ggml engine"
@echo " qwen3_5_moe-cuda - Build Qwen3.5 MoE runner with CUDA backend"
@echo " qwen3_5_moe-metal - Build Qwen3.5 MoE runner with Metal backend"
@echo " qwen3_5_moe-mlx - Build Qwen3.5 MoE runner with MLX backend"
Expand Down Expand Up @@ -562,6 +563,24 @@ muse-glimmer-batching-cuda:
@echo " Runner: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_runner"
@echo " Worker: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker"

.PHONY: muse-glimmer-batching-native
muse-glimmer-batching-native:
@echo "==> Building and installing ExecuTorch with the native ggml engine..."
cmake --preset llm-release -DEXECUTORCH_BUILD_NATIVE=ON \
-DEXECUTORCH_BUILD_NATIVE_GGML=ON -DGGML_OPENMP=OFF
cmake --build cmake-out --target install \
--parallel "$${CMAKE_BUILD_PARALLEL_LEVEL:-$$(sysctl -n hw.ncpu 2>/dev/null || nproc)}"
@echo "==> Configuring batching Muse Glimmer runners on the native engine..."
cd examples/models/muse-glimmer && cmake --preset muse-glimmer-native \
-B "$(CURDIR)/cmake-out/examples/models/muse-glimmer-batching"
cmake --build "$(CURDIR)/cmake-out/examples/models/muse-glimmer-batching" \
--target muse_glimmer_batching_runner muse_glimmer_batching_worker \
--parallel "$${CMAKE_BUILD_PARALLEL_LEVEL:-$$(sysctl -n hw.ncpu 2>/dev/null || nproc)}"
@echo ""
@echo "Build complete!"
@echo " Runner: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_runner"
@echo " Worker: $(CURDIR)/cmake-out/examples/models/muse-glimmer-batching/muse_glimmer_batching_worker"

gemma4_31b-cuda:
@echo "==> Building and installing ExecuTorch with CUDA..."
cmake --workflow --preset llm-release-cuda
Expand Down
193 changes: 108 additions & 85 deletions examples/models/muse-glimmer/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ list(
extension_flat_tensor
)

# Backend: CUDA or MLX (exactly one required)
# Backend: CUDA or MLX for every runner. The batching runner can also serve on
# the native ggml engine, from an installation with
# EXECUTORCH_BUILD_NATIVE_GGML.
if(EXECUTORCH_BUILD_CUDA)
find_package(CUDAToolkit REQUIRED)
list(APPEND link_libraries aoti_cuda_backend)
Expand All @@ -60,13 +62,22 @@ elseif(TARGET mlxdelegate)
list(APPEND link_libraries mlxdelegate mlx)
executorch_target_link_options_shared_lib(mlxdelegate)
add_compile_definitions(EXECUTORCH_BUILD_MLX)
else()
message(FATAL_ERROR "Set EXECUTORCH_BUILD_CUDA=ON or EXECUTORCH_BUILD_MLX=ON")
elseif(NOT TARGET native_ggml_engine)
message(
FATAL_ERROR
"Set EXECUTORCH_BUILD_CUDA=ON or EXECUTORCH_BUILD_MLX=ON, or use an ExecuTorch installation with EXECUTORCH_BUILD_NATIVE_GGML=ON"
)
endif()
if(TARGET native_ggml_engine)
add_compile_definitions(EXECUTORCH_BUILD_NATIVE)
endif()

# Use a supplied or local stb checkout before fetching it. The resize header is
# under deprecated/ in the pinned revision.
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
if(EXECUTORCH_BUILD_CUDA
OR TARGET mlxdelegate
OR TARGET native_ggml_engine
)
if(NOT MUSE_GLIMMER_STB_INCLUDE_DIR
AND EXISTS "${EXECUTORCH_ROOT}/third-party/stb/stb_image.h"
)
Expand Down Expand Up @@ -98,100 +109,106 @@ endif()
# Tokenizer (tiktoken via pytorch/tokenizers)
list(APPEND link_libraries tokenizers::tokenizers)

add_executable(
solo_runner runtime/runners/solo.cpp runtime/engine/muse_glimmer_engine.cpp
runtime/engine/dflash_session.cpp
)
# The other runners and the legacy worker run on CUDA or MLX.
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
target_sources(
solo_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp
add_executable(
solo_runner runtime/runners/solo.cpp runtime/engine/muse_glimmer_engine.cpp
runtime/engine/dflash_session.cpp
)
endif()
target_include_directories(
solo_runner PUBLIC ${_common_include_directories} ${_json_include}
)
target_link_libraries(solo_runner PUBLIC ${link_libraries})
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
target_sources(
solo_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp
)
endif()
target_include_directories(
solo_runner PUBLIC ${_common_include_directories} ${_json_include}
)
target_link_libraries(solo_runner PUBLIC ${link_libraries})

# Export symbols required by the dynamically loaded AOTI CUDA blob.
if(EXECUTORCH_BUILD_CUDA
AND NOT APPLE
AND NOT MSVC
)
target_link_options(solo_runner PRIVATE "LINKER:--export-dynamic")
endif()
# Export symbols required by the dynamically loaded AOTI CUDA blob.
if(EXECUTORCH_BUILD_CUDA
AND NOT APPLE
AND NOT MSVC
)
target_link_options(solo_runner PRIVATE "LINKER:--export-dynamic")
endif()

add_executable(
dflash_runner
runtime/runners/dflash.cpp runtime/engine/muse_glimmer_engine.cpp
runtime/engine/dflash_session.cpp
)
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
target_sources(
dflash_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp
add_executable(
dflash_runner
runtime/runners/dflash.cpp runtime/engine/muse_glimmer_engine.cpp
runtime/engine/dflash_session.cpp
)
endif()
target_include_directories(
dflash_runner PUBLIC ${_common_include_directories} ${_json_include}
)
target_link_libraries(dflash_runner PUBLIC ${link_libraries})
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
target_sources(
dflash_runner PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp
)
endif()
target_include_directories(
dflash_runner PUBLIC ${_common_include_directories} ${_json_include}
)
target_link_libraries(dflash_runner PUBLIC ${link_libraries})

add_executable(sampling_test tests/sampling_test.cpp)
target_include_directories(sampling_test PUBLIC ${_common_include_directories})
target_link_libraries(sampling_test PUBLIC extension_tensor executorch)
add_executable(sampling_test tests/sampling_test.cpp)
target_include_directories(
sampling_test PUBLIC ${_common_include_directories}
)
target_link_libraries(sampling_test PUBLIC extension_tensor executorch)

add_executable(dflash_timing_test tests/dflash_timing_test.cpp)
target_include_directories(
dflash_timing_test PUBLIC ${_common_include_directories}
)
add_executable(dflash_timing_test tests/dflash_timing_test.cpp)
target_include_directories(
dflash_timing_test PUBLIC ${_common_include_directories}
)

# Worker used by the OpenAI-compatible serving harness.
add_executable(
muse_glimmer_worker
runtime/runners/muse_glimmer_worker.cpp
runtime/engine/muse_glimmer_engine.cpp runtime/engine/dflash_session.cpp
)
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
target_sources(
muse_glimmer_worker PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp
# Worker used by the OpenAI-compatible serving harness.
add_executable(
muse_glimmer_worker
runtime/runners/muse_glimmer_worker.cpp
runtime/engine/muse_glimmer_engine.cpp runtime/engine/dflash_session.cpp
)
if(EXECUTORCH_BUILD_CUDA OR TARGET mlxdelegate)
target_sources(
muse_glimmer_worker
PRIVATE runtime/engine/muse_glimmer_vision_runtime.cpp
)
endif()
target_include_directories(
muse_glimmer_worker PUBLIC ${_common_include_directories} ${_json_include}
)
target_link_libraries(
muse_glimmer_worker PUBLIC ${link_libraries} Threads::Threads
)
endif()
target_include_directories(
muse_glimmer_worker PUBLIC ${_common_include_directories} ${_json_include}
)
target_link_libraries(
muse_glimmer_worker PUBLIC ${link_libraries} Threads::Threads
)

if(EXECUTORCH_BUILD_CUDA
AND NOT APPLE
AND NOT MSVC
)
target_link_options(dflash_runner PRIVATE "LINKER:--export-dynamic")
target_link_options(muse_glimmer_worker PRIVATE "LINKER:--export-dynamic")
endif()
if(EXECUTORCH_BUILD_CUDA
AND NOT APPLE
AND NOT MSVC
)
target_link_options(dflash_runner PRIVATE "LINKER:--export-dynamic")
target_link_options(muse_glimmer_worker PRIVATE "LINKER:--export-dynamic")
endif()

if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
target_link_options_gc_sections(solo_runner)
target_link_options_gc_sections(dflash_runner)
target_link_options_gc_sections(muse_glimmer_worker)
if(NOT APPLE AND NOT MSVC)
# CUDA AOTI blobs resolve symbols from the host executable.
if(NOT EXECUTORCH_BUILD_CUDA)
target_link_options(solo_runner PRIVATE "LINKER:-s")
target_link_options(dflash_runner PRIVATE "LINKER:-s")
target_link_options(muse_glimmer_worker PRIVATE "LINKER:-s")
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
target_link_options_gc_sections(solo_runner)
target_link_options_gc_sections(dflash_runner)
target_link_options_gc_sections(muse_glimmer_worker)
if(NOT APPLE AND NOT MSVC)
# CUDA AOTI blobs resolve symbols from the host executable.
if(NOT EXECUTORCH_BUILD_CUDA)
target_link_options(solo_runner PRIVATE "LINKER:-s")
target_link_options(dflash_runner PRIVATE "LINKER:-s")
target_link_options(muse_glimmer_worker PRIVATE "LINKER:-s")
endif()
endif()
endif()
endif()

if(TARGET mlxdelegate)
executorch_target_copy_mlx_metallib(solo_runner)
executorch_target_copy_mlx_metallib(dflash_runner)
executorch_target_copy_mlx_metallib(muse_glimmer_worker)
if(TARGET mlxdelegate)
executorch_target_copy_mlx_metallib(solo_runner)
executorch_target_copy_mlx_metallib(dflash_runner)
executorch_target_copy_mlx_metallib(muse_glimmer_worker)
endif()
endif()

option(MUSE_GLIMMER_BUILD_BATCHING
"Build the batching runner and worker (MLX or CUDA)" OFF
"Build the batching runner and worker (MLX, CUDA or native)" OFF
)
option(MUSE_GLIMMER_BUILD_BATCHING_TESTS
"Build preparation/materialization tests without the batching executor"
Expand All @@ -208,10 +225,12 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS)
set(_batching_backend cuda)
elseif(EXECUTORCH_BUILD_MLX AND TARGET mlxdelegate)
set(_batching_backend mlx)
elseif(TARGET native_ggml_engine)
set(_batching_backend native)
else()
message(
FATAL_ERROR
"Muse Glimmer batching targets require EXECUTORCH_BUILD_CUDA=ON or EXECUTORCH_BUILD_MLX=ON with an MLX-enabled ExecuTorch installation"
"Muse Glimmer batching targets require EXECUTORCH_BUILD_CUDA=ON, EXECUTORCH_BUILD_MLX=ON with an MLX-enabled ExecuTorch installation, or an installation with EXECUTORCH_BUILD_NATIVE_GGML=ON"
)
endif()
foreach(_dependency extension_module extension_tensor tokenizers::tokenizers)
Expand Down Expand Up @@ -244,10 +263,14 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS)
if(_batching_backend STREQUAL "cuda")
set(_batching_executor runtime/cuda_dflash_executor.cpp)
set(_batching_backend_library cuda_batching)
else()
elseif(_batching_backend STREQUAL "mlx")
set(_batching_executor runtime/mlx_executor.cpp)
set(_batching_backend_library mlx)
endif()
# The factory offers the native engine wherever it is installed.
if(TARGET native_ggml_engine)
list(APPEND _batching_backend_library native_batching native_ggml_engine)
endif()
set(_batching_dependencies
extension_llm_batching
extension_llm_cache
Expand Down Expand Up @@ -321,7 +344,7 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS)
muse_glimmer_batching_worker PRIVATE "LINKER:--export-dynamic"
)
endif()
else()
elseif(_batching_backend STREQUAL "mlx")
executorch_target_copy_mlx_metallib(muse_glimmer_batching_runner)
executorch_target_copy_mlx_metallib(muse_glimmer_batching_worker)
endif()
Expand All @@ -339,7 +362,7 @@ if(MUSE_GLIMMER_BUILD_BATCHING OR MUSE_GLIMMER_BUILD_BATCHING_TESTS)
muse_glimmer_batching_smoke PRIVATE "LINKER:--export-dynamic"
)
endif()
else()
elseif(_batching_backend STREQUAL "mlx")
executorch_target_copy_mlx_metallib(muse_glimmer_batching_smoke)
endif()
endif()
Expand Down
9 changes: 9 additions & 0 deletions examples/models/muse-glimmer/CMakePresets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@
"lhs": "${hostSystemName}",
"rhs": "Darwin"
}
},
{
"name": "muse-glimmer-native",
"displayName": "Muse Glimmer batching runner (native ggml engine)",
"inherits": ["muse-glimmer-base"],
"cacheVariables": {
"EXECUTORCH_BUILD_CUDA": "OFF",
"MUSE_GLIMMER_BUILD_BATCHING": "ON"
}
}
],
"buildPresets": [
Expand Down
62 changes: 62 additions & 0 deletions examples/models/muse-glimmer/runtime/backend_factory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
#if defined(EXECUTORCH_BUILD_MLX)
#include <executorch/examples/models/muse-glimmer/runtime/mlx_executor.h>
#endif
#if defined(EXECUTORCH_BUILD_NATIVE)
#include <executorch/backends/native/batching/native_executor.h>
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngine.h>
#endif
#if defined(EXECUTORCH_BUILD_CUDA)
#include <executorch/backends/cuda/batching/cuda_executor.h>
#include <executorch/examples/models/muse-glimmer/runtime/cuda_dflash_executor.h>
Expand All @@ -25,6 +29,64 @@ runtime::Result<MuseGlimmerBackend> create_muse_glimmer_backend(
return MuseGlimmerMLXExecutor::create(config);
}
#endif
#if defined(EXECUTORCH_BUILD_NATIVE)
if (config.backend == "native") {
// Needs a package exported with export_hf_llm.py --offgraph_cache
// --logits_to_keep selected; text only.
ET_CHECK_OR_RETURN_ERROR(
!config.use_dflash,
NotSupported,
"Muse Glimmer native: DFlash is unsupported");
ET_CHECK_OR_RETURN_ERROR(
(config.native_device == "cpu" || config.native_device == "gpu") &&
(config.native_kv_dtype == "fp32" ||
config.native_kv_dtype == "fp16"),
InvalidArgument,
"Muse Glimmer native: --native_device must be cpu or gpu and "
"--native_kv_dtype fp32 or fp16");
ET_CHECK_OR_RETURN_ERROR(
config.data_path.empty() && config.pos_embed_path.empty(),
InvalidArgument,
"Muse Glimmer native: the .ptn holds the weights and serves text "
"only; omit --data_path and --pos_embed_path");
ptn::register_ggml_cache_builders();
const ptn::GgmlDevice device = config.native_device == "cpu"
? ptn::GgmlDevice::Cpu
: ptn::GgmlDevice::Gpu;
ET_ASSIGN_OR_RETURN(
executor,
ptn::NativeExecutor::create(
{.model_path = config.model_path,
.make_host =
[device] {
return ptn::GgmlEngineHost::create({.device = device});
},
.cache_backend_id = ptn::kGgmlCacheBackendId,
.max_sessions = config.max_sessions,
.max_session_tokens = config.max_session_tokens,
.kv_dtype = config.native_kv_dtype == "fp16"
? ptn::ScalarType::Half
: ptn::ScalarType::Float}));
const int32_t vocab = executor->vocab_size();
ET_CHECK_OR_RETURN_ERROR(
vocab > 200092 && config.bos_id < static_cast<uint64_t>(vocab),
InvalidProgram,
"Muse Glimmer native: vocabulary %d cannot hold its special tokens",
vocab);
auto spec = std::make_shared<const MuseGlimmerPreparationSpec>(
MuseGlimmerPreparationSpec{
aten::ScalarType::Float,
/*hidden_dim=*/0,
executor->max_session_tokens(),
static_cast<int32_t>(executor->preferred_batch_tokens()),
vocab,
/*has_vision=*/false,
/*max_soft_tokens=*/0,
config.image_limits,
config.bos_id});
return MuseGlimmerBackend{std::move(executor), std::move(spec)};
}
#endif
#if defined(EXECUTORCH_BUILD_CUDA)
if (config.backend == "cuda") {
namespace cb = ::executorch::backends::cuda::batching;
Expand Down
Loading
Loading