From 74fb21ba01e89c09b196632c5c51f805bcad9d0b Mon Sep 17 00:00:00 2001 From: Abdelaziz Mahdy Date: Sun, 15 Nov 2026 10:16:49 -0400 Subject: [PATCH] [Wasm] Let the tokenizer bindings load HF and Tekken tokenizers Two build settings were missing for EXECUTORCH_BUILD_TOKENIZERS_WASM: - The tokenizers library catches its own exceptions in normal operation (HFTokenizer probes optional JSON keys and {"String": ...} patterns). Emscripten runs no catch blocks without -fexceptions, so the throw escaped to JS and HFTokenizer.load() failed. - HF and Tekken patterns use lookahead, which needs regex_lookahead. SUPPORT_REGEX_LOOKAHEAD was only set for the LLM runners. Fixes #25102 --- CMakeLists.txt | 5 +++++ extension/wasm/tokenizers/CMakeLists.txt | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43cfe244b87..1a27134bfde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1011,6 +1011,11 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) ) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() + if(EXECUTORCH_BUILD_TOKENIZERS_WASM) + # HF tokenizers (Llama 3, Qwen, Tekken...) use lookahead patterns that only + # the regex_lookahead engine supports. + set(SUPPORT_REGEX_LOOKAHEAD ON) + endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER diff --git a/extension/wasm/tokenizers/CMakeLists.txt b/extension/wasm/tokenizers/CMakeLists.txt index a1bead4bb80..6997b2bceed 100644 --- a/extension/wasm/tokenizers/CMakeLists.txt +++ b/extension/wasm/tokenizers/CMakeLists.txt @@ -39,3 +39,16 @@ target_include_directories( ) target_link_libraries(tokenizers_wasm PUBLIC ${link_libraries}) + +# The tokenizers library catches its own exceptions in normal operation: for +# example, HFTokenizer probes optional JSON keys and {"String": ...} patterns +# with try/catch. Emscripten disables exception catching unless -fexceptions +# is given, so without it those throws escape and load() fails on common +# tokenizer.json files. Compile the library and the bindings with it, and +# pass it on to the final link. +target_compile_options(tokenizers PRIVATE -fexceptions) +if(TARGET regex_lookahead) + target_compile_options(regex_lookahead PRIVATE -fexceptions) +endif() +target_compile_options(tokenizers_wasm PUBLIC -fexceptions) +target_link_options(tokenizers_wasm PUBLIC -fexceptions)