diff --git a/CMakeLists.txt b/CMakeLists.txt index 43cfe244b87..1a27134bfde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1011,6 +1011,11 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM) ) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif() + if(EXECUTORCH_BUILD_TOKENIZERS_WASM) + # HF tokenizers (Llama 3, Qwen, Tekken...) use lookahead patterns that only + # the regex_lookahead engine supports. + set(SUPPORT_REGEX_LOOKAHEAD ON) + endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers) if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER OR EXECUTORCH_BUILD_NATIVE_LLM_RUNNER diff --git a/extension/wasm/tokenizers/CMakeLists.txt b/extension/wasm/tokenizers/CMakeLists.txt index a1bead4bb80..6997b2bceed 100644 --- a/extension/wasm/tokenizers/CMakeLists.txt +++ b/extension/wasm/tokenizers/CMakeLists.txt @@ -39,3 +39,16 @@ target_include_directories( ) target_link_libraries(tokenizers_wasm PUBLIC ${link_libraries}) + +# The tokenizers library catches its own exceptions in normal operation: for +# example, HFTokenizer probes optional JSON keys and {"String": ...} patterns +# with try/catch. Emscripten disables exception catching unless -fexceptions +# is given, so without it those throws escape and load() fails on common +# tokenizer.json files. Compile the library and the bindings with it, and +# pass it on to the final link. +target_compile_options(tokenizers PRIVATE -fexceptions) +if(TARGET regex_lookahead) + target_compile_options(regex_lookahead PRIVATE -fexceptions) +endif() +target_compile_options(tokenizers_wasm PUBLIC -fexceptions) +target_link_options(tokenizers_wasm PUBLIC -fexceptions)