From b5c3fb16bd253558e962d832ba73315e2554f185 Mon Sep 17 00:00:00 2001
From: Scott Roy <scroy@meta.com>
Date: Wed, 11 Nov 2026 14:19:18 -0800
Subject: [PATCH] Replay sample sets without decoding their tokens

--verify_samples streamed each generated token through --tokenizer, which
only reads text for Llama 3 vocabularies and threw on any other model's
token IDs, though the replay compares logits and never needs text.
generate() now takes the tokenizer as optional and streams text only when
given one; sample replay passes none.
---
 .../examples/hf_llm/runtime/LlmRunner.cpp     | 36 +++++++++++--------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/backends/native/examples/hf_llm/runtime/LlmRunner.cpp b/backends/native/examples/hf_llm/runtime/LlmRunner.cpp
index 7e8df20df1a..c7cfce656a8 100644
--- a/backends/native/examples/hf_llm/runtime/LlmRunner.cpp
+++ b/backends/native/examples/hf_llm/runtime/LlmRunner.cpp
@@ -349,14 +349,15 @@ struct Generation {
 
 // Prefills `prompt_tokens` from `start_position` in calls of at most
 // `max_chunk` tokens, then decodes one token per call until `limit` tokens or
-// an EOS, streaming the text to stdout. With --warmup_prefill it prefills the
-// prompt once beforehand, then clears `offgraph_cache` if there is one.
+// an EOS, streaming the text to stdout unless `tokenizer` is null. With
+// --warmup_prefill it prefills the prompt once beforehand, then clears
+// `offgraph_cache` if there is one.
 Generation generate(
     ptn::EngineExecutable& executable,
     size_t logits_output,
     bool per_token_positions,
     size_t max_chunk,
-    ::tokenizers::Tiktoken& tokenizer,
+    const ::tokenizers::Tiktoken* tokenizer,
     const std::vector<uint64_t>& prompt_tokens,
     int64_t start_position,
     size_t limit,
@@ -488,19 +489,21 @@ Generation generate(
   size_t generated = 0;
   std::vector<uint64_t> generated_ids;
   uint64_t previous_token = prompt_tokens.back();
-  if (!FLAGS_interactive) {
+  if (tokenizer != nullptr && !FLAGS_interactive) {
     std::printf("text:      ");
   }
   std::fflush(stdout);
   while (generated < limit && !eos_ids.contains(next_token)) {
-    auto decoded = tokenizer.decode(previous_token, next_token);
-    if (!decoded.ok()) {
-      throw std::runtime_error(
-          "failed to decode token: tokenizer error " +
-          std::to_string(static_cast<uint32_t>(decoded.error())));
+    if (tokenizer != nullptr) {
+      auto decoded = tokenizer->decode(previous_token, next_token);
+      if (!decoded.ok()) {
+        throw std::runtime_error(
+            "failed to decode token: tokenizer error " +
+            std::to_string(static_cast<uint32_t>(decoded.error())));
+      }
+      std::printf("%s", decoded.get().c_str());
+      std::fflush(stdout);
     }
-    std::printf("%s", decoded.get().c_str());
-    std::fflush(stdout);
     previous_token = next_token;
     if (FLAGS_print_token_ids) {
       generated_ids.push_back(next_token);
@@ -517,7 +520,9 @@ Generation generate(
       ++generation.decode_steps;
     }
   }
-  std::printf("\n");
+  if (tokenizer != nullptr) {
+    std::printf("\n");
+  }
   generation.unfed_token = next_token;
   generation.ended_on_eos = eos_ids.contains(next_token);
   if (FLAGS_print_token_ids) {
@@ -805,7 +810,8 @@ int run_llm(
             logits_output,
             per_token_positions,
             static_cast<size_t>(token_shape[1]),
-            *tokenizer,
+            // The sample's tokens need not be in --tokenizer's vocabulary.
+            nullptr,
             prompt,
             0,
             calls.size(),
@@ -895,7 +901,7 @@ int run_llm(
             logits_output,
             per_token_positions,
             static_cast<size_t>(token_shape[1]),
-            *tokenizer,
+            tokenizer.get(),
             tokens,
             position,
             limit,
@@ -921,7 +927,7 @@ int run_llm(
         logits_output,
         per_token_positions,
         static_cast<size_t>(token_shape[1]),
-        *tokenizer,
+        tokenizer.get(),
         prompt_tokens,
         0,
         limit,