Skip to content

[Native] Replay sample sets without decoding their tokens #24950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
metascroy wants to merge 1 commit into
base: offgraph-kv-mg-gguf
Choose a base branch
from
Draft

[Native] Replay sample sets without decoding their tokens #24950

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 21 additions & 15 deletions backends/native/examples/hf_llm/runtime/LlmRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
// All rights reserved.
//
Expand Down Expand Up @@ -141,7 +141,7 @@
const auto arg = [&](std::string_view name,
size_t positional) -> const ptn::Argument& {
for (const ptn::NamedArgument& input : node.inputs) {
if (input.name == name) {

Check warning on line 144 in backends/native/examples/hf_llm/runtime/LlmRunner.cpp

View workflow job for this annotation

GitHub Actions / lintrunner

CPPCHECK useStlAlgorithm 

Consider using std::find_if algorithm instead of a raw loop.
CWE: CWE-398
return input.arg;
}
}
Expand Down Expand Up @@ -349,14 +349,15 @@

// Prefills `prompt_tokens` from `start_position` in calls of at most
// `max_chunk` tokens, then decodes one token per call until `limit` tokens or
// an EOS, streaming the text to stdout. With --warmup_prefill it prefills the
// prompt once beforehand, then clears `offgraph_cache` if there is one.
// an EOS, streaming the text to stdout unless `tokenizer` is null. With
// --warmup_prefill it prefills the prompt once beforehand, then clears
// `offgraph_cache` if there is one.
Generation generate(
ptn::EngineExecutable& executable,
size_t logits_output,
bool per_token_positions,
size_t max_chunk,
::tokenizers::Tiktoken& tokenizer,
const ::tokenizers::Tiktoken* tokenizer,
const std::vector<uint64_t>& prompt_tokens,
int64_t start_position,
size_t limit,
Expand Down Expand Up @@ -488,19 +489,21 @@
size_t generated = 0;
std::vector<uint64_t> generated_ids;
uint64_t previous_token = prompt_tokens.back();
if (!FLAGS_interactive) {
if (tokenizer != nullptr && !FLAGS_interactive) {
std::printf("text: ");
}
std::fflush(stdout);
while (generated < limit && !eos_ids.contains(next_token)) {
auto decoded = tokenizer.decode(previous_token, next_token);
if (!decoded.ok()) {
throw std::runtime_error(
"failed to decode token: tokenizer error " +
std::to_string(static_cast<uint32_t>(decoded.error())));
if (tokenizer != nullptr) {
auto decoded = tokenizer->decode(previous_token, next_token);
if (!decoded.ok()) {
throw std::runtime_error(
"failed to decode token: tokenizer error " +
std::to_string(static_cast<uint32_t>(decoded.error())));
}
std::printf("%s", decoded.get().c_str());
std::fflush(stdout);
}
std::printf("%s", decoded.get().c_str());
std::fflush(stdout);
previous_token = next_token;
if (FLAGS_print_token_ids) {
generated_ids.push_back(next_token);
Expand All @@ -517,7 +520,9 @@
++generation.decode_steps;
}
}
std::printf("\n");
if (tokenizer != nullptr) {
std::printf("\n");
}
generation.unfed_token = next_token;
generation.ended_on_eos = eos_ids.contains(next_token);
if (FLAGS_print_token_ids) {
Expand Down Expand Up @@ -805,7 +810,8 @@
logits_output,
per_token_positions,
static_cast<size_t>(token_shape[1]),
*tokenizer,
// The sample's tokens need not be in --tokenizer's vocabulary.
nullptr,
prompt,
0,
calls.size(),
Expand Down Expand Up @@ -895,7 +901,7 @@
logits_output,
per_token_positions,
static_cast<size_t>(token_shape[1]),
*tokenizer,
tokenizer.get(),
tokens,
position,
limit,
Expand All @@ -921,7 +927,7 @@
logits_output,
per_token_positions,
static_cast<size_t>(token_shape[1]),
*tokenizer,
tokenizer.get(),
prompt_tokens,
0,
limit,
Expand Down
Loading