From 6401e3151d2c0bfd9ee780bea0b898b72d866b29 Mon Sep 17 00:00:00 2001 From: Taksh Date: Wed, 24 Jun 2026 20:06:56 +0530 Subject: [PATCH] fix: remove off-by-one in inference prompt truncation --- fastchat/serve/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/inference.py b/fastchat/serve/inference.py index 6d155aab7..ef6744df5 100644 --- a/fastchat/serve/inference.py +++ b/fastchat/serve/inference.py @@ -94,7 +94,7 @@ def generate_stream( if model.config.is_encoder_decoder: max_src_len = context_len else: # truncate - max_src_len = context_len - max_new_tokens - 1 + max_src_len = context_len - max_new_tokens input_ids = input_ids[-max_src_len:] output_ids = list(input_ids)