From 4471d769b0e2f0859714cc321f68009ed8030d80 Mon Sep 17 00:00:00 2001 From: Taksh Date: Wed, 24 Jun 2026 20:06:56 +0530 Subject: [PATCH] fix: remove off-by-one in inference prompt truncation Co-authored-by: Cursor --- fastchat/serve/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastchat/serve/inference.py b/fastchat/serve/inference.py index 6d155aab7..ef6744df5 100644 --- a/fastchat/serve/inference.py +++ b/fastchat/serve/inference.py @@ -94,7 +94,7 @@ def generate_stream( if model.config.is_encoder_decoder: max_src_len = context_len else: # truncate - max_src_len = context_len - max_new_tokens - 1 + max_src_len = context_len - max_new_tokens input_ids = input_ids[-max_src_len:] output_ids = list(input_ids)