Merge pull request #1 from mpj1234/main

wangzhaode · web-flow · commit 3b07a267a739 · 2025-12-27T09:10:35.000+08:00
fest(tokenizer): Fix redundant im_end token output
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
@@ -1065,7 +1065,6 @@ struct PreTrainedTokenizer::Impl {
                 }
             }
         }
-        if (add_special_tokens && special_tokens_.eos != -1) input_ids.push_back(special_tokens_.eos);
         return input_ids;
     }
 

Original file line number	Diff line number	Diff line change
`@@ -1065,7 +1065,6 @@ struct PreTrainedTokenizer::Impl {`
`1065`	`1065`	`}`
`1066`	`1066`	`}`
`1067`	`1067`	`}`
`1068`		`- if (add_special_tokens && special_tokens_.eos != -1) input_ids.push_back(special_tokens_.eos);`
`1069`	`1068`	`return input_ids;`
`1070`	`1069`	`}`
`1071`	`1070`