feat: add legacy option

Author: terryyz

bigcodebench/generate.py

@@ -119,6 +119,7 @@ def main():
     parser.add_argument("--base_url", default=None, type=str)
     parser.add_argument("--tp", default=1, type=int)
     parser.add_argument("--trust_remote_code", action="store_true")
+    parser.add_argument("--tokenizer_legacy", action="store_true")
     parser.add_argument("--tokenizer_name", default=None, type=str)
 
     args = parser.parse_args()
@@ -144,7 +145,8 @@ def main():
         base_url=args.base_url,
         tp=args.tp,
         trust_remote_code=args.trust_remote_code,
-        tokenizer_name=args.tokenizer_name
+        tokenizer_name=args.tokenizer_name,
+        tokenizer_legacy=args.tokenizer_legacy
     )
 
     extra = "-" + args.subset if args.subset != "full" else ""

bigcodebench/model.py

@@ -92,6 +92,7 @@ def __init__(
         dtype: str = "bfloat16",  # default
         trust_remote_code: bool = False,
         tokenizer_name: str = None,
+        tokenizer_legacy: bool = False,
     ) -> None:
         print("Initializing a decoder model: {} ...".format(name))
         self.name = name
@@ -103,6 +104,7 @@ def __init__(
         self.dtype = dtype
         self.trust_remote_code = trust_remote_code
         self.tokenizer_name = tokenizer_name
+        self.tokenizer_legacy = tokenizer_legacy
 
     @abstractmethod
     def codegen(
@@ -133,7 +135,7 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None:
         if self.tokenizer_name is None:
             self.tokenizer_name = self.name
 
-        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=not self.tokenizer_legacy)
         if self.tokenizer.chat_template is None:
             self.eos += extra_eos_for_direct_completion(dataset)
         self.llm = LLM(model=name, max_model_len=2048, **kwargs)
@@ -193,7 +195,7 @@ def __init__(self, name: str, dataset: str, **kwargs):
         if self.tokenizer_name is None:
             self.tokenizer_name = self.name
 
-        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name, **kwargs, legacy=not self.tokenizer_legacy)
 
         if self.tokenizer.chat_template is None:
             self.eos += extra_eos_for_direct_completion(dataset)
@@ -249,7 +251,8 @@ def __init__(self, name: str, **kwargs):
         super().__init__(name=name, **kwargs)
         self.eos += ["\n```\n"]
         print(f"EOS strings: {self.eos}")
-        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name, **kwargs)
+        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name if self.tokenizer_name else self.name,
+                                                       **kwargs, legacy=not self.tokenizer_legacy)
 
     def codegen(
         self, prompt: str, do_sample: bool = True, num_samples: int = 200
@@ -483,6 +486,7 @@ def make_model(
     base_url=None,
     trust_remote_code=False,
     tokenizer_name=None,
+    tokenizer_legacy=True,
 ):
     if backend == "vllm":
         return GeneralVllmDecoder(
@@ -493,6 +497,7 @@ def make_model(
             tp=tp,
             trust_remote_code=trust_remote_code,
             tokenizer_name=tokenizer_name,
+            tokenizer_legacy=tokenizer_legacy,
         )
     elif backend == "hf":
         return GenenralHfTorchDecoder(
@@ -502,6 +507,7 @@ def make_model(
             dataset=dataset,
             trust_remote_code=trust_remote_code,
             tokenizer_name=tokenizer_name,
+            tokenizer_legacy=tokenizer_legacy,
         )
     elif backend == "openai":
         return OpenAIChatDecoder(