File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -9188,19 +9188,16 @@ def set_vocab(self):
91889188 vocab_size = self .hparams ["vocab_size" ]
91899189 assert tokenizer .vocab_size == vocab_size
91909190 special_tokens = tokenizer .special_tokens
9191- reverse_vocab = {id_ : encoded_tok for encoded_tok , id_ in {** vocab , ** special_tokens }.items () if id_ is not None }
9191+ reverse_vocab = {id_ : encoded_tok for encoded_tok , id_ in {** vocab , ** special_tokens }.items ()}
91929192 tokens : list [str ] = []
91939193 toktypes : list [int ] = []
91949194 for i in range (vocab_size ):
91959195 if i not in reverse_vocab :
9196- tokens .append (f"[PAD{ i } " )
9196+ tokens .append (f"[PAD{ i } ] " )
91979197 toktypes .append (gguf .TokenType .UNUSED )
91989198 else :
91999199 token = reverse_vocab [i ]
9200- if token is None :
9201- tokens .append (f"[PAD{ i } " )
9202- else :
9203- tokens .append (str (token ))
9200+ tokens .append (token )
92049201 if i in special_tokens .values ():
92059202 toktypes .append (gguf .TokenType .CONTROL )
92069203 else :
You can’t perform that action at this time.
0 commit comments