Skip to content

Commit cd46a28

Browse files
tamarPaltamarPal
authored andcommitted
fix: remove unintended HunYuanMoE changes
Restore HunYuanMoE code to upstream version - no modifications needed
1 parent 4b67f5c commit cd46a28

1 file changed

Lines changed: 3 additions & 6 deletions

File tree

convert_hf_to_gguf.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9188,19 +9188,16 @@ def set_vocab(self):
91889188
vocab_size = self.hparams["vocab_size"]
91899189
assert tokenizer.vocab_size == vocab_size
91909190
special_tokens = tokenizer.special_tokens
9191-
reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items() if id_ is not None}
9191+
reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **special_tokens}.items()}
91929192
tokens: list[str] = []
91939193
toktypes: list[int] = []
91949194
for i in range(vocab_size):
91959195
if i not in reverse_vocab:
9196-
tokens.append(f"[PAD{i}")
9196+
tokens.append(f"[PAD{i}]")
91979197
toktypes.append(gguf.TokenType.UNUSED)
91989198
else:
91999199
token = reverse_vocab[i]
9200-
if token is None:
9201-
tokens.append(f"[PAD{i}")
9202-
else:
9203-
tokens.append(str(token))
9200+
tokens.append(token)
92049201
if i in special_tokens.values():
92059202
toktypes.append(gguf.TokenType.CONTROL)
92069203
else:

0 commit comments

Comments
 (0)