File tree Expand file tree Collapse file tree 1 file changed +5
-8
lines changed
Expand file tree Collapse file tree 1 file changed +5
-8
lines changed Original file line number Diff line number Diff line change @@ -36,12 +36,11 @@ class HFTokenizer : public Tokenizer {
3636 return ret;
3737 }
3838
39- // use i32 to be consistent with sentencepiece
40- std::vector<int32_t > Encode (const std::string& text) final {
41- return Encode (text, false );
42- }
39+ // use i32 to be consistent with sentencepiece
40+ std::vector<int32_t > Encode (const std::string& text) final { return Encode (text, false ); }
4341
44- std::vector<std::vector<int32_t >> EncodeBatch (const std::vector<std::string>& texts, bool add_special_tokens) final {
42+ std::vector<std::vector<int32_t >> EncodeBatch (const std::vector<std::string>& texts,
43+ bool add_special_tokens) {
4544 std::vector<const char *> texts_raw;
4645 std::vector<size_t > seq_lens;
4746 size_t num_seqs = texts.size ();
@@ -78,9 +77,7 @@ class HFTokenizer : public Tokenizer {
7877 return std::string (data, len);
7978 }
8079
81- std::string Decode (const std::vector<int32_t >& ids) final {
82- return Decode (ids, false );
83- }
80+ std::string Decode (const std::vector<int32_t >& ids) final { return Decode (ids, false ); }
8481
8582 size_t GetVocabSize () final {
8683 size_t size;
You can’t perform that action at this time.
0 commit comments