Update to reduce memory realloc (mlc-ai#18)

tqchen · web-flow · commit 22b89324a0ee · 2023-10-28T11:25:53.000-04:00
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
@@ -81,12 +81,9 @@ impl TokenizerWrapper {
     }
 
     pub fn encode(&mut self, text: &str, add_special_tokens: bool) {
-        self.encode_ids = Vec::from(
-            self.tokenizer
-                .encode(text, add_special_tokens)
-                .unwrap()
-                .get_ids(),
-        );
+        let encoded = self.tokenizer.encode(text, add_special_tokens).unwrap();
+        self.encode_ids.resize(encoded.len(), 0);
+        self.encode_ids.copy_from_slice(encoded.get_ids());
     }
 
     pub fn decode(&mut self, ids: &[u32], skip_special_tokens: bool) {

Original file line number	Diff line number	Diff line change
`@@ -81,12 +81,9 @@ impl TokenizerWrapper {`
`81`	`81`	`}`
`82`	`82`
`83`	`83`	`pub fn encode(&mut self, text: &str, add_special_tokens: bool) {`
`84`		`- self.encode_ids = Vec::from(`
`85`		`- self.tokenizer`
`86`		`- .encode(text, add_special_tokens)`
`87`		`- .unwrap()`
`88`		`- .get_ids(),`
`89`		`- );`
	`84`	`+ let encoded = self.tokenizer.encode(text, add_special_tokens).unwrap();`
	`85`	`+ self.encode_ids.resize(encoded.len(), 0);`
	`86`	`+ self.encode_ids.copy_from_slice(encoded.get_ids());`
`90`	`87`	`}`
`91`	`88`
`92`	`89`	`pub fn decode(&mut self, ids: &[u32], skip_special_tokens: bool) {`