Merge pull request #88 from github/dependabot/cargo/tiktoken-rs-0.9

Update tiktoken-rs requirement from 0.7 to 0.9

Author: CleanCut

crates/bpe-openai/Cargo.toml

@@ -21,7 +21,7 @@ unicode-normalization = "0.1"
 
 [dev-dependencies]
 bpe = { version = "0.2", path = "../bpe", features = ["rand"] }
-tiktoken-rs = "0.7"
+tiktoken-rs = "0.9"
 
 [build-dependencies]
 base64 = "0.22"

crates/bpe/Cargo.toml

@@ -26,7 +26,7 @@ serde = { version = "1", features = ["derive"] }
 
 [dev-dependencies]
 bpe = { path = "." }
-tiktoken-rs = "0.7"
+tiktoken-rs = "0.9"
 
 [package.metadata.docs.rs]
 all-features = true

crates/bpe/benchmarks/Cargo.toml

@@ -22,5 +22,5 @@ bpe = { path = "../../bpe", features = ["rand", "tiktoken"] }
 bpe-openai = { path = "../../bpe-openai" }
 criterion = "0.7"
 rand = "0.9"
-tiktoken-rs = "0.7"
+tiktoken-rs = "0.9"
 tokenizers = { version = "0.22", features = ["http"] }

crates/bpe/tests/Cargo.toml

@@ -7,4 +7,4 @@ bpe = { path = "../../bpe", features = ["rand"] }
 bpe-openai = { path = "../../bpe-openai" }
 itertools = "0.14"
 rand = "0.9"
-tiktoken-rs = "0.7"
+tiktoken-rs = "0.9"