diff --git a/README.md b/README.md index 1d1c69f..17b8b42 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,82 @@ const App = () => { }; ``` +### Tokenization + +Convert text into tokens using the model's tokenizer. + +#### Class + +```typescript +import { CactusLM } from 'cactus-react-native'; + +const cactusLM = new CactusLM(); + +const result = await cactusLM.tokenize({ text: 'Hello, World!' }); +console.log('Token IDs:', result.tokens); +``` + +#### Hook + +```tsx +import { useCactusLM } from 'cactus-react-native'; + +const App = () => { + const cactusLM = useCactusLM(); + + const handleTokenize = async () => { + const result = await cactusLM.tokenize({ text: 'Hello, World!' }); + console.log('Token IDs:', result.tokens); + }; + + return