diff --git a/packages/navie/package.json b/packages/navie/package.json
index abefaae61e..f7e8fdbc31 100644
--- a/packages/navie/package.json
+++ b/packages/navie/package.json
@@ -44,6 +44,7 @@
     "@langchain/google-vertexai-web": "^0.1.0",
     "@langchain/openai": "^0.2.7",
     "fast-xml-parser": "^4.4.0",
+    "js-tiktoken": "^1.0.18",
     "js-yaml": "^4.1.0",
     "jsdom": "^16.6.0",
     "langchain": "^0.2.16",
diff --git a/packages/navie/src/services/openai-completion-service.ts b/packages/navie/src/services/openai-completion-service.ts
index d9ad77f63e..db03d4eda6 100644
--- a/packages/navie/src/services/openai-completion-service.ts
+++ b/packages/navie/src/services/openai-completion-service.ts
@@ -1,11 +1,11 @@
 import { isNativeError } from 'node:util/types';
 
+import { getModelNameForTiktoken } from '@langchain/core/language_models/base';
 import { ChatOpenAI } from '@langchain/openai';
 import type { ChatCompletion, ChatCompletionChunk } from 'openai/resources/index';
 import { z } from 'zod';
 import { zodResponseFormat } from 'openai/helpers/zod';
 import { warn } from 'console';
-import Message from '../message';
 import CompletionService, {
   Completion,
   CompletionRetries,
@@ -15,9 +15,16 @@ import CompletionService, {
   Usage,
 } from './completion-service';
 import Trajectory from '../lib/trajectory';
+import Message, { CHARACTERS_PER_TOKEN } from '../message';
 import { APIError } from 'openai';
 import MessageTokenReducerService from './message-token-reducer-service';
 
+// For some reason this doesn't work as import
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const { getEncodingNameForModel } = require('js-tiktoken/lite') as {
+  getEncodingNameForModel: (x: string) => string;
+};
+
 /*
   Generated on https://openai.com/api/pricing/ with
   Object.fromEntries(
@@ -183,6 +190,13 @@ export default class OpenAICompletionService implements CompletionService {
       streaming: true,
       onFailedAttempt,
     });
+    try {
+      getEncodingNameForModel(getModelNameForTiktoken(modelName));
+    } catch {
+      warn(`Unknown model ${modelName}, using estimated token count`);
+      this.model.getNumTokens = (c) =>
+        Promise.resolve(typeof c === 'string' ? c.length / CHARACTERS_PER_TOKEN : 0);
+    }
   }
   model: ChatOpenAI;
 
diff --git a/yarn.lock b/yarn.lock
index f934656664..8c7a9547da 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -475,6 +475,7 @@ __metadata:
     eslint-plugin-unicorn: ^39.0.0
     fast-xml-parser: ^4.4.0
     jest: ^29.7.0
+    js-tiktoken: ^1.0.18
     js-yaml: ^4.1.0
     jsdom: ^16.6.0
     langchain: ^0.2.16
@@ -28738,12 +28739,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"js-tiktoken@npm:^1.0.12":
-  version: 1.0.14
-  resolution: "js-tiktoken@npm:1.0.14"
+"js-tiktoken@npm:^1.0.12, js-tiktoken@npm:^1.0.18":
+  version: 1.0.18
+  resolution: "js-tiktoken@npm:1.0.18"
   dependencies:
     base64-js: ^1.5.1
-  checksum: 0feb0f4186221f9db4cf7224313e83b8b869460f63c40634b47946d8312c2e05a66a2fd256ecd577f913563dce05f6d52fa9e3eba6a5e9270f82a4630a77fc2c
+  checksum: ee86872b8ef77c72952905e9c321e00a4005883e9807a7ed83539b1bfd625e01000e640b18071f5072a2d4e423e66d29f8239eb3b7d743d8d5203ec8e83b78d3
   languageName: node
   linkType: hard