feat(tokens): replace char/4 heuristic with js-tiktoken (cl100k_base)

Centralizes token estimation across 14 files through
src/core/cache/token-estimator.ts and packages/sdk/src/token-estimator.ts.
Lazy-loads cl100k_base encoder with char/4 fallback if WASM fails.

Also ports context-budget hook to codex-sm exit handler for
compact/restart nudges matching Claude Code behavior.

Author: StackMemory Bot (CLI)

package-lock.json

@@ -179,6 +179,7 @@
     "helmet": "^8.1.0",
     "ignore": "^7.0.5",
     "inquirer": "^9.3.8",
+    "js-tiktoken": "^1.0.21",
     "msgpackr": "^1.10.1",
     "node-pty": "^1.1.0",
     "open": "^11.0.0",

package.json

@@ -22,6 +22,7 @@
   },
   "dependencies": {
     "better-sqlite3": "^11.8.1",
+    "js-tiktoken": "^1.0.21",
     "js-yaml": "^4.1.0",
     "zod": "^3.24.2"
   },

packages/sdk/package-lock.json

@@ -273,9 +273,13 @@ describe('StackMemory SDK', () => {
   });
 
   describe('pure functions', () => {
-    it('estimateTokens approximates', () => {
-      expect(estimateTokens('hello')).toBe(2);
+    it('estimateTokens returns positive count for non-empty strings', () => {
+      expect(estimateTokens('hello')).toBeGreaterThan(0);
       expect(estimateTokens('')).toBe(0);
+      // Longer text should produce more tokens
+      expect(estimateTokens('hello world foo bar baz')).toBeGreaterThan(
+        estimateTokens('hello')
+      );
     });
 
     it('hashContent is deterministic', () => {

packages/sdk/package.json

@@ -15,7 +15,11 @@ export { ProvenanceStore } from './provenance.js';
 
 // Pure functions
 export { scoreConfidence } from './confidence-scorer.js';
-export { estimateTokens, hashContent } from './token-estimator.js';
+export {
+  estimateTokens,
+  isTiktokenActive,
+  hashContent,
+} from './token-estimator.js';
 
 // Types
 export type {

packages/sdk/src/__tests__/sdk.test.ts

@@ -1,15 +1,46 @@
 /**
  * Token estimation and content hashing utilities.
+ *
+ * Uses js-tiktoken (cl100k_base) for accurate counts.
+ * Falls back to chars/4 heuristic if encoder fails to load.
  */
 
 import { createHash } from 'crypto';
+import { createRequire } from 'module';
 
-/** Estimate token count using chars/4 approximation. */
+type Encoder = { encode: (text: string) => number[] };
+
+let encoder: Encoder | null = null;
+let initAttempted = false;
+
+function getEncoder(): Encoder | null {
+  if (initAttempted) return encoder;
+  initAttempted = true;
+  try {
+    const require = createRequire(import.meta.url);
+    const tiktoken = require('js-tiktoken');
+    encoder = tiktoken.getEncoding('cl100k_base');
+  } catch {
+    encoder = null;
+  }
+  return encoder;
+}
+
+/** Estimate token count. Accurate when tiktoken loads, heuristic otherwise. */
 export function estimateTokens(content: string): number {
   if (!content) return 0;
+  const enc = getEncoder();
+  if (enc) {
+    return enc.encode(content).length;
+  }
   return Math.ceil(content.length / 4);
 }
 
+/** Whether tiktoken is active (for diagnostics). */
+export function isTiktokenActive(): boolean {
+  return getEncoder() !== null;
+}
+
 /** SHA-256 hex digest of content. */
 export function hashContent(content: string): string {
   return createHash('sha256').update(content).digest('hex');

packages/sdk/src/index.ts

@@ -253,6 +253,54 @@ class CodexSM {
     }
   }
 
+  /**
+   * Emit context budget advice based on tool-call count from checkpoint state.
+   * Mirrors the Claude Code context-budget.js hook.
+   */
+  private emitContextBudgetAdvice(): void {
+    const COMPACT_SUGGEST = 50;
+    const COMPACT_STRONG = 65;
+    const RESTART_RECOMMEND = 80;
+
+    try {
+      const stateFile = path.join(
+        os.homedir(),
+        '.stackmemory',
+        `checkpoint-state-${this.config.instanceId}.json`
+      );
+      if (!fs.existsSync(stateFile)) return;
+
+      const state = JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
+      const cwd = process.cwd();
+      const toolCount = state.projects?.[cwd]?.toolCount || 0;
+
+      if (toolCount >= RESTART_RECOMMEND) {
+        console.log(
+          chalk.yellow(
+            `[CONTEXT_BUDGET] ${toolCount} tool calls (~150K+ tokens). ` +
+              `Recommend: save context then start fresh session.`
+          )
+        );
+      } else if (toolCount >= COMPACT_STRONG) {
+        console.log(
+          chalk.yellow(
+            `[CONTEXT_BUDGET] ${toolCount} tool calls (~100-130K tokens). ` +
+              `Context heavy — consider compacting or restarting.`
+          )
+        );
+      } else if (toolCount >= COMPACT_SUGGEST) {
+        console.log(
+          chalk.gray(
+            `[CONTEXT_BUDGET] ${toolCount} tool calls (~80-100K tokens). ` +
+              `Context getting heavy.`
+          )
+        );
+      }
+    } catch {
+      // Silent — never block exit
+    }
+  }
+
   private loadContext(): void {
     if (!this.config.contextEnabled) return;
     try {
@@ -591,6 +639,9 @@ class CodexSM {
         // Non-fatal: don't block exit
       }
 
+      // Context budget check
+      this.emitContextBudgetAdvice();
+
       if (this.config.tracingEnabled) {
         const summary = trace.getExecutionSummary();
         console.log();

packages/sdk/src/token-estimator.ts

@@ -9,6 +9,7 @@
  */
 
 import { spawn, execSync, type ChildProcess } from 'child_process';
+import { estimateTokens } from '../../core/cache/token-estimator.js';
 import {
   appendFileSync,
   existsSync,
@@ -2295,7 +2296,7 @@ export class Conductor {
                 }
                 if (block.type === 'text' && block.text) {
                   const text = block.text as string;
-                  run.tokensUsed += Math.ceil(text.length / 4);
+                  run.tokensUsed += estimateTokens(text);
                   turnTextParts.push(text);
                 }
               }
@@ -2489,7 +2490,7 @@ export class Conductor {
 
             // Estimate tokens from message sizes
             if (msg.method === 'item/text' && params?.text) {
-              run.tokensUsed += Math.ceil((params.text as string).length / 4);
+              run.tokensUsed += estimateTokens(params.text as string);
             }
 
             // Update agent status file periodically (every 5 tool calls)

src/cli/codex-sm.ts

@@ -1,18 +1,48 @@
 /**
- * Token estimation and content hashing utilities
+ * Token estimation and content hashing utilities.
+ *
+ * Uses js-tiktoken (cl100k_base) for accurate counts.
+ * Falls back to chars/4 heuristic if encoder fails to load.
  */
 
 import { createHash } from 'crypto';
+import { createRequire } from 'module';
+
+type Encoder = { encode: (text: string) => number[] };
+
+let encoder: Encoder | null = null;
+let initAttempted = false;
+
+function getEncoder(): Encoder | null {
+  if (initAttempted) return encoder;
+  initAttempted = true;
+  try {
+    const require = createRequire(import.meta.url);
+    const tiktoken = require('js-tiktoken');
+    encoder = tiktoken.getEncoding('cl100k_base');
+  } catch {
+    encoder = null;
+  }
+  return encoder;
+}
 
 /**
- * Estimate token count using chars/4 approximation.
- * Good enough for cache dedup -- no tiktoken dependency needed.
+ * Estimate token count. Accurate when tiktoken loads, heuristic otherwise.
  */
 export function estimateTokens(content: string): number {
   if (!content) return 0;
+  const enc = getEncoder();
+  if (enc) {
+    return enc.encode(content).length;
+  }
   return Math.ceil(content.length / 4);
 }
 
+/** Whether tiktoken is active (for diagnostics). */
+export function isTiktokenActive(): boolean {
+  return getEncoder() !== null;
+}
+
 /**
  * SHA-256 hex digest of content for content-addressable lookup.
  */