Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ jobs:
uses: tauri-apps/tauri-action@v0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
with:
projectPath: widget
tagName: ${{ github.ref_name }}
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ Thumbs.db
.env
.env.local

# Local Tauri updater signing keys
.tauri/

# Claude Code local state
.claude/
CLAUDE.md
Expand Down
33 changes: 33 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Repository Guidelines

## Project Structure & Module Organization

TokenBBQ is a TypeScript ESM CLI and dashboard. Core source lives in `src/`: `index.ts` is the CLI entry point, `loaders/` normalizes usage data from supported tools, `aggregator.ts` prepares summaries, `pricing.ts` handles model pricing, and `server.ts`/`dashboard.ts` serve the web UI. Tests sit next to source as `*.test.ts`; loader fixtures live in `src/loaders/__fixtures__/`. Build helpers are in `scripts/`. The desktop widget is a separate Tauri/Vite app under `widget/`, with Rust backend code in `widget/src-tauri/` and frontend assets in `widget/src/`. Generated output belongs in `dist/`.

## Build, Test, and Development Commands

- `npm install`: install root dependencies.
- `npm run dev`: inline required assets, then run the CLI locally through `tsx`.
- `npm run lint`: run TypeScript type checking with `tsc --noEmit`.
- `npm run test`: run Node's test runner against `src/**/*.test.ts`.
- `npm run build`: produce the publishable CLI bundle in `dist/`.
- `npm run widget:install`: install widget dependencies.
- `npm run widget:dev`: build the sidecar and launch the Tauri widget.
- `npm run widget:build`: build the CLI, sidecar, and desktop widget package locally. Updater artifacts are disabled (via `widget/src-tauri/tauri.dev.conf.json`), so no signing key is needed.
- `npm run widget:build:release`: full signed build with updater artifacts. Requires `TAURI_SIGNING_PRIVATE_KEY` (and `TAURI_SIGNING_PRIVATE_KEY_PASSWORD`); release CI builds this path via `tauri-action`.

## Coding Style & Naming Conventions

Use strict TypeScript, ESM `import`/`export`, and Node 20+ APIs. Keep module names lowercase and descriptive, for example `event-merge.ts` or `platform-paths.ts`. Tests should mirror the target module name, such as `pricing.test.ts`. Prefer small functions with explicit types at module boundaries. The codebase currently uses two-space indentation in TypeScript files; avoid unrelated formatting churn.

## Testing Guidelines

Tests use `node:test` and `node:assert/strict`, executed via `scripts/run-tests.mjs` so glob expansion works across supported Node versions. Add focused tests beside changed code when modifying aggregation, pricing, persistence, or loader behavior. For new loaders, include representative fixture data under `src/loaders/__fixtures__/` when practical and verify missing data directories return empty results rather than throwing.

## Commit & Pull Request Guidelines

Recent history follows Conventional Commit-style subjects, for example `fix(audit): ...`, `docs: ...`, and `chore(release): ...`. Keep subjects imperative and scoped when useful. Pull requests should fill out `.github/PULL_REQUEST_TEMPLATE.md`: explain what changed, why it is needed, confirm `npm run build`, note loader registration changes, and update `README.md` for user-facing behavior. Include screenshots or recordings for widget/dashboard UI changes.

## Security & Configuration Tips

Do not commit local usage databases, credentials, or generated `dist/` artifacts unless release packaging requires them. Preserve cross-platform path handling; CI runs Linux, macOS, and Windows.
18 changes: 9 additions & 9 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@
"build:sidecar": "node scripts/inline-wasm.mjs && node scripts/inline-dashboard-icon.mjs && node scripts/build-sidecar.mjs",
"widget:install": "npm install --prefix widget",
"widget:dev": "node scripts/build-sidecar.mjs --skip-if-no-bun && npm run --prefix widget tauri dev",
"widget:build": "npm run build && npm run build:sidecar && npm run --prefix widget tauri build"
"widget:build": "npm run build && npm run build:sidecar && npm run --prefix widget tauri build -- --config src-tauri/tauri.dev.conf.json",
"widget:build:release": "npm run build && npm run build:sidecar && npm run --prefix widget tauri build"
},
"dependencies": {
"@hono/node-server": "^1.13.0",
Expand Down
14 changes: 8 additions & 6 deletions src/loaders/amp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { glob } from 'tinyglobby';
import type { UnifiedTokenEvent } from '../types.js';
import { isValidTimestamp } from '../types.js';
import { getPlatformDataDirs } from '../platform-paths.js';
import { loadCachedFileEvents } from './cache.js';

function getAmpPath(): string | null {
const envPath = (process.env.AMP_DATA_DIR ?? '').trim();
Expand All @@ -31,21 +32,21 @@ export async function loadAmpEvents(): Promise<UnifiedTokenEvent[]> {
if (!existsSync(threadsDir)) return [];

const files = await glob('**/*.json', { cwd: threadsDir, absolute: true });
const events: UnifiedTokenEvent[] = [];

for (const file of files) {
const events = await loadCachedFileEvents('amp', files, async (file) => {
const fileEvents: UnifiedTokenEvent[] = [];
let content: string;
try {
content = await readFile(file, 'utf-8');
} catch {
continue;
return fileEvents;
}

let thread: Record<string, unknown>;
try {
thread = JSON.parse(content);
} catch {
continue;
return fileEvents;
}

const threadId = String(thread.id ?? path.basename(file, '.json'));
Expand Down Expand Up @@ -85,7 +86,7 @@ export async function loadAmpEvents(): Promise<UnifiedTokenEvent[]> {
}
}

events.push({
fileEvents.push({
source: 'amp',
timestamp: evt.timestamp,
sessionId: threadId,
Expand All @@ -100,7 +101,8 @@ export async function loadAmpEvents(): Promise<UnifiedTokenEvent[]> {
costUSD: 0,
});
}
}
return fileEvents;
});

events.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
return events;
Expand Down
52 changes: 52 additions & 0 deletions src/loaders/cache.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import { afterEach, beforeEach, describe, test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { loadCachedFileEvents } from './cache.js';
import type { UnifiedTokenEvent } from '../types.js';

let tmp: string;

beforeEach(() => {
tmp = mkdtempSync(path.join(tmpdir(), 'tbq-loader-cache-'));
process.env.TOKENBBQ_DATA_DIR = path.join(tmp, 'data');
});

afterEach(() => {
delete process.env.TOKENBBQ_DATA_DIR;
delete process.env.TOKENBBQ_DISABLE_LOADER_CACHE;
rmSync(tmp, { recursive: true, force: true });
});

function event(sessionId: string): UnifiedTokenEvent {
return {
source: 'codex',
timestamp: '2026-04-22T14:02:11.812Z',
sessionId,
model: 'gpt-5',
tokens: { input: 1, output: 2, cacheCreation: 0, cacheRead: 0, reasoning: 0 },
costUSD: 0,
};
}

describe('loadCachedFileEvents', () => {
test('reuses parsed events while file mtime and size are unchanged', async () => {
const file = path.join(tmp, 'session.jsonl');
writeFileSync(file, 'first', 'utf-8');
let parses = 0;

const parseFile = async (target: string): Promise<UnifiedTokenEvent[]> => {
parses++;
return [event(readFileSync(target, 'utf-8'))];
};

assert.equal((await loadCachedFileEvents('codex', [file], parseFile))[0]?.sessionId, 'first');
assert.equal((await loadCachedFileEvents('codex', [file], parseFile))[0]?.sessionId, 'first');
assert.equal(parses, 1);

writeFileSync(file, 'second-value', 'utf-8');
assert.equal((await loadCachedFileEvents('codex', [file], parseFile))[0]?.sessionId, 'second-value');
assert.equal(parses, 2);
});
});
134 changes: 134 additions & 0 deletions src/loaders/cache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { getStoreDir } from '../store.js';
import type { Source, UnifiedTokenEvent } from '../types.js';

const CACHE_VERSION = 1;

interface FileCacheEntry<T> {
mtimeMs: number;
size: number;
records: T[];
}

interface LoaderCacheFile<T> {
v: number;
files: Record<string, FileCacheEntry<T>>;
}

function cacheEnabled(): boolean {
return process.env.TOKENBBQ_DISABLE_LOADER_CACHE !== '1';
}

function cachePath(source: Source): string {
return path.join(getStoreDir(), 'cache', 'loaders', `${source}.json`);
}

function isValidEvent(v: unknown): v is UnifiedTokenEvent {
if (!v || typeof v !== 'object') return false;
const e = v as Record<string, unknown>;
const tokens = e.tokens as Record<string, unknown> | undefined;
return (
typeof e.source === 'string' &&
typeof e.timestamp === 'string' &&
typeof e.sessionId === 'string' &&
typeof e.model === 'string' &&
!!tokens &&
typeof tokens === 'object'
);
}

function isValidEntry<T>(v: unknown, isValidRecord: (v: unknown) => v is T): v is FileCacheEntry<T> {
if (!v || typeof v !== 'object') return false;
const e = v as Record<string, unknown>;
return (
typeof e.mtimeMs === 'number' &&
typeof e.size === 'number' &&
Array.isArray(e.records) &&
e.records.every(isValidRecord)
);
}

async function readCache<T>(
source: Source,
isValidRecord: (v: unknown) => v is T,
): Promise<LoaderCacheFile<T>> {
try {
const parsed = JSON.parse(await readFile(cachePath(source), 'utf-8')) as unknown;
if (!parsed || typeof parsed !== 'object') return { v: CACHE_VERSION, files: {} };
const obj = parsed as Record<string, unknown>;
if (obj.v !== CACHE_VERSION || !obj.files || typeof obj.files !== 'object') {
return { v: CACHE_VERSION, files: {} };
}
const files: Record<string, FileCacheEntry<T>> = {};
for (const [file, entry] of Object.entries(obj.files as Record<string, unknown>)) {
if (isValidEntry(entry, isValidRecord)) files[file] = entry;
}
return { v: CACHE_VERSION, files };
} catch {
return { v: CACHE_VERSION, files: {} };
}
}

async function writeCache<T>(source: Source, cache: LoaderCacheFile<T>): Promise<void> {
const file = cachePath(source);
const dir = path.dirname(file);
const tmp = path.join(dir, `${path.basename(file)}.${process.pid}.${Date.now()}.tmp`);
try {
await mkdir(dir, { recursive: true });
await writeFile(tmp, JSON.stringify(cache), 'utf-8');
await rename(tmp, file);
} catch {
// Loader caches are performance-only. A failed write must never make scans fail.
}
}

export async function loadCachedFileRecords<T>(
source: Source,
files: string[],
parseFile: (file: string) => Promise<T[]>,
isValidRecord: (v: unknown) => v is T,
): Promise<T[]> {
if (!cacheEnabled()) {
const records: T[] = [];
for (const file of files) records.push(...await parseFile(file));
return records;
}

const cache = await readCache(source, isValidRecord);
const nextFiles: Record<string, FileCacheEntry<T>> = {};
const records: T[] = [];

for (const file of files) {
let info: { mtimeMs: number; size: number };
try {
const s = await stat(file);
info = { mtimeMs: s.mtimeMs, size: s.size };
} catch {
continue;
}

const hit = cache.files[file];
if (hit && hit.mtimeMs === info.mtimeMs && hit.size === info.size) {
nextFiles[file] = hit;
records.push(...hit.records);
continue;
}

const parsed = await parseFile(file);
const entry = { ...info, records: parsed };
nextFiles[file] = entry;
records.push(...parsed);
}

await writeCache(source, { v: CACHE_VERSION, files: nextFiles });
return records;
}

export async function loadCachedFileEvents(
source: Source,
files: string[],
parseFile: (file: string) => Promise<UnifiedTokenEvent[]>,
): Promise<UnifiedTokenEvent[]> {
return loadCachedFileRecords(source, files, parseFile, isValidEvent);
}
Loading
Loading