parse-claude-data.js

#!/usr/bin/env node
/**
 * Claude Code Usage Data Parser v2
 * Reads ~/.claude/projects/ recursively, parses all JSONL session files,
 * outputs a consolidated usage-stats.json with accurate cost modeling.
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const readline = require('readline');

const CLAUDE_DIR = path.join(os.homedir(), '.claude');
const OUTPUT_FILE = path.join(__dirname, 'usage-stats.json');

// Cost rates per million tokens (Anthropic API pricing)
// Cache read = 10% of base input price
// Cache creation = 125% of base input price
const COST_RATES = {
  opus:   { input: 15,    output: 75,   cacheRead: 1.5,   cacheWrite: 18.75  },
  sonnet: { input: 3,     output: 15,   cacheRead: 0.3,   cacheWrite: 3.75   },
  haiku:  { input: 0.25,  output: 1.25, cacheRead: 0.025, cacheWrite: 0.3125 },
};

function classifyModel(modelStr) {
  if (!modelStr || modelStr === '<synthetic>') return 'synthetic';
  const m = modelStr.toLowerCase();
  if (m.includes('opus'))   return 'opus';
  if (m.includes('sonnet')) return 'sonnet';
  if (m.includes('haiku'))  return 'haiku';
  return 'unknown';
}

function classifySessionType(totalTokens) {
  if (totalTokens > 500000) return 'Heavy';
  if (totalTokens >= 50000) return 'Medium';
  return 'Light';
}

function projectNameFromCwd(cwd) {
  if (!cwd) return 'Unknown';
  const parts = cwd.replace(/\\/g, '/').split('/').filter(Boolean);
  return parts.slice(-2).join('/');
}

function projectKeyFromCwd(cwd) {
  if (!cwd) return 'unknown';
  return cwd.replace(/\\/g, '/');
}

function findJsonlFiles(dir) {
  const results = [];
  let entries;
  try { entries = fs.readdirSync(dir, { withFileTypes: true }); }
  catch { return results; }
  for (const entry of entries) {
    const fullPath = path.join(dir, entry.name);
    if (entry.isDirectory()) {
      results.push(...findJsonlFiles(fullPath));
    } else if (entry.name.endsWith('.jsonl')) {
      results.push(fullPath);
    }
  }
  return results;
}

async function parseJsonlFile(filePath) {
  const messages = [];
  let errors = 0;
  const stream = fs.createReadStream(filePath, { encoding: 'utf-8' });
  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });

  for await (const line of rl) {
    if (!line.trim()) continue;
    try {
      messages.push(JSON.parse(line));
    } catch {
      errors++;
    }
  }
  return { messages, errors };
}

// Extract file paths from Bash commands
function extractBashFiles(cmd) {
  const files = new Set();
  if (!cmd || typeof cmd !== 'string') return files;

  // Common file-referencing patterns in bash commands
  const patterns = [
    // Redirect output: > file, >> file, 2> file
    /[12]?>>\s*"?([^\s";&|]+)"?/g,
    /[12]?>\s*"?([^\s";&|]+)"?/g,
    // cat/touch/rm/cp/mv/chmod with file args
    /(?:cat|touch|rm|cp|mv|chmod|mkdir|head|tail|less|more|wc|sort|uniq)\s+(?:-[^\s]*\s+)*"?([^\s";&|>]+\.[a-zA-Z0-9]+)"?/g,
    // python/node/bash running a script
    /(?:python3?|node|bash|sh)\s+(?:-[^\s]*\s+)*"?([^\s";&|]+\.(?:py|js|sh|ts))"?/g,
    // tee to file
    /tee\s+(?:-a\s+)?"?([^\s";&|]+)"?/g,
  ];

  for (const regex of patterns) {
    let m;
    while ((m = regex.exec(cmd)) !== null) {
      const f = m[1];
      // Skip common non-file patterns
      if (f && !f.startsWith('-') && !f.startsWith('/dev/') && !f.startsWith('/tmp/') && f !== '.' && f !== '..') {
        files.add(f);
      }
    }
  }
  return files;
}

// Count code lines in Bash heredocs and inline scripts
function extractBashCode(cmd) {
  let added = 0;
  if (!cmd || typeof cmd !== 'string') return added;

  // Heredoc: << 'EOF' ... EOF, << EOF ... EOF, << 'PYEOF' ... PYEOF
  const heredocRegex = /<<\s*'?(\w+)'?\s*\n([\s\S]*?)\n\1/g;
  let m;
  while ((m = heredocRegex.exec(cmd)) !== null) {
    added += m[2].split('\n').length;
  }

  // python -c "..." or node -e "..." inline scripts (multiline)
  const inlineRegex = /(?:python3?|node)\s+(?:-[ce])\s+"((?:[^"\\]|\\.)*)"/g;
  while ((m = inlineRegex.exec(cmd)) !== null) {
    added += m[1].split('\n').length;
  }

  return added;
}

function extractContentData(content) {
  let codeAdded = 0, codeRemoved = 0;
  const files = new Set();

  // Handle string content (rare but possible)
  if (typeof content === 'string') {
    const codeBlockRegex = /```[\s\S]*?```/g;
    let match;
    while ((match = codeBlockRegex.exec(content)) !== null) {
      const lines = match[0].split('\n').slice(1, -1);
      for (const l of lines) {
        if (l.startsWith('+')) codeAdded++;
        else if (l.startsWith('-')) codeRemoved++;
        else codeAdded++;
      }
    }
    return { codeAdded, codeRemoved, files };
  }

  if (!Array.isArray(content)) return { codeAdded, codeRemoved, files };

  for (const block of content) {
    if (!block || typeof block !== 'object') continue;

    // Text blocks with code fences
    if (block.type === 'text' && typeof block.text === 'string') {
      const codeBlockRegex = /```[\s\S]*?```/g;
      let match;
      while ((match = codeBlockRegex.exec(block.text)) !== null) {
        const lines = match[0].split('\n').slice(1, -1);
        for (const l of lines) {
          if (l.startsWith('+')) codeAdded++;
          else if (l.startsWith('-')) codeRemoved++;
          else codeAdded++;
        }
      }
    }

    if (block.type === 'tool_use') {
      const inp = block.input || {};

      // File path extraction from all file-referencing tools
      if (inp.file_path) files.add(inp.file_path);
      if (inp.path) files.add(inp.path);

      if (block.name === 'Write' && typeof inp.content === 'string') {
        codeAdded += inp.content.split('\n').length;
      }
      if (block.name === 'Edit') {
        if (typeof inp.new_string === 'string') codeAdded += inp.new_string.split('\n').length;
        if (typeof inp.old_string === 'string') codeRemoved += inp.old_string.split('\n').length;
      }

      // Extract files and code from Bash commands
      if (block.name === 'Bash' && typeof inp.command === 'string') {
        const bashFiles = extractBashFiles(inp.command);
        for (const f of bashFiles) files.add(f);
        codeAdded += extractBashCode(inp.command);
      }
    }
  }
  return { codeAdded, codeRemoved, files };
}

async function main() {
  console.log('Scanning', CLAUDE_DIR, '...');

  if (!fs.existsSync(CLAUDE_DIR)) {
    console.error('~/.claude/ not found');
    fs.writeFileSync(OUTPUT_FILE, JSON.stringify({ error: 'no_data', message: 'No Claude Code data found at ~/.claude/' }));
    return;
  }

  const jsonlFiles = findJsonlFiles(path.join(CLAUDE_DIR, 'projects'));
  console.log(`Found ${jsonlFiles.length} JSONL files`);

  const sessions = {};      // sessionId -> session data (main session only)
  const subagentSessions = {}; // unique key per subagent file -> data
  let totalErrors = 0;
  let totalLinesProcessed = 0;

  for (const filePath of jsonlFiles) {
    const { messages, errors } = await parseJsonlFile(filePath);
    totalErrors += errors;
    totalLinesProcessed += messages.length + errors;

    const isSubagentFile = filePath.includes('subagent');

    // Subagent files get their own session keyed by file path
    // Main files get keyed by sessionId
    const targetMap = isSubagentFile ? subagentSessions : sessions;

    for (const msg of messages) {
      const sessionId = msg.sessionId;
      if (!sessionId) continue;

      const sessionKey = isSubagentFile ? filePath : sessionId;
      const type = msg.type;
      const timestamp = msg.timestamp;

      if (!targetMap[sessionKey]) {
        targetMap[sessionKey] = {
          id: sessionId,
          firstTimestamp: null,
          lastTimestamp: null,
          userMessages: 0,
          assistantMessages: 0,
          syntheticMessages: 0,
          totalMessages: 0,
          inputTokens: 0,
          outputTokens: 0,
          cacheCreationTokens: 0,
          cacheReadTokens: 0,
          models: new Set(),
          realModels: new Set(),
          files: new Set(),
          codeAdded: 0,
          codeRemoved: 0,
          cwd: msg.cwd || '',
          isSubagent: isSubagentFile,
          entrypoint: msg.entrypoint || 'cli',
          modelTokens: {},
        };
      }

      const session = targetMap[sessionKey];

      // Track entrypoint — first explicit value wins
      if (msg.entrypoint && !session._entrypointLocked) {
        session.entrypoint = msg.entrypoint;
        session._entrypointLocked = true;
      }

      // Track timestamps
      if (timestamp) {
        const ts = typeof timestamp === 'number' ? new Date(timestamp).toISOString() : timestamp;
        if (!session.firstTimestamp || ts < session.firstTimestamp) session.firstTimestamp = ts;
        if (!session.lastTimestamp || ts > session.lastTimestamp) session.lastTimestamp = ts;
      }

      if (type === 'user') {
        session.userMessages++;
        session.totalMessages++;
      }

      if (type === 'assistant') {
        const message = msg.message || {};
        const model = message.model || '';
        const modelClass = classifyModel(model);
        const isSynthetic = modelClass === 'synthetic';

        if (model) session.models.add(model);
        if (model && !isSynthetic) session.realModels.add(model);

        if (isSynthetic) {
          session.syntheticMessages++;
        } else {
          session.assistantMessages++;
        }
        session.totalMessages++;

        // Token usage — only from real API calls
        const usage = message.usage || {};
        const inTok = usage.input_tokens || 0;
        const outTok = usage.output_tokens || 0;
        const cacheCr = usage.cache_creation_input_tokens || 0;
        const cacheRd = usage.cache_read_input_tokens || 0;

        session.inputTokens += inTok;
        session.outputTokens += outTok;
        session.cacheCreationTokens += cacheCr;
        session.cacheReadTokens += cacheRd;

        // Track tokens per model class for accurate cost calc
        if (!isSynthetic && (inTok || outTok || cacheCr || cacheRd)) {
          if (!session.modelTokens[modelClass]) {
            session.modelTokens[modelClass] = { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 };
          }
          session.modelTokens[modelClass].input += inTok;
          session.modelTokens[modelClass].output += outTok;
          session.modelTokens[modelClass].cacheCreation += cacheCr;
          session.modelTokens[modelClass].cacheRead += cacheRd;
        }

        // Code lines & files from content
        const { codeAdded, codeRemoved, files } = extractContentData(message.content);
        session.codeAdded += codeAdded;
        session.codeRemoved += codeRemoved;
        for (const f of files) session.files.add(f);
      }
    }
  }

  // Build output
  const sessionList = [];
  let totalUserMsgs = 0, totalAssistantMsgs = 0, totalSyntheticMsgs = 0, totalMsgs = 0;
  let totalInputTokens = 0, totalOutputTokens = 0;
  let totalCacheCreation = 0, totalCacheRead = 0;
  let totalCodeAdded = 0, totalCodeRemoved = 0;
  const allFiles = new Set();
  const dailyMessages = {};
  const weeklyMessages = {};
  const monthlyMessages = {};
  const modelTokens = {};   // modelClass -> { input, output, cacheCreation, cacheRead }
  const projectStats = {};  // projectKey -> { name, sessions, messages, tokens, cost, models }
  const entrypointCounts = {};

  // Merge both maps for aggregation
  const allSessions = { ...sessions };
  for (const [key, s] of Object.entries(subagentSessions)) {
    allSessions['sub:' + key] = s;
  }

  for (const [id, s] of Object.entries(allSessions)) {
    for (const f of s.files) allFiles.add(f);
    totalCodeAdded += s.codeAdded;
    totalCodeRemoved += s.codeRemoved;

    totalUserMsgs += s.userMessages;
    totalAssistantMsgs += s.assistantMessages;
    totalSyntheticMsgs += s.syntheticMessages;
    totalMsgs += s.totalMessages;
    totalInputTokens += s.inputTokens;
    totalOutputTokens += s.outputTokens;
    totalCacheCreation += s.cacheCreationTokens;
    totalCacheRead += s.cacheReadTokens;

    // Entrypoint tracking
    entrypointCounts[s.entrypoint] = (entrypointCounts[s.entrypoint] || 0) + 1;

    const totalTokens = s.inputTokens + s.outputTokens + s.cacheCreationTokens + s.cacheReadTokens;

    // Daily/weekly/monthly aggregation
    if (s.firstTimestamp) {
      const day = s.firstTimestamp.substring(0, 10);
      dailyMessages[day] = (dailyMessages[day] || 0) + s.totalMessages;

      // ISO week (Monday-based)
      const d = new Date(s.firstTimestamp);
      const dayOfWeek = (d.getUTCDay() + 6) % 7;
      const monday = new Date(d);
      monday.setUTCDate(d.getUTCDate() - dayOfWeek);
      const weekKey = monday.toISOString().substring(0, 10);
      weeklyMessages[weekKey] = (weeklyMessages[weekKey] || 0) + s.totalMessages;

      const monthKey = s.firstTimestamp.substring(0, 7);
      monthlyMessages[monthKey] = (monthlyMessages[monthKey] || 0) + s.totalMessages;
    }

    // Model token aggregation (per-message tracking, not per-session split)
    for (const [cls, tok] of Object.entries(s.modelTokens)) {
      if (!modelTokens[cls]) modelTokens[cls] = { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 };
      modelTokens[cls].input += tok.input;
      modelTokens[cls].output += tok.output;
      modelTokens[cls].cacheCreation += tok.cacheCreation;
      modelTokens[cls].cacheRead += tok.cacheRead;
    }

    // Project aggregation
    const projKey = projectKeyFromCwd(s.cwd);
    if (!projectStats[projKey]) {
      projectStats[projKey] = {
        name: projectNameFromCwd(s.cwd),
        path: s.cwd,
        sessions: 0,
        subagentSessions: 0,
        messages: 0,
        inputTokens: 0,
        outputTokens: 0,
        cacheCreationTokens: 0,
        cacheReadTokens: 0,
        totalTokens: 0,
        cost: 0,
        models: new Set(),
        files: new Set(),
        codeAdded: 0,
        codeRemoved: 0,
        modelTokens: {},  // per-model tokens for accurate cost
      };
    }
    const proj = projectStats[projKey];
    if (s.isSubagent) proj.subagentSessions++;
    else proj.sessions++;
    proj.messages += s.totalMessages;
    proj.inputTokens += s.inputTokens;
    proj.outputTokens += s.outputTokens;
    proj.cacheCreationTokens += s.cacheCreationTokens;
    proj.cacheReadTokens += s.cacheReadTokens;
    proj.totalTokens += totalTokens;
    proj.codeAdded += s.codeAdded;
    proj.codeRemoved += s.codeRemoved;
    for (const m of s.realModels) proj.models.add(m);
    for (const f of s.files) proj.files.add(f);
    // Aggregate model tokens for project-level cost
    for (const [cls, tok] of Object.entries(s.modelTokens)) {
      if (!proj.modelTokens[cls]) proj.modelTokens[cls] = { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 };
      proj.modelTokens[cls].input += tok.input;
      proj.modelTokens[cls].output += tok.output;
      proj.modelTokens[cls].cacheCreation += tok.cacheCreation;
      proj.modelTokens[cls].cacheRead += tok.cacheRead;
    }

    const durationMs = (s.firstTimestamp && s.lastTimestamp)
      ? new Date(s.lastTimestamp) - new Date(s.firstTimestamp) : 0;

    sessionList.push({
      id,
      date: s.firstTimestamp ? s.firstTimestamp.substring(0, 10) : 'unknown',
      firstTimestamp: s.firstTimestamp,
      lastTimestamp: s.lastTimestamp,
      durationMinutes: Math.round(durationMs / 60000),
      userMessages: s.userMessages,
      assistantMessages: s.assistantMessages,
      syntheticMessages: s.syntheticMessages,
      totalMessages: s.totalMessages,
      inputTokens: s.inputTokens,
      outputTokens: s.outputTokens,
      cacheCreationTokens: s.cacheCreationTokens,
      cacheReadTokens: s.cacheReadTokens,
      totalTokens,
      models: [...s.realModels],
      modelClasses: [...s.realModels].map(classifyModel),
      sessionType: classifySessionType(totalTokens),
      filesCount: s.files.size,
      codeAdded: s.codeAdded,
      codeRemoved: s.codeRemoved,
      cwd: s.cwd,
      isSubagent: s.isSubagent,
      entrypoint: s.entrypoint,
    });
  }

  // Sort sessions by date descending
  sessionList.sort((a, b) => (b.firstTimestamp || '').localeCompare(a.firstTimestamp || ''));

  // Session type breakdown
  const sessionTypes = { Heavy: { count: 0, totalTokens: 0 }, Medium: { count: 0, totalTokens: 0 }, Light: { count: 0, totalTokens: 0 } };
  for (const s of sessionList) {
    const st = sessionTypes[s.sessionType];
    if (st) { st.count++; st.totalTokens += s.totalTokens; }
  }

  // Cost estimation with proper cache pricing
  function calcCost(modelClass, tokens) {
    const rates = COST_RATES[modelClass] || COST_RATES.sonnet;
    const inputCost = (tokens.input / 1e6) * rates.input;
    const outputCost = (tokens.output / 1e6) * rates.output;
    const cacheReadCost = (tokens.cacheRead / 1e6) * rates.cacheRead;
    const cacheWriteCost = (tokens.cacheCreation / 1e6) * rates.cacheWrite;
    return {
      inputCost: Math.round(inputCost * 100) / 100,
      outputCost: Math.round(outputCost * 100) / 100,
      cacheReadCost: Math.round(cacheReadCost * 100) / 100,
      cacheWriteCost: Math.round(cacheWriteCost * 100) / 100,
      totalCost: Math.round((inputCost + outputCost + cacheReadCost + cacheWriteCost) * 100) / 100,
    };
  }

  const costByModel = {};
  for (const [cls, tokens] of Object.entries(modelTokens)) {
    if (cls === 'synthetic') continue; // skip synthetic
    const costs = calcCost(cls, tokens);
    costByModel[cls] = { ...tokens, ...costs };
  }

  const totalCost = Object.values(costByModel).reduce((sum, m) => sum + m.totalCost, 0);

  // Per-project cost calculation — uses actual model-level tokens
  const projectList = Object.values(projectStats).map(p => {
    let estCost = 0;
    for (const [cls, tok] of Object.entries(p.modelTokens)) {
      if (cls === 'synthetic') continue;
      const c = calcCost(cls, tok);
      estCost += c.totalCost;
    }
    estCost = Math.round(estCost * 100) / 100;
    return {
      name: p.name,
      path: p.path,
      sessions: p.sessions,
      subagentSessions: p.subagentSessions,
      messages: p.messages,
      totalTokens: p.totalTokens,
      inputTokens: p.inputTokens,
      outputTokens: p.outputTokens,
      cacheCreationTokens: p.cacheCreationTokens,
      cacheReadTokens: p.cacheReadTokens,
      estimatedCost: estCost,
      models: [...p.models].map(classifyModel).filter((v, i, a) => a.indexOf(v) === i),
      filesCount: p.files.size,
      codeAdded: p.codeAdded,
      codeRemoved: p.codeRemoved,
    };
  });
  projectList.sort((a, b) => b.totalTokens - a.totalTokens);

  // Model distribution for chart
  const modelDistribution = {};
  for (const [cls, tok] of Object.entries(modelTokens)) {
    if (cls === 'synthetic') continue;
    const total = tok.input + tok.output + tok.cacheCreation + tok.cacheRead;
    modelDistribution[cls] = total;
  }

  // Token breakdown for stacked bar
  const tokenBreakdown = {
    input: totalInputTokens,
    output: totalOutputTokens,
    cacheCreation: totalCacheCreation,
    cacheRead: totalCacheRead,
  };

  const activeDays = Object.keys(dailyMessages).length;
  const totalTokensAll = totalInputTokens + totalOutputTokens + totalCacheCreation + totalCacheRead;

  const output = {
    generatedAt: new Date().toISOString(),
    summary: {
      totalMessages: totalMsgs,
      userMessages: totalUserMsgs,
      assistantMessages: totalAssistantMsgs,
      syntheticMessages: totalSyntheticMsgs,
      totalSessions: sessionList.filter(s => !s.isSubagent).length,
      subagentSessions: sessionList.filter(s => s.isSubagent).length,
      activeDays,
      messagesPerDay: activeDays > 0 ? Math.round(totalMsgs / activeDays) : 0,
      totalFiles: allFiles.size,
      codeAdded: totalCodeAdded,
      codeRemoved: totalCodeRemoved,
      totalInputTokens,
      totalOutputTokens,
      totalCacheCreationTokens: totalCacheCreation,
      totalCacheReadTokens: totalCacheRead,
      totalTokens: totalTokensAll,
      totalCostEstimate: Math.round(totalCost * 100) / 100,
    },
    sessionTypes,
    costByModel,
    tokenBreakdown,
    modelDistribution,
    dailyMessages,
    weeklyMessages,
    monthlyMessages,
    entrypoints: entrypointCounts,
    projects: projectList,
    sessions: sessionList,
    parsing: {
      totalFiles: jsonlFiles.length,
      totalLinesProcessed,
      malformedLines: totalErrors,
    },
  };

  const jsonStr = JSON.stringify(output, null, 2);
  fs.writeFileSync(OUTPUT_FILE, jsonStr);
  console.log(`\nDone! Wrote ${OUTPUT_FILE}`);
  console.log(`  Sessions: ${sessionList.length} (${sessionList.filter(s => s.isSubagent).length} subagent)`);
  console.log(`  Messages: ${totalMsgs.toLocaleString()} (${totalSyntheticMsgs} synthetic)`);
  console.log(`  Tokens: input=${totalInputTokens.toLocaleString()} output=${totalOutputTokens.toLocaleString()} cacheRead=${totalCacheRead.toLocaleString()} cacheWrite=${totalCacheCreation.toLocaleString()}`);
  console.log(`  Malformed lines skipped: ${totalErrors}`);
  console.log(`  Est. API cost: $${totalCost.toFixed(2)}`);
  console.log(`  Entrypoints: ${JSON.stringify(entrypointCounts)}`);
  console.log(`  Projects: ${projectList.length}`);

  // Generate dashboard.html with embedded data for file:// access (offline/no-server)
  const htmlPath = path.join(__dirname, 'index.html');
  const dashboardPath = path.join(__dirname, 'dashboard.html');
  if (fs.existsSync(htmlPath)) {
    let html = fs.readFileSync(htmlPath, 'utf-8');
    // Remove any previously embedded data (in case index.html was modified)
    html = html.replace(/\nconst EMBEDDED_DATA = [\s\S]*?;\n/, '\n');
    // Write clean index.html back
    fs.writeFileSync(htmlPath, html);
    // Inject data into dashboard.html copy
    const tag = '<script>';
    const idx = html.lastIndexOf(tag);
    if (idx !== -1) {
      html = html.slice(0, idx + tag.length) + '\nconst EMBEDDED_DATA = ' + JSON.stringify(output) + ';\n' + html.slice(idx + tag.length);
      fs.writeFileSync(dashboardPath, html);
      console.log(`  Generated dashboard.html (${(html.length/1024).toFixed(0)} KB) — open directly, no server needed`);
    }
  }
}

main().catch(err => {
  console.error('Fatal error:', err);
  process.exit(1);
});