chore(cli): add file logging for background errors

Author: Jannchie

.github/workflows/publish.yml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
     inputs:
       dist_tag:
-        description: 'npm dist-tag channel (latest / beta / next). Git tag is auto-selected.'
+        description: npm dist-tag channel (latest / beta / next). Git tag is auto-selected.
         required: false
         default: latest
 

packages/cli/src/adapters/opencode.ts

@@ -42,15 +42,35 @@ async function parseOpenCodeSessionFile(
   const events: CanonicalEvent[] = []
 
   try {
+    // Older OpenCode databases predate the `path` column on session.
+    // Probe schema first and only select columns that exist.
+    const sessionCols = new Set(
+      (db.prepare('PRAGMA table_info(session)').all() as Array<{ name: string }>)
+        .map(row => row.name),
+    )
+    const hasDirectory = sessionCols.has('directory')
+    const hasPath = sessionCols.has('path')
+    const hasArchived = sessionCols.has('time_archived')
+    const selectCols = ['id', 'title', 'time_created']
+    if (hasDirectory) {
+      selectCols.push('directory')
+    }
+    if (hasPath) {
+      selectCols.push('path')
+    }
+    if (hasArchived) {
+      selectCols.push('time_archived')
+    }
+
     const sessions = db.prepare(
-      'SELECT id, directory, path, title, time_created, time_archived FROM session WHERE time_created IS NOT NULL ORDER BY time_created',
+      `SELECT ${selectCols.join(', ')} FROM session WHERE time_created IS NOT NULL ORDER BY time_created`,
     ).all() as Array<{
       id: string
-      directory: string | null
-      path: string | null
+      directory?: string | null
+      path?: string | null
       title: string
       time_created: number
-      time_archived: number | null
+      time_archived?: number | null
     }>
 
     for (const session of sessions) {

packages/cli/src/cli.ts

@@ -7,7 +7,6 @@ import type {
 } from '@codetime/shared'
 import type { BackfillSourceDefinition } from './lib/backfill.js'
 import type { BackfillImportCounts, BackfillIncrementalState, BackfillSourceFile, ParsedArgs, RunContext, SyncLocalLock, SyncLocalTriggerState, WritableLike } from './lib/types.js'
-import { BACKFILL_STATE_SCHEMA_VERSION } from './lib/types.js'
 import { spawn } from 'node:child_process'
 import { mkdir, rm, stat, writeFile } from 'node:fs/promises'
 import os from 'node:os'
@@ -34,6 +33,7 @@ import { defaultMachineName, ensureLocalMachineId, readConfig, writeConfig } fro
 import { DEFAULT_API_URL, DEFAULT_BACKFILL_BATCH_BYTES, DEFAULT_BACKFILL_BATCH_SIZE, DEFAULT_HOOK_SYNC_MIN_INTERVAL_SECONDS, PACKAGE_VERSION } from './lib/constants.js'
 import { isPlainObject, numberOption, stringOption, valuesOption } from './lib/fields.js'
 import { countDirectoryEntries, listJsonlFiles, pathExists, readJsonIfExists } from './lib/fs.js'
+import { logError } from './lib/logger.js'
 import { ProgressBar } from './lib/progress.js'
 import {
   deleteMachine,
@@ -43,6 +43,7 @@ import {
   renameMachine,
   resolveRemote,
 } from './lib/remote.js'
+import { BACKFILL_STATE_SCHEMA_VERSION } from './lib/types.js'
 
 // ── Registry ──
 
@@ -98,6 +99,7 @@ export async function run(argv: string[], context: Partial<RunContext> = {}): Pr
   }
   catch (error) {
     write(ctx.stderr, `${(error as Error).message}\n`)
+    await logError('cli', error, { argv })
     return 1
   }
 }
@@ -288,22 +290,33 @@ async function installCommand(options: ParsedArgs, ctx: RunContext, registry: Ad
 }
 
 async function hookCommand(options: ParsedArgs, ctx: RunContext): Promise<number> {
-  const agent = requiredOption(options, 'agent')
-  const payload = await readHookPayload(ctx.stdin)
-  const enrichment = await enrichFromTranscript(payload, claudeUsageFromMessage)
-  const events = hookEventsFromPayload(agent, payload, options, enrichment, { tokenUsageFromPayload })
-  assertValidEvents(events)
+  const home = resolveHome(options, ctx)
+  try {
+    const agent = requiredOption(options, 'agent')
+    const payload = await readHookPayload(ctx.stdin)
+    const enrichment = await enrichFromTranscript(payload, claudeUsageFromMessage)
+    const events = hookEventsFromPayload(agent, payload, options, enrichment, { tokenUsageFromPayload })
+    assertValidEvents(events)
+
+    if (options['dry-run']) {
+      write(ctx.stdout, `${JSON.stringify(events.length === 1 ? events[0] : events, null, 2)}\n`)
+      return 0
+    }
 
-  if (options['dry-run']) {
-    write(ctx.stdout, `${JSON.stringify(events.length === 1 ? events[0] : events, null, 2)}\n`)
+    return await syncLocalTriggerCommand({
+      ...options,
+      agent,
+      'min-interval': stringOption(options['min-interval']) || String(DEFAULT_HOOK_SYNC_MIN_INTERVAL_SECONDS),
+    }, ctx)
+  }
+  catch (error) {
+    // Hooks run inside the user's agent (Claude Code, Codex, etc).
+    // Bubbling an error there spams the user with stderr; persist to the
+    // log file and exit 0 so the agent isn't disturbed.
+    await logError('hook', error, { agent: stringOption(options.agent) }, home)
+    debug(ctx, `[codetime] hook failed: ${(error as Error).message}\n`)
     return 0
   }
-
-  return syncLocalTriggerCommand({
-    ...options,
-    agent,
-    'min-interval': stringOption(options['min-interval']) || String(DEFAULT_HOOK_SYNC_MIN_INTERVAL_SECONDS),
-  }, ctx)
 }
 
 async function syncLocalTriggerCommand(options: ParsedArgs, ctx: RunContext, _registry?: AdapterRegistry): Promise<number> {
@@ -369,6 +382,12 @@ async function syncLocalRunnerCommand(options: ParsedArgs, ctx: RunContext, _reg
     }, ctx)
     return exitCode
   }
+  catch (error) {
+    // The runner is spawned detached with stdio: 'ignore', so the stack
+    // trace would be lost otherwise. Persist it so users can diagnose.
+    await logError('sync-local-runner', error, { home }, home)
+    throw error
+  }
   finally {
     const nextState = await readSyncLocalTriggerState(statePath)
     nextState.lastStartedAt = state.lastStartedAt
@@ -506,23 +525,40 @@ async function createBackfillEventsFromDefs(
   overrideFiles?: string[],
 ): Promise<CanonicalEvent[]> {
   const events: CanonicalEvent[] = []
+  const home = resolveHome(options, ctx)
 
+  // Same isolation policy as the import path: one source blowing up
+  // (e.g. older opencode SQLite schemas) must not poison the whole plan.
   for (const item of sourceDefs) {
     const parser = registry.getParser(item.id)
     if (!parser) {
       continue
     }
 
-    const sourceFiles = await listBackfillSourceFiles(item, options, ctx)
-    const files = overrideFiles ?? sourceFiles.map(f => f.path)
+    let files: string[]
+    try {
+      const sourceFiles = await listBackfillSourceFiles(item, options, ctx)
+      files = overrideFiles ?? sourceFiles.map(f => f.path)
+    }
+    catch (error) {
+      await logError('backfill.listFiles', error, { source: item.id, phase: 'plan' }, home)
+      debug(ctx, `[codetime] skip ${item.id} in plan: list files failed: ${(error as Error).message}\n`)
+      continue
+    }
 
     for (const filePath of files) {
-      const parsed = await parser(filePath, options)
-      for (const event of parsed) {
-        if (matchesBackfillFilters(event, options)) {
-          events.push(event)
+      try {
+        const parsed = await parser(filePath, options)
+        for (const event of parsed) {
+          if (matchesBackfillFilters(event, options)) {
+            events.push(event)
+          }
         }
       }
+      catch (error) {
+        await logError('backfill.parse', error, { source: item.id, file: filePath, phase: 'plan' }, home)
+        debug(ctx, `[codetime] skip ${item.id} file ${filePath} in plan: ${(error as Error).message}\n`)
+      }
     }
   }
 
@@ -747,30 +783,59 @@ async function collectCanonicalEvents(
 }> {
   const selectedFilesBySource = new Map<BackfillSourceId, BackfillSourceFile[]>()
   const canonicalEvents: CanonicalEvent[] = []
+  const home = resolveHome(options, ctx)
 
   for (const item of sourceDefs) {
     const parser = registry.getParser(item.id)
     if (!parser) {
       continue
     }
-    const sourceFiles = await listBackfillSourceFiles(item, options, ctx)
-    const selectedFiles = selectBackfillFilesForImport(sourceFiles, incrementalState?.sources[item.id]?.watermarkTs)
+    // Per-source isolation: a broken parser or missing history dir for
+    // one source (e.g. opencode's older schemas) must not abort the
+    // whole run. Failures land in ~/.codetime/logs/cli.log; the
+    // watermark stays unadvanced because selectedFilesBySource only
+    // gets populated on success.
+    let selectedFiles: BackfillSourceFile[]
+    try {
+      const sourceFiles = await listBackfillSourceFiles(item, options, ctx)
+      selectedFiles = selectBackfillFilesForImport(sourceFiles, incrementalState?.sources[item.id]?.watermarkTs)
+    }
+    catch (error) {
+      await logError('backfill.listFiles', error, { source: item.id }, home)
+      debug(ctx, `[codetime] skip ${item.id}: list files failed: ${(error as Error).message}\n`)
+      continue
+    }
     selectedFilesBySource.set(item.id, selectedFiles)
     const filePaths = selectedFiles.map(f => f.path)
     const sourceEvents: CanonicalEvent[] = []
+    let sourceFailed = false
 
     const bar = options.json ? undefined : new ProgressBar(ctx.stdout, `${item.id.padEnd(12)}`)
     bar?.init(filePaths.length, `0 events`)
     for (let fi = 0; fi < filePaths.length; fi += 1) {
-      const parsed = await parser(filePaths[fi], options)
-      for (const event of parsed) {
-        if (matchesBackfillFilters(event, options)) {
-          sourceEvents.push(event)
+      try {
+        const parsed = await parser(filePaths[fi], options)
+        for (const event of parsed) {
+          if (matchesBackfillFilters(event, options)) {
+            sourceEvents.push(event)
+          }
         }
       }
+      catch (error) {
+        sourceFailed = true
+        await logError('backfill.parse', error, { source: item.id, file: filePaths[fi] }, home)
+        debug(ctx, `[codetime] skip ${item.id} file ${filePaths[fi]}: ${(error as Error).message}\n`)
+      }
       bar?.tick(`${fi + 1}/${filePaths.length} files, ${sourceEvents.length} events`)
     }
-    bar?.finalize(`${sourceEvents.length} events`)
+    bar?.finalize(`${sourceEvents.length} events${sourceFailed ? ' (partial — see logs)' : ''}`)
+
+    // If any file failed to parse, drop this source from the watermark
+    // update set so we retry on the next run instead of marking it as
+    // fully imported.
+    if (sourceFailed) {
+      selectedFilesBySource.delete(item.id)
+    }
 
     for (const event of sourceEvents) canonicalEvents.push(event)
   }

packages/cli/src/lib/logger.ts

@@ -0,0 +1,77 @@
+// File logger for the CLI. Hooks and the background sync-local runner
+// detach with stdio: 'ignore', so stderr is gone. Persist failures here
+// so users can diagnose problems after the fact.
+import { appendFile, mkdir, rename, stat } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import path from 'node:path'
+
+const MAX_BYTES = 1 * 1024 * 1024 // 1 MiB; rotate to .1 when exceeded
+
+export function logDir(home: string = homedir()): string {
+  return path.join(home, '.codetime', 'logs')
+}
+
+export function logPath(home: string = homedir(), name = 'cli.log'): string {
+  return path.join(logDir(home), name)
+}
+
+function serializeError(error: unknown): Record<string, unknown> {
+  if (error instanceof Error) {
+    return { message: error.message, stack: error.stack, name: error.name }
+  }
+  return { value: String(error) }
+}
+
+async function rotateIfNeeded(file: string): Promise<void> {
+  try {
+    const info = await stat(file)
+    if (info.size > MAX_BYTES) {
+      await rename(file, `${file}.1`).catch(() => {})
+    }
+  }
+  catch { /* missing file is fine */ }
+}
+
+export interface LogEntry {
+  scope: string
+  message?: string
+  error?: unknown
+  meta?: Record<string, unknown>
+  level?: 'info' | 'warn' | 'error'
+}
+
+export async function writeLog(entry: LogEntry, home: string = homedir(), fileName = 'cli.log'): Promise<void> {
+  try {
+    const dir = logDir(home)
+    await mkdir(dir, { recursive: true })
+    const file = logPath(home, fileName)
+    await rotateIfNeeded(file)
+    const record: Record<string, unknown> = {
+      ts: new Date().toISOString(),
+      level: entry.level || (entry.error ? 'error' : 'info'),
+      scope: entry.scope,
+      pid: process.pid,
+    }
+    if (entry.message) {
+      record.message = entry.message
+    }
+    if (entry.meta) {
+      Object.assign(record, entry.meta)
+    }
+    if (entry.error !== undefined) {
+      record.error = serializeError(entry.error)
+    }
+    await appendFile(file, `${JSON.stringify(record)}\n`)
+  }
+  catch {
+    // Never let logging crash the caller.
+  }
+}
+
+export async function logError(scope: string, error: unknown, meta: Record<string, unknown> = {}, home: string = homedir()): Promise<void> {
+  await writeLog({ scope, error, meta, level: 'error' }, home)
+}
+
+export async function logInfo(scope: string, message: string, meta: Record<string, unknown> = {}, home: string = homedir()): Promise<void> {
+  await writeLog({ scope, message, meta, level: 'info' }, home)
+}