refactor(dev): improve debug log organization

- Move raw console logs to debug/console/ subdirectory - Rename pino
logs to .jsonl extension for clarity - Rename web.log to next.log to
distinguish from pino web.jsonl - Simplify log paths (removed
debug/proc/ nesting)

New structure: debug/ ├── cli.jsonl (CLI app, structured JSON) ├──
web.jsonl (Web app, structured JSON) └── console/ ├── next.log (Next.js
framework) ├── db.log (Database startup) ├── sdk.log (SDK build) └──
studio.log (Drizzle Studio)

Author: brandonkachen

cli/src/utils/logger.ts

@@ -1,4 +1,4 @@
-import { existsSync, mkdirSync, unlinkSync } from 'fs'
+import { appendFileSync, existsSync, mkdirSync, unlinkSync } from 'fs'
 import path, { dirname } from 'path'
 import { format as stringFormat } from 'util'
 
@@ -49,7 +49,7 @@ function setLogPath(p: string): void {
   const fileStream = pino.destination({
     dest: p, // absolute or relative file path
     mkdir: true, // create parent dirs if they don’t exist
-    sync: false, // set true if you *must* block on every write
+    sync: true, // set true if you *must* block on every write
   })
 
   pinoLogger = pino(
@@ -66,7 +66,7 @@ function setLogPath(p: string): void {
 
 export function clearLogFile(): void {
   const projectRoot = getProjectRoot()
-  const defaultLog = path.join(projectRoot, 'debug', 'cli.log')
+  const defaultLog = path.join(projectRoot, 'debug', 'cli.jsonl')
   const targets = new Set<string>()
 
   if (logPath) {
@@ -102,7 +102,7 @@ function sendAnalyticsAndLog(
 
     const logTarget =
       env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev'
-        ? path.join(projectRoot, 'debug', 'cli.log')
+        ? path.join(projectRoot, 'debug', 'cli.jsonl')
         : path.join(getCurrentChatDir(), 'log.jsonl')
 
     setLogPath(logTarget)
@@ -143,7 +143,22 @@ function sendAnalyticsAndLog(
     trackEvent(analyticsEventId, toTrack)
   }
 
-  if (pinoLogger !== undefined) {
+  // In dev mode, use appendFileSync for real-time logging (Bun has issues with pino sync)
+  // In prod mode, use pino for better performance
+  if (env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev' && logPath) {
+    const logEntry = JSON.stringify({
+      level: level.toUpperCase(),
+      timestamp: new Date().toISOString(),
+      ...loggerContext,
+      ...(includeData ? { data: normalizedData } : {}),
+      msg: stringFormat(normalizedMsg ?? '', ...args),
+    })
+    try {
+      appendFileSync(logPath, logEntry + '\n')
+    } catch {
+      // Ignore write errors
+    }
+  } else if (pinoLogger !== undefined) {
     const base = { ...loggerContext }
     const obj = includeData ? { ...base, data: normalizedData } : base
     pinoLogger[level](obj, normalizedMsg as any, ...args)

scripts/dev.sh

@@ -1,112 +1,100 @@
 #!/bin/bash
+# Development Environment Startup Script
+
+set -e
 
 # =============================================================================
-# Development Environment Startup Script
+# Configuration
 # =============================================================================
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
-export PATH="$PROJECT_ROOT/.bin:$PATH"
+LOG_DIR="$PROJECT_ROOT/debug/console"
+BUN="$PROJECT_ROOT/.bin/bun"
 
-LOG_DIR="$PROJECT_ROOT/debug/proc"
+export PATH="$PROJECT_ROOT/.bin:$PATH"
 mkdir -p "$LOG_DIR"
 
 # =============================================================================
 # UI Helpers
 # =============================================================================
 
-SPINNER_FRAMES=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏')
-
-# Print a success line
-ok() {
-    printf "  \033[32m✓\033[0m %-10s %s\033[K\n" "$1" "$2"
-}
-
-# Wait for a pattern in a log file, with spinner animation
-wait_for_log() {
-    local name="$1"
-    local pattern="$2"
-    local timeout="${3:-60}"
-    local frame=0
-    local elapsed=0
-    
-    printf "\033[?25l"  # Hide cursor
-    printf "  %s %-10s starting...\n" "${SPINNER_FRAMES[0]}" "$name"
-    
-    while ! grep -q "$pattern" "$LOG_DIR/$name.log" 2>/dev/null; do
-        printf "\033[1A"  # Move up one line
-        printf "  %s %-10s starting...\033[K\n" "${SPINNER_FRAMES[$frame]}" "$name"
-        
-        frame=$(( (frame + 1) % ${#SPINNER_FRAMES[@]} ))
-        sleep 0.1
-        elapsed=$(( elapsed + 1 ))
-        
-        if (( elapsed > timeout * 10 )); then
-            printf "\033[?25h"  # Show cursor
-            echo "Timeout waiting for $name" >&2
+SPINNER=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏')
+
+ok()   { printf "  \033[32m✓\033[0m %-10s %s\033[K\n" "$1" "$2"; }
+fail() { printf "  \033[31m✗\033[0m %-10s %s\033[K\n" "$1" "$2"; }
+
+# Wait for a condition with spinner animation
+# Usage: wait_for <name> <check_command> [timeout_seconds]
+wait_for() {
+    local name="$1" check="$2" timeout="${3:-60}"
+    local frame=0 elapsed=0
+
+    printf "\033[?25l"  # hide cursor
+    while ! eval "$check" >/dev/null 2>&1; do
+        printf "\r  %s %-10s starting..." "${SPINNER[$frame]}" "$name"
+        frame=$(( (frame + 1) % ${#SPINNER[@]} ))
+        sleep 0.5
+        elapsed=$((elapsed + 1))
+        if ((elapsed > timeout * 2)); then
+            printf "\033[?25h\n"
+            fail "$name" "timeout"
             return 1
         fi
     done
-    
-    printf "\033[1A"  # Move up one line
+    printf "\r"
     ok "$name" "ready!"
-    printf "\033[?25h"  # Show cursor
+    printf "\033[?25h"  # show cursor
 }
 
 # =============================================================================
 # Cleanup
 # =============================================================================
 
-kill_proc() {
-    local name="$1"
-    local pattern="$2"
-    if pkill -f "$pattern" 2>/dev/null; then
-        echo "[$(date '+%H:%M:%S')] Killed $name" >> "$LOG_DIR/$name.log"
-        return 0
-    fi
-    return 1
-}
-
 cleanup() {
-    printf "\033[?25h"  # Restore cursor
+    printf "\033[?25h"  # restore cursor
     echo ""
     echo "Shutting down..."
     echo ""
-    
-    kill_proc "web" 'bun.*--cwd web dev' || kill_proc "web" 'next-server'
-    ok "web" "stopped"
-    
-    kill_proc "studio" 'drizzle-kit studio' && ok "studio" "stopped"
-    kill_proc "sdk" 'bun.*--cwd sdk' && ok "sdk" "stopped"
-    
+    tmux kill-session -t codebuff-web 2>/dev/null && ok "web" "stopped"
+    pkill -f 'drizzle-kit studio' 2>/dev/null && ok "studio" "stopped"
+    pkill -f 'bun.*--cwd sdk' 2>/dev/null && ok "sdk" "stopped"
     echo ""
 }
 trap cleanup EXIT INT TERM
 
 # =============================================================================
-# Start Processes
+# Start Services
 # =============================================================================
 
 echo "Starting development environment..."
 echo ""
 
-# 1. Database (blocking)
-printf "  %s %-10s starting...\r" "${SPINNER_FRAMES[0]}" "db"
+# 1. Database (blocking - must complete before other services)
+printf "  %s %-10s starting...\r" "${SPINNER[0]}" "db"
 bun --cwd packages/internal db:start > "$LOG_DIR/db.log" 2>&1
 ok "db" "ready!"
 
-# 2. Background processes (fire and forget)
-bun --cwd sdk run build > "$LOG_DIR/sdk.log" 2>&1 &
-ok "sdk" "(background)"
-
+# 2. Background services (non-blocking)
+bun run --cwd sdk build > "$LOG_DIR/sdk.log" 2>&1 &
 bun --cwd packages/internal db:studio > "$LOG_DIR/studio.log" 2>&1 &
 ok "studio" "(background)"
 
-# 3. Web server (wait for ready)
-bun --cwd web dev > "$LOG_DIR/web.log" 2>&1 &
-wait_for_log "web" "Ready in" 60
+# 3. Web server (wait for health check)
+tmux kill-session -t codebuff-web 2>/dev/null || true
+pkill -f 'next-server' 2>/dev/null || true
+
+# Use unbuffer for real-time log output (creates pseudo-TTY to prevent buffering)
+# Strip ANSI escape codes with sed -l for line-buffered output
+if command -v unbuffer &>/dev/null; then
+    tmux new-session -d -s codebuff-web "cd $PROJECT_ROOT && unbuffer $BUN --cwd web dev 2>&1 | sed -l 's/\x1b\[[0-9;]*m//g' | tee $LOG_DIR/web.log"
+else
+    tmux new-session -d -s codebuff-web "cd $PROJECT_ROOT && $BUN --cwd web dev 2>&1 | sed -l 's/\x1b\[[0-9;]*m//g' | tee $LOG_DIR/web.log"
+fi
+
+wait_for "web" "curl -sf ${NEXT_PUBLIC_APP_URL}/api/healthz"
 
-# 4. CLI
+# 4. CLI (foreground - user interaction)
 echo ""
 echo "Starting CLI..."
 bun --cwd cli dev "$@"

sdk/package.json

@@ -39,7 +39,8 @@
     "fetch-ripgrep": "bun scripts/fetch-ripgrep.ts",
     "prepack": "bun run build",
     "typecheck": "tsc --noEmit -p .",
-    "test": "bun test"
+    "test": "bun test",
+    "dev": "bun run build"
   },
   "engines": {
     "node": ">=18.0.0"

web/src/util/logger.ts

@@ -1,4 +1,4 @@
-import fs from 'fs'
+import fs, { appendFileSync } from 'fs'
 import path from 'path'
 import { format } from 'util'
 
@@ -19,40 +19,32 @@ function getDebugDir(): string | null {
   if (debugDir !== undefined) {
     return debugDir
   }
-  debugDir = __dirname
-  while (true) {
-    if (fs.existsSync(path.join(debugDir, '.git'))) {
-      break
+  // Walk up from cwd to find the git root (where .git exists)
+  let dir = process.cwd()
+  while (dir !== path.dirname(dir)) {
+    if (fs.existsSync(path.join(dir, '.git'))) {
+      debugDir = path.join(dir, 'debug')
+      return debugDir
     }
-    const parent = path.dirname(debugDir)
-    if (parent === debugDir) {
-      debugDir = null
-      console.error('Failed to find git root directory for logger')
-      break
-    }
-    debugDir = parent
-  }
-
-  if (debugDir) {
-    debugDir = path.join(debugDir, 'debug')
+    dir = path.dirname(dir)
   }
-
+  debugDir = null
+  console.error('Failed to find git root directory for logger')
   return debugDir
 }
 
-setLocalDebugDir: if (
+// Initialize debug directory in dev environment
+if (
   env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev' &&
   process.env.CODEBUFF_GITHUB_ACTIONS !== 'true'
 ) {
-  const debugDir = getDebugDir()
-  if (!debugDir) {
-    break setLocalDebugDir
-  }
-
-  try {
-    fs.mkdirSync(debugDir, { recursive: true })
-  } catch (err) {
-    console.error('Failed to create debug directory:', err)
+  const dir = getDebugDir()
+  if (dir) {
+    try {
+      fs.mkdirSync(dir, { recursive: true })
+    } catch {
+      // Ignore errors when creating debug directory
+    }
   }
 }
 
@@ -67,12 +59,10 @@ const pinoLogger = pino(
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   },
   debugDir
-    ? pino.transport({
-        target: 'pino/file',
-        options: {
-          destination: path.join(debugDir, 'web.log'),
-        },
-        level: 'debug',
+    ? pino.destination({
+        dest: path.join(debugDir, 'web.jsonl'),
+        mkdir: true,
+        sync: true, // sync writes for real-time logging
       })
     : undefined,
 )
@@ -108,18 +98,43 @@ function splitAndLog(
   })
 }
 
-export const logger: Record<LogLevel, pino.LogFn> =
-  env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev'
-    ? pinoLogger
-    : (Object.fromEntries(
-        loggingLevels.map((level) => {
-          return [
-            level,
-            (data: any, msg?: string, ...args: any[]) =>
-              splitAndLog(level, data, msg, ...args),
-          ]
-        }),
-      ) as Record<LogLevel, pino.LogFn>)
+// In dev mode, use appendFileSync for real-time file logging (Bun has issues with pino sync)
+// Also output to console via pinoLogger so logs remain visible in the terminal
+function logWithSync(level: LogLevel, data: any, msg?: string, ...args: any[]): void {
+  const formattedMsg = format(msg ?? '', ...args)
+  
+  if (env.NEXT_PUBLIC_CB_ENVIRONMENT === 'dev') {
+    // Write to file for real-time logging
+    if (debugDir) {
+      const logEntry = JSON.stringify({
+        level: level.toUpperCase(),
+        timestamp: new Date().toISOString(),
+        ...(data && typeof data === 'object' ? data : { data }),
+        msg: formattedMsg,
+      })
+      try {
+        appendFileSync(path.join(debugDir, 'web.jsonl'), logEntry + '\n')
+      } catch {
+        // Ignore write errors
+      }
+    }
+    // Also output to console for interactive debugging
+    pinoLogger[level](data, msg, ...args)
+  } else {
+    // In prod, use pino with splitAndLog for large payloads
+    splitAndLog(level, data, msg, ...args)
+  }
+}
+
+export const logger: Record<LogLevel, pino.LogFn> = Object.fromEntries(
+  loggingLevels.map((level) => {
+    return [
+      level,
+      (data: any, msg?: string, ...args: any[]) =>
+        logWithSync(level, data, msg, ...args),
+    ]
+  }),
+) as Record<LogLevel, pino.LogFn>
 
 export function loggerWithContext(
   context: ParamsOf<LoggerWithContextFn>,