dataset.ts

/**
 * Open Dataset Collection Engine
 *
 * Opt-in data collection for building an open source research dataset.
 * Researchers who enable `contribute_to_dataset: true` in their requests
 * have their (anonymized) interaction data stored for the community.
 *
 * Stored data:
 * - Messages sent and received (no API keys, no IPs)
 * - AutoTune parameters and context detection results
 * - Model used and response metadata
 * - User feedback/ratings
 * - Parseltongue and STM pipeline metadata
 *
 * Privacy guarantees:
 * - Strictly opt-in per request
 * - No PII: API keys, IPs, and auth tokens are NEVER stored
 * - Dataset is exportable via GET /v1/dataset/export
 * - Caller can request deletion via DELETE /v1/dataset/:id
 *
 * Persistence: auto-publishes to HuggingFace when buffer fills up.
 */

import { randomUUID } from 'crypto'
import { registerDatasetStore, checkDatasetThreshold } from './hf-publisher'

// ── Types ────────────────────────────────────────────────────────────

export interface DatasetEntry {
  id: string
  timestamp: number

  // Request metadata
  endpoint: string  // which API endpoint was called
  model: string
  mode: 'standard' | 'ultraplinian'

  // Messages (stripped of system prompts to avoid leaking custom prompts)
  messages: Array<{ role: string; content: string }>
  response: string

  // AutoTune data
  autotune?: {
    strategy: string
    detected_context: string
    confidence: number
    params: Record<string, number>
    reasoning: string
  }

  // Parseltongue data
  parseltongue?: {
    triggers_found: string[]
    technique_used: string
    transformations_count: number
  }

  // STM data
  stm?: {
    modules_applied: string[]
  }

  // ULTRAPLINIAN race data
  ultraplinian?: {
    tier: string
    models_queried: string[]
    winner_model: string
    all_scores: Array<{ model: string; score: number; duration_ms: number; success: boolean }>
    total_duration_ms: number
  }

  // Feedback (added later via POST /v1/feedback if user rates)
  feedback?: {
    rating: 1 | -1
    heuristics?: {
      response_length: number
      repetition_score: number
      vocabulary_diversity: number
    }
  }
}

// ── In-Memory Store with Auto-Publish ────────────────────────────────
// Buffer auto-flushes to HuggingFace when it hits 80% capacity.
// Falls back to FIFO eviction if HF publishing is not configured.

let dataset: DatasetEntry[] = []
const MAX_ENTRIES = 10000 // Cap to prevent unbounded memory growth

// Track how many entries have been flushed so we can use index-based draining
// instead of copying the entire array on each snapshot.
let datasetFlushIndex = 0

// Register with HF publisher so it can snapshot/clear our buffer
registerDatasetStore({
  snapshot: () => dataset.slice(datasetFlushIndex),
  clear: (count: number) => {
    datasetFlushIndex += count
    // Compact the array when more than half has been drained to free memory
    if (datasetFlushIndex > dataset.length / 2) {
      dataset = dataset.slice(datasetFlushIndex)
      datasetFlushIndex = 0
    }
  },
})

// ── PII Scrubber ────────────────────────────────────────────────────
// Best-effort removal of common PII patterns before data hits the buffer.
// Not perfect — users are warned to avoid submitting PII regardless.

const PII_PATTERNS: Array<{ pattern: RegExp; replacement: string }> = [
  // Email addresses
  { pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, replacement: '[EMAIL_REDACTED]' },
  // Phone numbers (US and international formats)
  { pattern: /(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}/g, replacement: '[PHONE_REDACTED]' },
  { pattern: /\+[0-9]{1,3}[-.\s]?[0-9]{4,14}/g, replacement: '[PHONE_REDACTED]' },
  // SSN (US)
  { pattern: /\b[0-9]{3}[-\s]?[0-9]{2}[-\s]?[0-9]{4}\b/g, replacement: '[SSN_REDACTED]' },
  // Credit card numbers (13-19 digits with optional separators)
  { pattern: /\b(?:[0-9]{4}[-\s]?){3,4}[0-9]{1,4}\b/g, replacement: '[CC_REDACTED]' },
  // IPv4 addresses
  { pattern: /\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b/g, replacement: '[IP_REDACTED]' },
  // IPv6 addresses (simplified)
  { pattern: /\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b/g, replacement: '[IP_REDACTED]' },
  // API keys (common patterns: sk-, pk-, key-, bearer tokens)
  { pattern: /\b(?:sk|pk|api[_-]?key)[_-][a-zA-Z0-9]{20,}\b/gi, replacement: '[APIKEY_REDACTED]' },
  { pattern: /\bBearer\s+[a-zA-Z0-9._-]{20,}\b/g, replacement: 'Bearer [TOKEN_REDACTED]' },
  // AWS access keys
  { pattern: /\bAKIA[0-9A-Z]{16}\b/g, replacement: '[AWS_KEY_REDACTED]' },
]

function scrubPII(text: string): string {
  let result = text
  for (const { pattern, replacement } of PII_PATTERNS) {
    // Reset lastIndex for global regexes
    pattern.lastIndex = 0
    result = result.replace(pattern, replacement)
  }
  return result
}

function scrubEntryPII(entry: Omit<DatasetEntry, 'id' | 'timestamp'>): Omit<DatasetEntry, 'id' | 'timestamp'> {
  return {
    ...entry,
    messages: entry.messages.map(m => ({
      ...m,
      content: typeof m.content === 'string' ? scrubPII(m.content) : m.content,
    })),
    response: scrubPII(entry.response),
  }
}

// ── Public API ───────────────────────────────────────────────────────

export function addEntry(entry: Omit<DatasetEntry, 'id' | 'timestamp'>): string {
  const id = randomUUID()
  const scrubbed = scrubEntryPII(entry)
  const record: DatasetEntry = {
    ...scrubbed,
    id,
    timestamp: Date.now(),
  }

  dataset.push(record)

  // Auto-flush to HF when approaching capacity (async, non-blocking)
  checkDatasetThreshold(dataset.length, MAX_ENTRIES)

  // Evict oldest entries if over cap (fallback if HF not configured or failed)
  if (dataset.length > MAX_ENTRIES) {
    dataset = dataset.slice(dataset.length - MAX_ENTRIES)
  }

  return id
}

export function addFeedbackToEntry(
  entryId: string,
  feedback: DatasetEntry['feedback'],
): boolean {
  const entry = dataset.find(e => e.id === entryId)
  if (!entry) return false
  entry.feedback = feedback
  return true
}

export function deleteEntry(id: string): boolean {
  const idx = dataset.findIndex(e => e.id === id)
  if (idx === -1) return false
  dataset.splice(idx, 1)
  return true
}

export function getDataset(): DatasetEntry[] {
  return [...dataset]
}

export function getDatasetStats(): {
  total_entries: number
  entries_with_feedback: number
  mode_breakdown: Record<string, number>
  model_breakdown: Record<string, number>
  context_breakdown: Record<string, number>
  oldest_entry: number | null
  newest_entry: number | null
} {
  const modeBreakdown: Record<string, number> = {}
  const modelBreakdown: Record<string, number> = {}
  const contextBreakdown: Record<string, number> = {}
  let withFeedback = 0

  for (const entry of dataset) {
    modeBreakdown[entry.mode] = (modeBreakdown[entry.mode] || 0) + 1
    modelBreakdown[entry.model] = (modelBreakdown[entry.model] || 0) + 1
    if (entry.autotune?.detected_context) {
      const ctx = entry.autotune.detected_context
      contextBreakdown[ctx] = (contextBreakdown[ctx] || 0) + 1
    }
    if (entry.feedback) withFeedback++
  }

  return {
    total_entries: dataset.length,
    entries_with_feedback: withFeedback,
    mode_breakdown: modeBreakdown,
    model_breakdown: modelBreakdown,
    context_breakdown: contextBreakdown,
    oldest_entry: dataset.length > 0 ? dataset[0].timestamp : null,
    newest_entry: dataset.length > 0 ? dataset[dataset.length - 1].timestamp : null,
  }
}