diff --git a/src/utils/logMasker.ts b/src/utils/logMasker.ts index 792c7cb5..d0216f83 100644 --- a/src/utils/logMasker.ts +++ b/src/utils/logMasker.ts @@ -1,87 +1,135 @@ /** * Log Masking Utility - * Scrubs sensitive data (secrets, API keys, passwords, etc.) from log output - * to prevent leaking confidential information to stdout/stderr. + * Scrubs sensitive data (secrets, API keys, passwords, private hashes, + * internal IPs, admin data, webhook URLs, JWTs) from log output before + * entries are written to any transport or external storage. */ -// Patterns for detecting sensitive values -const SENSITIVE_PATTERNS = [ - // Environment variable names that contain sensitive data - /\b(SECRET|PASSWORD|TOKEN|KEY|CREDENTIAL|PRIVATE|API_KEY|APIKEY|AUTH|PK)\b/gi, +// --------------------------------------------------------------------------- +// Pattern groups +// --------------------------------------------------------------------------- - // Stellar secret keys (start with 'S' and are 56 characters base32, typically A-Z and 2-7) - /\bS[A-Z2-7]{48,56}\b/g, +/** Patterns that redact secret values embedded in strings. */ +const SENSITIVE_VALUE_PATTERNS: RegExp[] = [ + // Env-var-style assignments: KEY=value or KEY: value (captures the value) + /\b(SECRET|PASSWORD|TOKEN|KEY|CREDENTIAL|PRIVATE|API_KEY|APIKEY|AUTH|PK)\s*[:=]\s*['"]?([^\s'"&,}\]]{6,})['"]?/gi, - // Ethereum-style private keys (64 hex chars or 66 with 0x prefix) + // Stellar secret keys (S + 55 base32 chars, total 56) + /\bS[A-Z2-7]{55}\b/g, + + // Ethereum / generic 64-char hex private keys (with or without 0x prefix) /\b(0x)?[a-fA-F0-9]{64}\b/g, - // Common Bearer tokens - /Bearer\s+[A-Za-z0-9._-]+/gi, + // JWT tokens: three base64url segments separated by dots + /\bey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, + + // Bearer tokens + /Bearer\s+[A-Za-z0-9._\-+/=]{10,}/gi, + + // Database / Redis / AMQP connection strings — mask the password segment + /(:\/\/[^:@\s]+:)([^@\s]{1,})(@)/g, + + // AWS-style access keys + /\bAKIA[0-9A-Z]{16}\b/g, + + // Discord & Slack webhook URLs + /https:\/\/(?:discord(?:app)?\.com\/api\/webhooks|hooks\.slack\.com\/services)\/[^\s"'<>]+/gi, - // Database connection strings with passwords - /(:\/\/[^:]+:)([^@]+)(@)/g, + // Generic API-key-looking strings after key/token assignments in JSON or query strings + /(?:api[_-]?key|apikey|access[_-]?token|secret[_-]?key|private[_-]?key)\s*[:=]\s*['"]?([A-Za-z0-9_\-+/=]{16,})['"]?/gi, +]; - // AWS-style access keys (AKIA followed by 16 alphanumeric chars) - /AKIA[0-9A-Z]{16}/g, +/** + * Private/internal IP address patterns (RFC 1918, loopback, link-local). + * These are redacted from log strings to prevent leaking internal topology. + */ +const PRIVATE_IP_PATTERNS: RegExp[] = [ + // Loopback: 127.x.x.x + /\b127\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, + // RFC 1918: 10.x.x.x + /\b10\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, + // RFC 1918: 172.16.x.x – 172.31.x.x + /\b172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}\b/g, + // RFC 1918: 192.168.x.x + /\b192\.168\.\d{1,3}\.\d{1,3}\b/g, + // Link-local: 169.254.x.x + /\b169\.254\.\d{1,3}\.\d{1,3}\b/g, + // IPv6 loopback + /\b::1\b/g, + // IPv6 Unique Local Addresses (fd00::/8) + /\bfd[0-9a-fA-F]{2}:[0-9a-fA-F:]{2,}\b/gi, ]; +// Object keys whose values should always be fully redacted regardless of value. +const SENSITIVE_KEY_RE = + /secret|password|passwd|token|key|credential|private|api|auth|hash|seed|mnemonic|pin|ssn|card/i; + +// Object keys that represent admin-specific data. +const ADMIN_KEY_RE = /admin|superuser|root|internal|cluster|node_?ip|server_?ip/i; + +// --------------------------------------------------------------------------- +// Core masking helpers +// --------------------------------------------------------------------------- + /** - * Masks sensitive values in a string by replacing them with [REDACTED] - * @param input - The string to mask - * @returns The masked string with sensitive data replaced + * Masks sensitive values and internal IPs in a plain string. */ export function maskSensitiveData(input: string): string { - if (!input || typeof input !== "string") { - return input; - } + if (!input || typeof input !== "string") return input; let masked = input; - // Apply each pattern - for (const pattern of SENSITIVE_PATTERNS) { + for (const pattern of SENSITIVE_VALUE_PATTERNS) { + // Reset lastIndex for global regexes so successive calls work correctly. + pattern.lastIndex = 0; masked = masked.replace(pattern, (match) => { - // For database connection strings, preserve the connection type + if (/^https?:\/\//i.test(match)) return "[REDACTED_URL]"; + if (match.toLowerCase().startsWith("bearer")) return "Bearer [REDACTED]"; + // Preserve connection-string prefix and host, redact only the password. if (match.includes("://")) { - return match.replace(/(:\/\/[^:]+:)([^@]+)(@)/, "$1[REDACTED]$3"); - } - // For Bearer tokens, preserve the scheme - if (match.toLowerCase().startsWith("bearer")) { - return "Bearer [REDACTED]"; + return match.replace(/(:\/\/[^:@\s]+:)([^@\s]+)(@)/, "$1[REDACTED]$3"); } - // For other matches, just redact return "[REDACTED]"; }); } + for (const pattern of PRIVATE_IP_PATTERNS) { + pattern.lastIndex = 0; + masked = masked.replace(pattern, "[INTERNAL_IP]"); + } + return masked; } /** - * Masks sensitive data in an object (recursively) - * @param obj - The object to mask - * @returns A new object with sensitive values masked + * Recursively masks sensitive data in an object. + * - Keys matching SENSITIVE_KEY_RE or ADMIN_KEY_RE are fully redacted. + * - String values are run through maskSensitiveData. + * - Arrays of strings are individually masked. + * - Nested objects are processed recursively. */ export function maskSensitiveObject( - obj: Record, -): Record { - if (!obj || typeof obj !== "object") { - return obj; - } + obj: Record, +): Record { + if (!obj || typeof obj !== "object" || Array.isArray(obj)) return obj; - const masked: Record = {}; + const masked: Record = {}; for (const [key, value] of Object.entries(obj)) { - // Check if the key name suggests sensitive data - if (/secret|password|token|key|credential|private|api|auth/i.test(key)) { + if (SENSITIVE_KEY_RE.test(key) || ADMIN_KEY_RE.test(key)) { masked[key] = "[REDACTED]"; } else if (typeof value === "string") { masked[key] = maskSensitiveData(value); } else if (Array.isArray(value)) { masked[key] = value.map((item) => - typeof item === "string" ? maskSensitiveData(item) : item, + typeof item === "string" + ? maskSensitiveData(item) + : typeof item === "object" && item !== null + ? maskSensitiveObject(item as Record) + : item, ); } else if (typeof value === "object" && value !== null) { - masked[key] = maskSensitiveObject(value); + masked[key] = maskSensitiveObject(value as Record); } else { masked[key] = value; } @@ -90,90 +138,104 @@ export function maskSensitiveObject( return masked; } +// --------------------------------------------------------------------------- +// Winston-level scrubbing +// --------------------------------------------------------------------------- + /** - * Creates a masked console object that automatically scrubs logs - * Replace console.log, console.error, etc. with these versions + * Scrubs a Winston log info object in-place, returning a sanitised copy. + * + * The `level` and `timestamp` fields are preserved verbatim. + * Symbol-keyed properties (Winston internals such as Symbol(level) and + * Symbol(splat)) are copied across without modification. + */ +export function scrubLogInfo(info: unknown): unknown { + if (!info || typeof info !== "object") return info; + + const src = info as Record; + const result: Record = Object.create( + Object.getPrototypeOf(src), + ); + + // Own string-keyed properties + for (const key of Object.getOwnPropertyNames(src)) { + const value = src[key]; + + // Never alter level or timestamp — they are not sensitive. + if (key === "level" || key === "timestamp") { + result[key] = value; + continue; + } + + if (key === "message" && typeof value === "string") { + result[key] = maskSensitiveData(value); + } else if (typeof value === "string") { + result[key] = maskSensitiveData(value); + } else if (typeof value === "object" && value !== null && !Array.isArray(value)) { + result[key] = maskSensitiveObject(value as Record); + } else if (Array.isArray(value)) { + result[key] = (value as unknown[]).map((item) => + typeof item === "string" + ? maskSensitiveData(item) + : typeof item === "object" && item !== null + ? maskSensitiveObject(item as Record) + : item, + ); + } else { + result[key] = value; + } + } + + // Copy Winston's Symbol-keyed internals (Symbol(level), Symbol(splat), …) + // without modification — they contain internal state, not user data. + for (const sym of Object.getOwnPropertySymbols(src)) { + result[sym] = src[sym]; + } + + return result; +} + +// --------------------------------------------------------------------------- +// Global console interception +// --------------------------------------------------------------------------- + +/** + * Masked console wrapper — individual methods apply scrubbing before output. */ export const maskedConsole = { - log: (...args: any[]): void => { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - console.log(...maskedArgs); - }, - - error: (...args: any[]): void => { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - console.error(...maskedArgs); - }, - - warn: (...args: any[]): void => { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - console.warn(...maskedArgs); - }, - - info: (...args: any[]): void => { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - console.info(...maskedArgs); - }, - - debug: (...args: any[]): void => { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - console.debug(...maskedArgs); - }, + log: (...args: unknown[]): void => + console.log(...args.map((a) => (typeof a === "string" ? maskSensitiveData(a) : a))), + error: (...args: unknown[]): void => + console.error(...args.map((a) => (typeof a === "string" ? maskSensitiveData(a) : a))), + warn: (...args: unknown[]): void => + console.warn(...args.map((a) => (typeof a === "string" ? maskSensitiveData(a) : a))), + info: (...args: unknown[]): void => + console.info(...args.map((a) => (typeof a === "string" ? maskSensitiveData(a) : a))), + debug: (...args: unknown[]): void => + console.debug(...args.map((a) => (typeof a === "string" ? maskSensitiveData(a) : a))), }; /** - * Intercepts all console methods and applies masking - * Call this once at application startup to enable global log masking + * Monkey-patches all `console.*` methods so every string argument is scrubbed. + * Call once at application startup (already wired in index.ts). */ export function enableGlobalLogMasking(): void { - const originalLog = console.log; - const originalError = console.error; - const originalWarn = console.warn; - const originalInfo = console.info; - const originalDebug = console.debug; - - console.log = function (...args: any[]): void { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - originalLog(...maskedArgs); + const originals = { + log: console.log, + error: console.error, + warn: console.warn, + info: console.info, + debug: console.debug, }; - console.error = function (...args: any[]): void { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - originalError(...maskedArgs); - }; + const wrap = + (fn: (...a: unknown[]) => void) => + (...args: unknown[]): void => + fn(...args.map((a) => (typeof a === "string" ? maskSensitiveData(a) : a))); - console.warn = function (...args: any[]): void { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - originalWarn(...maskedArgs); - }; - - console.info = function (...args: any[]): void { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - originalInfo(...maskedArgs); - }; - - console.debug = function (...args: any[]): void { - const maskedArgs = args.map((arg) => - typeof arg === "string" ? maskSensitiveData(arg) : arg, - ); - originalDebug(...maskedArgs); - }; + console.log = wrap(originals.log); + console.error = wrap(originals.error); + console.warn = wrap(originals.warn); + console.info = wrap(originals.info); + console.debug = wrap(originals.debug); } diff --git a/src/utils/redactingTransport.ts b/src/utils/redactingTransport.ts new file mode 100644 index 00000000..0e3208fb --- /dev/null +++ b/src/utils/redactingTransport.ts @@ -0,0 +1,100 @@ +/** + * RedactingTransport — custom Winston transport layer for PII scrubbing. + * + * Wraps any inner Winston transport (e.g. DailyRotateFile for file storage, + * or a remote HTTP transport for external log aggregators) and applies + * full PII/secret scrubbing via scrubLogInfo() before the log entry is + * forwarded to the underlying transport. + * + * Architecture: + * + * Logger → redactFormat (format-level guard) + * → RedactingTransport.log() + * └─ scrubLogInfo() ← second scrub layer (defense-in-depth) + * └─ inner.log() ← DailyRotateFile / HTTP / etc. + * + * Using two independent scrubbing layers ensures that even data injected + * after format processing (e.g. by other middleware) is caught before it + * reaches disk or an external sink. + */ + +import Transport, { TransportStreamOptions } from "winston-transport"; +import { scrubLogInfo } from "./logMasker"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface RedactingTransportOptions extends TransportStreamOptions { + /** The underlying transport to write sanitised entries to. */ + inner: Transport; + /** + * Optional label included in the [REDACTED] replacement strings for + * debugging. Defaults to "RedactingTransport". + */ + label?: string; +} + +// --------------------------------------------------------------------------- +// RedactingTransport +// --------------------------------------------------------------------------- + +/** + * A custom Winston Transport that scrubs sensitive data from every log entry + * before delegating to an inner transport. + * + * Usage: + * ```ts + * import DailyRotateFile from 'winston-daily-rotate-file'; + * import { RedactingTransport } from './redactingTransport'; + * + * const fileTransport = new DailyRotateFile({ filename: 'app-%DATE%.log' }); + * const safeFileTransport = new RedactingTransport({ inner: fileTransport }); + * ``` + */ +export class RedactingTransport extends Transport { + private readonly inner: Transport; + readonly label: string; + + constructor({ inner, label = "RedactingTransport", ...opts }: RedactingTransportOptions) { + super(opts); + this.inner = inner; + this.label = label; + } + + /** + * Called by Winston for every log entry directed at this transport. + * + * 1. Scrubs the info object (message, metadata, nested objects). + * 2. Forwards the sanitised copy to the inner transport. + * 3. Emits 'logged' so Winston can track backpressure correctly. + */ + log(info: unknown, callback: () => void): void { + const scrubbed = scrubLogInfo(info); + + if (typeof this.inner.log === "function") { + this.inner.log(scrubbed, () => { + this.emit("logged", scrubbed); + callback(); + }); + } else { + // Fallback: inner transport doesn't expose log() — emit and proceed. + this.emit("logged", scrubbed); + callback(); + } + } + + /** + * Propagates close() to the inner transport so file handles are released + * cleanly on shutdown. + */ + close(): void { + if (typeof (this.inner as unknown as { close?: () => void }).close === "function") { + (this.inner as unknown as { close: () => void }).close(); + } + // super.close is optional in winston-transport — guard before calling. + if (typeof super.close === "function") { + super.close(); + } + } +} diff --git a/src/utils/winstonLogger.ts b/src/utils/winstonLogger.ts index a8a0e847..1fc3e62f 100644 --- a/src/utils/winstonLogger.ts +++ b/src/utils/winstonLogger.ts @@ -1,42 +1,106 @@ -import { createLogger, format, transports } from 'winston'; -import DailyRotateFile from 'winston-daily-rotate-file'; -import path from 'path'; +/** + * Winston logger with two-layer PII/secret redaction. + * + * Layer 1 — redactFormat (format pipeline): + * Applied globally before any transport sees a log entry. + * Catches secrets, private IPs, JWTs, webhook URLs, and admin data + * by running scrubLogInfo() inside a custom Winston format. + * + * Layer 2 — RedactingTransport (transport layer): + * Wraps the DailyRotateFile transport that writes to external storage. + * Applies scrubLogInfo() a second time as a defense-in-depth measure, + * ensuring that even data added after format processing is sanitised + * before it reaches disk or any remote log aggregator. + * + * The Console transport is covered by Layer 1 (global format) and by the + * enableGlobalLogMasking() call in index.ts which patches console.* methods. + */ -const logDir = path.resolve(__dirname, '../../logs'); +import { createLogger, format, transports } from "winston"; +import DailyRotateFile from "winston-daily-rotate-file"; +import path from "path"; +import { scrubLogInfo } from "./logMasker"; +import { RedactingTransport } from "./redactingTransport"; +const logDir = path.resolve(__dirname, "../../logs"); + +// --------------------------------------------------------------------------- +// Layer 1: redactFormat +// A custom Winston format that sanitises every log info object before it +// reaches any transport. This is the first line of defence. +// --------------------------------------------------------------------------- +const redactFormat = format((info) => scrubLogInfo(info) as ReturnType)(); + +// --------------------------------------------------------------------------- +// Inner file transport (not exposed directly — wrapped by RedactingTransport) +// --------------------------------------------------------------------------- +const dailyRotateFileTransport = new DailyRotateFile({ + filename: path.join(logDir, "application-%DATE%.log"), + datePattern: "YYYY-MM-DD", + maxSize: "100m", + maxFiles: "10", + zippedArchive: true, + handleExceptions: true, + handleRejections: true, + // Per-transport format: timestamp + JSON. Redaction is handled by the + // wrapping RedactingTransport, so we only need structural formatting here. + format: format.combine( + format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), + format.errors({ stack: true }), + format.json(), + ), +}); + +// --------------------------------------------------------------------------- +// Layer 2: RedactingTransport +// Wraps the file transport so every entry is scrubbed a second time before +// it is written to the log file (external storage). +// --------------------------------------------------------------------------- +const safeFileTransport = new RedactingTransport({ + inner: dailyRotateFileTransport, + label: "FileTransport", + // Inherit the same exception/rejection handling flags so unhandled errors + // and promise rejections are still captured. + handleExceptions: true, + handleRejections: true, +}); + +// --------------------------------------------------------------------------- +// Console transport (format-level redaction via redactFormat is sufficient; +// enableGlobalLogMasking() in index.ts provides an additional fallback) +// --------------------------------------------------------------------------- +const consoleTransport = new transports.Console({ + format: format.combine(format.colorize(), format.simple()), + handleExceptions: true, + handleRejections: true, +}); + +// --------------------------------------------------------------------------- +// Logger +// --------------------------------------------------------------------------- const logger = createLogger({ - level: 'info', + level: process.env.LOG_LEVEL ?? "info", format: format.combine( - format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), + // Layer 1: scrub before any transport sees the entry. + redactFormat, + format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), format.errors({ stack: true }), format.splat(), - format.json() + format.json(), ), transports: [ - new DailyRotateFile({ - filename: path.join(logDir, 'application-%DATE%.log'), - datePattern: 'YYYY-MM-DD', - maxSize: '100m', - maxFiles: '10', - zippedArchive: true, - handleExceptions: true, - handleRejections: true, - }), - new transports.Console({ - format: format.combine( - format.colorize(), - format.simple() - ), - handleExceptions: true, - handleRejections: true, - }) + // External storage: file transport wrapped in the redacting layer. + safeFileTransport, + // Console: covered by global format + console monkey-patch. + consoleTransport, ], exitOnError: false, }); -// Add custom methods for fetcher-specific logging -(logger as any).fetcherError = (message: string, meta?: any) => { - logger.error(`[FETCHER_ERROR] ${message}`, meta); -}; +// Convenience method kept for backwards compatibility with existing callers. +(logger as typeof logger & { fetcherError: (msg: string, meta?: unknown) => void }).fetcherError = + (message: string, meta?: unknown) => { + logger.error(`[FETCHER_ERROR] ${message}`, meta as object | undefined); + }; export default logger;