From 03280d253479e5b9cf35bbfce36469bb7d1a9a85 Mon Sep 17 00:00:00 2001 From: Hugo Richard Date: Wed, 10 Jun 2026 21:21:01 +0100 Subject: [PATCH 1/2] feat(core): add recursive key-based redaction --- .changeset/recursive-key-redaction.md | 14 ++ apps/docs/content/2.learn/6.redaction.md | 43 ++++-- .../4.use-cases/4.audit/05.compliance.md | 7 +- .../content/6.reference/1.configuration.md | 2 + apps/docs/skills/build-audit-logs/SKILL.md | 4 +- packages/evlog/src/audit.ts | 69 +++------- packages/evlog/src/redact.ts | 125 ++++++++++++++++-- packages/evlog/src/types.ts | 17 ++- packages/evlog/test/core/audit.test.ts | 33 ++++- packages/evlog/test/core/redact.test.ts | 100 ++++++++++++++ 10 files changed, 325 insertions(+), 89 deletions(-) create mode 100644 .changeset/recursive-key-redaction.md diff --git a/.changeset/recursive-key-redaction.md b/.changeset/recursive-key-redaction.md new file mode 100644 index 00000000..6c0f1295 --- /dev/null +++ b/.changeset/recursive-key-redaction.md @@ -0,0 +1,14 @@ +--- +"evlog": minor +--- + +Add recursive key-based redaction to `RedactConfig`. Use `keys` to redact object property names at any nesting depth (e.g. `password` covers `user.password` and `data.a.b.password`), and `keyPatterns` for regex on key names. `auditRedactPreset` now uses `keys` instead of explicit dot-notation paths. + +```ts +initLogger({ + redact: { + keys: ['password', 'apiKey', 'authorization'], + keyPatterns: [/.*_token$/i], + }, +}) +``` diff --git a/apps/docs/content/2.learn/6.redaction.md b/apps/docs/content/2.learn/6.redaction.md index e0211780..91f7ea5d 100644 --- a/apps/docs/content/2.learn/6.redaction.md +++ b/apps/docs/content/2.learn/6.redaction.md @@ -76,19 +76,37 @@ Built-in patterns use **partial masking** instead of flat `[REDACTED]` — prese ## Configuration +### Key-Based Redaction + +Redact fields by **key name at any nesting depth** — no need to know the full dot-notation path. A single `password` entry covers `user.password`, `data.a.b.password`, and every other occurrence: + +```typescript +evlog: { + redact: { + keys: ['password', 'apiKey', 'authorization', 'cookie'], + keyPatterns: [/.*_token$/i], + } +} +``` + +Key-based redaction replaces the **entire value** (including nested objects) with `replacement`. Use `keyPatterns` for regex on property names; use `patterns` when you need regex on **string values**. + +This matches the semantics of `auditDiff({ redactPaths: ['password'] })` — the same key-name rules, but applied globally at emit time. + ### Custom Paths -Add dot-notation paths to redact specific fields with `[REDACTED]`, on top of the built-in patterns: +Add exact dot-notation paths when you need to target one location only (e.g. hyphenated header keys): ```typescript evlog: { redact: { - paths: ['user.password', 'headers.authorization'], + paths: ['headers.x-forwarded-for'], + keys: ['authorization'], } } ``` -Path-based redaction replaces the **entire value** with the `replacement` string (default `[REDACTED]`), regardless of content. +Path-based redaction replaces the **entire leaf value** with the `replacement` string (default `[REDACTED]`), regardless of content. Unlike `keys`, `paths: ['password']` only redacts a top-level `password` field. ### Selective Built-ins @@ -134,10 +152,12 @@ evlog: { | Option | Type | Default | Description | |--------|------|---------|-------------| | `redact` | `boolean \| RedactConfig` | `true` in production | Enabled by default in production. `false` to disable. Object for fine-grained control | -| `paths` | `string[]` | `undefined` | Dot-notation paths to redact entirely (e.g. `user.password`) | -| `patterns` | `RegExp[]` | `undefined` | Custom regex patterns. Uses flat `replacement` string | +| `keys` | `string[]` | `undefined` | Object key names to redact at any depth (e.g. `password` → all `password` fields) | +| `keyPatterns` | `RegExp[]` | `undefined` | Regex on object key names at any depth (e.g. `/.*_token$/`) | +| `paths` | `string[]` | `undefined` | Exact dot-notation paths only (e.g. `headers.x-forwarded-for`) | +| `patterns` | `RegExp[]` | `undefined` | Custom regex on string values. Uses flat `replacement` string | | `builtins` | `false \| string[]` | All enabled | `false` disables built-ins. Array selects specific ones | -| `replacement` | `string` | `'[REDACTED]'` | Replacement string for paths and custom patterns. Built-in patterns use smart masking instead | +| `replacement` | `string` | `'[REDACTED]'` | Replacement for keys, paths, and custom patterns. Built-ins use smart masking instead | Available built-in names: `creditCard`, `email`, `ipv4`, `phone`, `jwt`, `bearer`, `iban`. @@ -145,11 +165,12 @@ Available built-in names: `creditCard`, `email`, `ipv4`, `phone`, `jwt`, `bearer Redaction runs inside the emit pipeline, after the wide event is fully built but before any output: -1. **Path redaction** — targeted fields replaced with `[REDACTED]` -2. **Smart masking** — built-in patterns scan all string values recursively with partial masking -3. **Pattern redaction** — custom regex patterns scan all string values with flat replacement -4. **Console output** — masked event printed to stdout -5. **Drain** — masked event sent to external services +1. **Key redaction** — matching key names at any depth replaced with `[REDACTED]` +2. **Path redaction** — exact dot-notation leaves replaced with `[REDACTED]` +3. **Smart masking** — built-in patterns scan all string values recursively with partial masking +4. **Pattern redaction** — custom regex patterns scan all string values with flat replacement +5. **Console output** — masked event printed to stdout +6. **Drain** — masked event sent to external services ::callout{icon="i-lucide-zap" color="info"} Redaction runs **after** the HTTP response is sent, so it adds zero latency to your API responses. diff --git a/apps/docs/content/4.use-cases/4.audit/05.compliance.md b/apps/docs/content/4.use-cases/4.audit/05.compliance.md index 0dd30b30..15664ce2 100644 --- a/apps/docs/content/4.use-cases/4.audit/05.compliance.md +++ b/apps/docs/content/4.use-cases/4.audit/05.compliance.md @@ -45,15 +45,14 @@ import { auditRedactPreset } from 'evlog' initLogger({ redact: { - paths: [ - ...(auditRedactPreset.paths ?? []), - 'user.password', + keys: [ + ...(auditRedactPreset.keys ?? []), ], }, }) ``` -The preset drops `Authorization` / `Cookie` headers and common credential field names (`password`, `token`, `apiKey`, `cardNumber`, `cvv`, `ssn`) wherever they appear inside `audit.changes.before` and `audit.changes.after`. +The preset redacts `authorization`, `cookie`, `set-cookie`, and common credential key names (`password`, `token`, `apiKey`, `cardNumber`, `cvv`, `ssn`) **at any nesting depth** — including inside `audit.changes.before` / `audit.changes.after`. ## GDPR vs append-only diff --git a/apps/docs/content/6.reference/1.configuration.md b/apps/docs/content/6.reference/1.configuration.md index 9c07696e..64cb4e05 100644 --- a/apps/docs/content/6.reference/1.configuration.md +++ b/apps/docs/content/6.reference/1.configuration.md @@ -50,6 +50,8 @@ initLogger({ | `redact` | `boolean \| RedactConfig` | `true` in production | Enabled by default in production. `false` to disable. Object for fine-grained control. See [Auto-Redaction](/learn/redaction) | | `drain` | `(ctx: DrainContext) => void` | `undefined` | Drain callback for sending events to external services | +`RedactConfig` fields (when `redact` is an object): `keys` (key names at any depth), `keyPatterns` (regex on key names), `paths` (exact dot-notation), `patterns` (regex on string values), `builtins`, `replacement`. Full table in [Auto-Redaction](/learn/redaction#configuration-reference). + ### `minLevel` vs sampling - **`minLevel`** is a **hard threshold** on the simple `log.*` API: levels below the threshold are never emitted. It does **not** apply to wide events from `useLogger` / `createLogger().emit()` — use **`sampling.rates`** (and tail `keep`) for request volume. diff --git a/apps/docs/skills/build-audit-logs/SKILL.md b/apps/docs/skills/build-audit-logs/SKILL.md index b1d71957..8855fcc7 100644 --- a/apps/docs/skills/build-audit-logs/SKILL.md +++ b/apps/docs/skills/build-audit-logs/SKILL.md @@ -282,14 +282,14 @@ function authorize(actor, action, resource) { ### Step 5 — Redact -Apply `auditRedactPreset` (or merge it into the existing `RedactConfig`). It drops `Authorization` / `Cookie` headers and common credential field names (`password`, `token`, `apiKey`, `cardNumber`, `cvv`, `ssn`) wherever they appear inside `audit.changes.before` / `audit.changes.after`: +Apply `auditRedactPreset` (or merge it into the existing `RedactConfig`). It redacts `authorization`, `cookie`, `set-cookie`, and common credential key names (`password`, `token`, `apiKey`, `cardNumber`, `cvv`, `ssn`) at any nesting depth — including inside `audit.changes.before` / `audit.changes.after`: ```ts import { initLogger, auditRedactPreset } from 'evlog' initLogger({ redact: { - paths: [...(auditRedactPreset.paths ?? []), 'user.password', 'user.token'], + keys: [...(auditRedactPreset.keys ?? [])], }, }) ``` diff --git a/packages/evlog/src/audit.ts b/packages/evlog/src/audit.ts index 8cc20afb..5b958a90 100644 --- a/packages/evlog/src/audit.ts +++ b/packages/evlog/src/audit.ts @@ -1,5 +1,6 @@ import type { AuditActor, AuditActionDefinition, AuditFields, AuditPatchOp, AuditTarget, DrainContext, EnrichContext, FieldContext, RedactConfig, RequestLogger, WideEvent } from './types' import { createLogger } from './logger' +import { buildKeyMatcher, redactValueByKeys } from './redact' import { getHeader as getSharedHeader } from './shared/headers' /** @@ -323,18 +324,9 @@ export function auditDiff( options: AuditDiffOptions = {}, ): { before?: unknown, after?: unknown, patch: AuditPatchOp[] } { const replacement = options.replacement ?? '[REDACTED]' - const redactSet = new Set((options.redactPaths ?? []).map(p => p)) + const keyMatcher = buildKeyMatcher(options.redactPaths) ?? (() => false) const patch: AuditPatchOp[] = [] - function isRedacted(path: string): boolean { - if (redactSet.size === 0) return false - if (redactSet.has(path)) return true - for (const p of redactSet) { - if (path.endsWith(`.${p}`)) return true - } - return false - } - function diff(a: unknown, b: unknown, path: string): void { if (a === b) return @@ -363,20 +355,7 @@ export function auditDiff( } function redactValue(value: unknown, path: string): unknown { - if (value === null || typeof value !== 'object') { - const segs = path.split('/').filter(Boolean) - const last = segs[segs.length - 1] - if (last && isRedacted(last)) return replacement - return value - } - if (Array.isArray(value)) { - return value.map((v, i) => redactValue(v, `${path}/${i}`)) - } - const out: Record = {} - for (const [k, v] of Object.entries(value as Record)) { - out[k] = isRedacted(k) ? replacement : redactValue(v, `${path}/${k}`) - } - return out + return redactValueByKeys(value, keyMatcher, replacement, path) } diff(before, after, '') @@ -867,7 +846,7 @@ function stripIntegrity(event: WideEvent): WideEvent { * Strict redact preset for audit events. * * Combine with the user's existing redact configuration via spread: - * `initLogger({ redact: { paths: [...auditRedactPreset.paths!, ...mine] } })`. + * `initLogger({ redact: { keys: [...auditRedactPreset.keys!, ...mine] } })`. * * Hardens PII handling: * - Drops `Authorization` and `Cookie` headers anywhere they appear. @@ -880,31 +859,19 @@ function stripIntegrity(event: WideEvent): WideEvent { * enough signal to be useful. */ export const auditRedactPreset: RedactConfig = { - paths: [ - 'audit.changes.before.password', - 'audit.changes.before.passwordHash', - 'audit.changes.before.token', - 'audit.changes.before.apiKey', - 'audit.changes.before.secret', - 'audit.changes.before.accessToken', - 'audit.changes.before.refreshToken', - 'audit.changes.before.cardNumber', - 'audit.changes.before.cvv', - 'audit.changes.before.ssn', - 'audit.changes.after.password', - 'audit.changes.after.passwordHash', - 'audit.changes.after.token', - 'audit.changes.after.apiKey', - 'audit.changes.after.secret', - 'audit.changes.after.accessToken', - 'audit.changes.after.refreshToken', - 'audit.changes.after.cardNumber', - 'audit.changes.after.cvv', - 'audit.changes.after.ssn', - 'headers.authorization', - 'headers.cookie', - 'headers.set-cookie', - 'audit.context.headers.authorization', - 'audit.context.headers.cookie', + keys: [ + 'password', + 'passwordHash', + 'token', + 'apiKey', + 'secret', + 'accessToken', + 'refreshToken', + 'cardNumber', + 'cvv', + 'ssn', + 'authorization', + 'cookie', + 'set-cookie', ], } diff --git a/packages/evlog/src/redact.ts b/packages/evlog/src/redact.ts index 7602d518..b948abe0 100644 --- a/packages/evlog/src/redact.ts +++ b/packages/evlog/src/redact.ts @@ -4,6 +4,87 @@ const DEFAULT_REPLACEMENT = '[REDACTED]' export type Masker = [RegExp, (match: string) => string] +/** Predicate that returns whether an object key name should be fully redacted. */ +export type KeyMatcher = (key: string) => boolean + +/** + * Build a matcher from exact key names and regex patterns on key names. + * Returns `undefined` when both inputs are empty. + */ +export function buildKeyMatcher(keys?: string[], keyPatterns?: RegExp[]): KeyMatcher | undefined { + const keySet = new Set(keys ?? []) + const patterns = (keyPatterns ?? []).map(cloneRegex) + if (keySet.size === 0 && patterns.length === 0) return undefined + + return (key: string) => { + if (keySet.has(key)) return true + for (const pattern of patterns) { + pattern.lastIndex = 0 + if (pattern.test(key)) return true + } + return false + } +} + +/** + * Redact values whose key names match `matcher`, recursively at any depth. + * Mutates `obj` in place (intended for use on a clone). + */ +export function redactKeysInTree(obj: unknown, matcher: KeyMatcher, replacement: string): void { + if (obj === null || obj === undefined) return + + if (Array.isArray(obj)) { + for (const item of obj) { + redactKeysInTree(item, matcher, replacement) + } + return + } + + if (typeof obj === 'object') { + const record = obj as Record + for (const key in record) { + if (matcher(key)) { + record[key] = replacement + } else { + redactKeysInTree(record[key], matcher, replacement) + } + } + } +} + +/** + * Return a copy of `value` with key-name matches replaced by `replacement`. + * Used by audit diffs; does not mutate the input. + * + * When `value` is a scalar and `path` is provided, the last JSON Pointer + * segment is checked against `matcher` (for patch leaf values). + */ +export function redactValueByKeys( + value: unknown, + matcher: KeyMatcher, + replacement: string, + path?: string, +): unknown { + if (value === null || typeof value !== 'object') { + if (path) { + const last = path.split('/').filter(Boolean).at(-1) + if (last && matcher(last)) return replacement + } + return value + } + + if (Array.isArray(value)) { + return value.map((v, i) => redactValueByKeys(v, matcher, replacement, path ? `${path}/${i}` : undefined)) + } + + const out: Record = {} + for (const [k, v] of Object.entries(value as Record)) { + const childPath = path ? `${path}/${k}` : k + out[k] = matcher(k) ? replacement : redactValueByKeys(v, matcher, replacement, childPath) + } + return out +} + /** * Built-in PII detection patterns with smart masking. * Each builtin preserves just enough signal for debugging while scrubbing PII. @@ -148,10 +229,11 @@ function cloneForRedaction(event: Record): Record, config: RedactConfig const clone = cloneForRedaction(event) const replacement = config.replacement ?? DEFAULT_REPLACEMENT + const keyMatcher = buildKeyMatcher(config.keys, config.keyPatterns) + if (keyMatcher) { + redactKeysInTree(clone, keyMatcher, replacement) + } + if (config.paths?.length) { for (const path of config.paths) { redactPath(clone, path.split('.'), replacement) @@ -280,6 +367,10 @@ export function normalizeRedactConfig(raw: boolean | Record | u config.paths = raw.paths as string[] } + if (Array.isArray(raw.keys)) { + config.keys = raw.keys as string[] + } + if (typeof raw.replacement === 'string') { config.replacement = raw.replacement } @@ -291,16 +382,24 @@ export function normalizeRedactConfig(raw: boolean | Record | u } if (Array.isArray(raw.patterns)) { - config.patterns = (raw.patterns as unknown[]).map((p) => { - if (p instanceof RegExp) return p - if (typeof p === 'string') return new RegExp(p, 'g') - if (typeof p === 'object' && p !== null) { - const obj = p as Record - return new RegExp(obj.source, obj.flags ?? 'g') - } - return null - }).filter((p): p is RegExp => p !== null) + config.patterns = deserializeRegexList(raw.patterns) + } + + if (Array.isArray(raw.keyPatterns)) { + config.keyPatterns = deserializeRegexList(raw.keyPatterns) } return resolveRedactConfig(config) } + +function deserializeRegexList(raw: unknown[]): RegExp[] { + return raw.map((p) => { + if (p instanceof RegExp) return p + if (typeof p === 'string') return new RegExp(p, 'g') + if (typeof p === 'object' && p !== null) { + const obj = p as Record + return new RegExp(obj.source, obj.flags ?? 'g') + } + return null + }).filter((p): p is RegExp => p !== null) +} diff --git a/packages/evlog/src/types.ts b/packages/evlog/src/types.ts index 2180a4f8..2bf7a0d7 100644 --- a/packages/evlog/src/types.ts +++ b/packages/evlog/src/types.ts @@ -98,8 +98,21 @@ export interface IngestPayload { * or select specific ones with `builtins: ['email', 'creditCard']`. */ export interface RedactConfig { - /** Dot-notation paths to redact (e.g., 'user.email', 'headers.x-forwarded-for') */ + /** + * Exact dot-notation paths to redact (e.g. `'user.email'`, `'headers.x-forwarded-for'`). + * Only the leaf at that path is replaced — does not match the same key name elsewhere. + */ paths?: string[] + /** + * Object key names to redact at any nesting depth (e.g. `'password'` redacts + * `user.password`, `data.a.b.password`, etc.). Replaces the entire value. + */ + keys?: string[] + /** + * RegExp patterns matched against object key names at any nesting depth. + * Replaces the entire value (e.g. `/.*_token$/` matches `access_token`, `refresh_token`). + */ + keyPatterns?: RegExp[] /** Additional regex patterns to match and replace string values anywhere in the event */ patterns?: RegExp[] /** @@ -112,7 +125,7 @@ export interface RedactConfig { */ builtins?: false | Array<'creditCard' | 'email' | 'ipv4' | 'phone' | 'jwt' | 'bearer' | 'iban'> /** - * Replacement string used for path-based and custom pattern redaction. + * Replacement string used for path-, key-, and custom pattern redaction. * Built-in patterns use smart partial masking instead (e.g. `****1111` for credit cards). * @default '[REDACTED]' */ diff --git a/packages/evlog/test/core/audit.test.ts b/packages/evlog/test/core/audit.test.ts index 4684c68d..af5f80e7 100644 --- a/packages/evlog/test/core/audit.test.ts +++ b/packages/evlog/test/core/audit.test.ts @@ -16,7 +16,7 @@ import { } from '../../src/audit' import type { AuditFields, DrainContext, EnrichContext, WideEvent } from '../../src/types' import { createLogger, createRequestLogger, initLogger } from '../../src/logger' -import { resolveRedactConfig } from '../../src/redact' +import { redactEvent, resolveRedactConfig } from '../../src/redact' import { defined } from '../helpers/defined' function createDrainCtx(event: Partial = {}): DrainContext { @@ -508,15 +508,36 @@ describe('end-to-end: audit + auditOnly + global drain', () => { }) describe('auditRedactPreset', () => { - it('drops authorization headers', () => { + it('redacts authorization and cookie key names at any depth', () => { const config = defined(resolveRedactConfig(auditRedactPreset), 'audit redact preset') - expect(config.paths).toContain('headers.authorization') + expect(config.keys).toContain('authorization') + expect(config.keys).toContain('cookie') + expect(config.keys).toContain('set-cookie') }) - it('drops password fields under audit.changes', () => { + it('redacts credential key names at any depth', () => { const config = defined(resolveRedactConfig(auditRedactPreset), 'audit redact preset') - expect(config.paths).toContain('audit.changes.before.password') - expect(config.paths).toContain('audit.changes.after.password') + expect(config.keys).toContain('password') + expect(config.keys).toContain('token') + expect(config.keys).toContain('apiKey') + }) + + it('redacts nested audit.changes password fields via keys', () => { + const config = defined(resolveRedactConfig(auditRedactPreset), 'audit redact preset') + const event = redactEvent( + { + audit: { + changes: { + before: { password: 'old' }, + after: { password: 'new' }, + }, + }, + }, + config, + ) + const changes = (event.audit as Record).changes as Record + expect((changes.before as Record).password).toBe('[REDACTED]') + expect((changes.after as Record).password).toBe('[REDACTED]') }) }) diff --git a/packages/evlog/test/core/redact.test.ts b/packages/evlog/test/core/redact.test.ts index 39a51742..a135ea1d 100644 --- a/packages/evlog/test/core/redact.test.ts +++ b/packages/evlog/test/core/redact.test.ts @@ -68,6 +68,95 @@ describe('redactEvent - path-based', () => { expect(user.age).toBe('[REDACTED]') expect(user.settings).toBe('[REDACTED]') }) + + it('does not redact nested password with top-level path only', () => { + const source: Record = { + password: 'top', + user: { password: 'nested' }, + } + const event = redactEvent(source, { paths: ['password'] }) + expect(event.password).toBe('[REDACTED]') + expect((event.user as Record).password).toBe('nested') + expect(source.user).toEqual({ password: 'nested' }) + }) +}) + +describe('redactEvent - key-based', () => { + it('redacts a key at any nesting depth', () => { + const source: Record = { + password: 'top', + user: { name: 'Alice', password: 'secret' }, + data: { a: { b: { password: 'deep' } } }, + } + const event = redactEvent(source, { keys: ['password'] }) + expect(event.password).toBe('[REDACTED]') + expect((event.user as Record).password).toBe('[REDACTED]') + expect((event.user as Record).name).toBe('Alice') + const deep = (event.data as Record).a as Record + expect((deep.b as Record).password).toBe('[REDACTED]') + expect(source.user).toEqual({ name: 'Alice', password: 'secret' }) + }) + + it('replaces the entire matched value without recursing into children', () => { + const event = redactEvent( + { password: { nested: 'still-here' } }, + { keys: ['password'] }, + ) + expect(event.password).toBe('[REDACTED]') + }) + + it('redacts multiple key names', () => { + const event = redactEvent( + { + password: 'p', + token: 't', + user: { apiKey: 'k', name: 'Alice' }, + }, + { keys: ['password', 'token', 'apiKey'] }, + ) + expect(event.password).toBe('[REDACTED]') + expect(event.token).toBe('[REDACTED]') + expect((event.user as Record).apiKey).toBe('[REDACTED]') + expect((event.user as Record).name).toBe('Alice') + }) + + it('matches key names with keyPatterns', () => { + const event = redactEvent( + { + access_token: 'a', + refresh_token: 'r', + username: 'alice', + }, + { keyPatterns: [/.*_token$/] }, + ) + expect(event.access_token).toBe('[REDACTED]') + expect(event.refresh_token).toBe('[REDACTED]') + expect(event.username).toBe('alice') + }) + + it('combines keys and exact paths', () => { + const event = redactEvent( + { + password: 'everywhere', + headers: { 'x-forwarded-for': '1.2.3.4', authorization: 'Bearer x' }, + }, + { + keys: ['password', 'authorization'], + paths: ['headers.x-forwarded-for'], + }, + ) + expect(event.password).toBe('[REDACTED]') + expect((event.headers as Record).authorization).toBe('[REDACTED]') + expect((event.headers as Record)['x-forwarded-for']).toBe('[REDACTED]') + }) + + it('uses custom replacement string', () => { + const event = redactEvent( + { user: { password: 'secret' } }, + { keys: ['password'], replacement: '***' }, + ) + expect((event.user as Record).password).toBe('***') + }) }) describe('redactEvent - pattern-based', () => { @@ -307,6 +396,17 @@ describe('normalizeRedactConfig', () => { expect(config?.replacement).toBe('***') }) + it('preserves keys and deserializes keyPatterns', () => { + const config = defined(normalizeRedactConfig({ + keys: ['password', 'token'], + keyPatterns: ['.*_secret$'], + builtins: false, + }), 'redact config') + expect(config.keys).toEqual(['password', 'token']) + expect(config.keyPatterns).toHaveLength(1) + expect(defined(config.keyPatterns?.[0], 'keyPatterns[0]').source).toBe('.*_secret$') + }) + it('converts string patterns to RegExp separately from built-in maskers', () => { const config = defined(normalizeRedactConfig({ patterns: ['\\b\\d{4}\\b'], From 5d966a234796133c6749c3e4f9cd2d72171996cd Mon Sep 17 00:00:00 2001 From: Hugo Richard Date: Wed, 10 Jun 2026 21:32:33 +0100 Subject: [PATCH 2/2] refine api --- .changeset/recursive-key-redaction.md | 5 +- apps/docs/content/2.learn/6.redaction.md | 54 ++-- .../4.use-cases/4.audit/05.compliance.md | 4 +- .../content/6.reference/1.configuration.md | 2 +- apps/docs/skills/build-audit-logs/SKILL.md | 4 +- packages/evlog/src/audit.ts | 21 +- packages/evlog/src/redact.ts | 238 ++++++++++++------ packages/evlog/src/types.ts | 21 +- packages/evlog/src/utils.ts | 30 ++- packages/evlog/test/core/audit.test.ts | 36 ++- packages/evlog/test/core/redact.test.ts | 119 ++++++--- 11 files changed, 343 insertions(+), 191 deletions(-) diff --git a/.changeset/recursive-key-redaction.md b/.changeset/recursive-key-redaction.md index 6c0f1295..5ee2fac9 100644 --- a/.changeset/recursive-key-redaction.md +++ b/.changeset/recursive-key-redaction.md @@ -2,13 +2,12 @@ "evlog": minor --- -Add recursive key-based redaction to `RedactConfig`. Use `keys` to redact object property names at any nesting depth (e.g. `password` covers `user.password` and `data.a.b.password`), and `keyPatterns` for regex on key names. `auditRedactPreset` now uses `keys` instead of explicit dot-notation paths. +Add glob path redaction to `RedactConfig.paths`. Single-segment patterns like `password` are shorthand for `**.password` (any nesting depth). Key-name globs (`*_token`) and path globs (`user.*`) are supported. `auditRedactPreset` simplified to path globs. ```ts initLogger({ redact: { - keys: ['password', 'apiKey', 'authorization'], - keyPatterns: [/.*_token$/i], + paths: ['password', '*_token', 'headers.x-forwarded-for'], }, }) ``` diff --git a/apps/docs/content/2.learn/6.redaction.md b/apps/docs/content/2.learn/6.redaction.md index 91f7ea5d..bf9eb743 100644 --- a/apps/docs/content/2.learn/6.redaction.md +++ b/apps/docs/content/2.learn/6.redaction.md @@ -76,37 +76,34 @@ Built-in patterns use **partial masking** instead of flat `[REDACTED]` — prese ## Configuration -### Key-Based Redaction +### Path Patterns -Redact fields by **key name at any nesting depth** — no need to know the full dot-notation path. A single `password` entry covers `user.password`, `data.a.b.password`, and every other occurrence: +Use a single `paths` array with dot-notation and globs. A bare segment like `password` is shorthand for `**.password` — it redacts that key at **any nesting depth**: ```typescript evlog: { redact: { - keys: ['password', 'apiKey', 'authorization', 'cookie'], - keyPatterns: [/.*_token$/i], + paths: [ + 'password', // same as '**.password' + '*_token', // key-name glob at any depth + 'headers.x-forwarded-for', // exact path + 'user.*', // everything directly under user + ], } } ``` -Key-based redaction replaces the **entire value** (including nested objects) with `replacement`. Use `keyPatterns` for regex on property names; use `patterns` when you need regex on **string values**. +| Pattern | Matches | +|---------|---------| +| `user.email` | Exact path only | +| `password` or `**.password` | `password` key at any depth | +| `*_token` | Key names like `access_token`, `refresh_token` | +| `user.*` | `user.email`, `user.password`, etc. | +| `audit.changes.*.password` | Mixed exact + wildcard segments | -This matches the semantics of `auditDiff({ redactPaths: ['password'] })` — the same key-name rules, but applied globally at emit time. +Path redaction replaces the **entire value** (including nested objects) with `replacement`. Use `patterns` when you need regex on **string values** inside fields. -### Custom Paths - -Add exact dot-notation paths when you need to target one location only (e.g. hyphenated header keys): - -```typescript -evlog: { - redact: { - paths: ['headers.x-forwarded-for'], - keys: ['authorization'], - } -} -``` - -Path-based redaction replaces the **entire leaf value** with the `replacement` string (default `[REDACTED]`), regardless of content. Unlike `keys`, `paths: ['password']` only redacts a top-level `password` field. +This matches `auditDiff({ redactPaths: ['password'] })` — same glob syntax, applied globally at emit time. ### Selective Built-ins @@ -152,12 +149,10 @@ evlog: { | Option | Type | Default | Description | |--------|------|---------|-------------| | `redact` | `boolean \| RedactConfig` | `true` in production | Enabled by default in production. `false` to disable. Object for fine-grained control | -| `keys` | `string[]` | `undefined` | Object key names to redact at any depth (e.g. `password` → all `password` fields) | -| `keyPatterns` | `RegExp[]` | `undefined` | Regex on object key names at any depth (e.g. `/.*_token$/`) | -| `paths` | `string[]` | `undefined` | Exact dot-notation paths only (e.g. `headers.x-forwarded-for`) | +| `paths` | `string[]` | `undefined` | Dot-notation paths with globs (`password`, `**.password`, `*_token`, `user.*`) | | `patterns` | `RegExp[]` | `undefined` | Custom regex on string values. Uses flat `replacement` string | | `builtins` | `false \| string[]` | All enabled | `false` disables built-ins. Array selects specific ones | -| `replacement` | `string` | `'[REDACTED]'` | Replacement for keys, paths, and custom patterns. Built-ins use smart masking instead | +| `replacement` | `string` | `'[REDACTED]'` | Replacement for paths and custom patterns. Built-ins use smart masking instead | Available built-in names: `creditCard`, `email`, `ipv4`, `phone`, `jwt`, `bearer`, `iban`. @@ -165,12 +160,11 @@ Available built-in names: `creditCard`, `email`, `ipv4`, `phone`, `jwt`, `bearer Redaction runs inside the emit pipeline, after the wide event is fully built but before any output: -1. **Key redaction** — matching key names at any depth replaced with `[REDACTED]` -2. **Path redaction** — exact dot-notation leaves replaced with `[REDACTED]` -3. **Smart masking** — built-in patterns scan all string values recursively with partial masking -4. **Pattern redaction** — custom regex patterns scan all string values with flat replacement -5. **Console output** — masked event printed to stdout -6. **Drain** — masked event sent to external services +1. **Path redaction** — exact paths and globs replaced with `[REDACTED]` +2. **Smart masking** — built-in patterns scan all string values recursively with partial masking +3. **Pattern redaction** — custom regex patterns scan all string values with flat replacement +4. **Console output** — masked event printed to stdout +5. **Drain** — masked event sent to external services ::callout{icon="i-lucide-zap" color="info"} Redaction runs **after** the HTTP response is sent, so it adds zero latency to your API responses. diff --git a/apps/docs/content/4.use-cases/4.audit/05.compliance.md b/apps/docs/content/4.use-cases/4.audit/05.compliance.md index 15664ce2..23f67f0c 100644 --- a/apps/docs/content/4.use-cases/4.audit/05.compliance.md +++ b/apps/docs/content/4.use-cases/4.audit/05.compliance.md @@ -45,8 +45,8 @@ import { auditRedactPreset } from 'evlog' initLogger({ redact: { - keys: [ - ...(auditRedactPreset.keys ?? []), + paths: [ + ...(auditRedactPreset.paths ?? []), ], }, }) diff --git a/apps/docs/content/6.reference/1.configuration.md b/apps/docs/content/6.reference/1.configuration.md index 64cb4e05..955efea4 100644 --- a/apps/docs/content/6.reference/1.configuration.md +++ b/apps/docs/content/6.reference/1.configuration.md @@ -50,7 +50,7 @@ initLogger({ | `redact` | `boolean \| RedactConfig` | `true` in production | Enabled by default in production. `false` to disable. Object for fine-grained control. See [Auto-Redaction](/learn/redaction) | | `drain` | `(ctx: DrainContext) => void` | `undefined` | Drain callback for sending events to external services | -`RedactConfig` fields (when `redact` is an object): `keys` (key names at any depth), `keyPatterns` (regex on key names), `paths` (exact dot-notation), `patterns` (regex on string values), `builtins`, `replacement`. Full table in [Auto-Redaction](/learn/redaction#configuration-reference). +`RedactConfig` fields (when `redact` is an object): `paths` (dot-notation with globs), `patterns` (regex on string values), `builtins`, `replacement`. Full table in [Auto-Redaction](/learn/redaction#configuration-reference). ### `minLevel` vs sampling diff --git a/apps/docs/skills/build-audit-logs/SKILL.md b/apps/docs/skills/build-audit-logs/SKILL.md index 8855fcc7..7411fa14 100644 --- a/apps/docs/skills/build-audit-logs/SKILL.md +++ b/apps/docs/skills/build-audit-logs/SKILL.md @@ -282,14 +282,14 @@ function authorize(actor, action, resource) { ### Step 5 — Redact -Apply `auditRedactPreset` (or merge it into the existing `RedactConfig`). It redacts `authorization`, `cookie`, `set-cookie`, and common credential key names (`password`, `token`, `apiKey`, `cardNumber`, `cvv`, `ssn`) at any nesting depth — including inside `audit.changes.before` / `audit.changes.after`: +Apply `auditRedactPreset` (or merge it into the existing `RedactConfig`). It redacts HTTP auth headers and common credential field names (`password`, `token`, `apiKey`, `cardNumber`, `cvv`, `ssn`, plus `cookie` / `set-cookie`) at any nesting depth — including inside `audit.changes.before` / `audit.changes.after`: ```ts import { initLogger, auditRedactPreset } from 'evlog' initLogger({ redact: { - keys: [...(auditRedactPreset.keys ?? [])], + paths: [...(auditRedactPreset.paths ?? [])], }, }) ``` diff --git a/packages/evlog/src/audit.ts b/packages/evlog/src/audit.ts index 5b958a90..b765e0ee 100644 --- a/packages/evlog/src/audit.ts +++ b/packages/evlog/src/audit.ts @@ -1,6 +1,6 @@ import type { AuditActor, AuditActionDefinition, AuditFields, AuditPatchOp, AuditTarget, DrainContext, EnrichContext, FieldContext, RedactConfig, RequestLogger, WideEvent } from './types' import { createLogger } from './logger' -import { buildKeyMatcher, redactValueByKeys } from './redact' +import { compileRedactPathMatchers, redactValueByPaths } from './redact' import { getHeader as getSharedHeader } from './shared/headers' /** @@ -304,8 +304,8 @@ export interface WithAuditContext { * `changes` field. Output is a JSON Patch-style array (RFC 6902 subset: * `add`, `remove`, `replace`) — small enough to ship over the wire. * - * Object keys whose name matches one of the `redactPaths` (dot-notation, e.g. - * `'user.password'`, `'card.cvv'`) are replaced with `'[REDACTED]'` so PII + * Fields matching `redactPaths` glob patterns (e.g. `'password'`, `'**.password'`, + * `'*_token'`, `'user.email'`) are replaced with `'[REDACTED]'` so PII * never leaks through the diff. * * @example @@ -324,7 +324,12 @@ export function auditDiff( options: AuditDiffOptions = {}, ): { before?: unknown, after?: unknown, patch: AuditPatchOp[] } { const replacement = options.replacement ?? '[REDACTED]' - const keyMatcher = buildKeyMatcher(options.redactPaths) ?? (() => false) + const pathMatchers = compileRedactPathMatchers(options.redactPaths) ?? { + exactPaths: new Set(), + pathGlobs: [], + keyGlobs: [], + caseInsensitiveLeaves: new Set(), + } const patch: AuditPatchOp[] = [] function diff(a: unknown, b: unknown, path: string): void { @@ -355,7 +360,7 @@ export function auditDiff( } function redactValue(value: unknown, path: string): unknown { - return redactValueByKeys(value, keyMatcher, replacement, path) + return redactValueByPaths(value, pathMatchers, replacement, path) } diff(before, after, '') @@ -369,7 +374,7 @@ export type { AuditPatchOp } from './types' /** Options for {@link auditDiff}. */ export interface AuditDiffOptions { - /** Object keys (dot-notation) whose values should be replaced with `[REDACTED]`. */ + /** Path globs (same syntax as `RedactConfig.paths`) whose values are replaced with `[REDACTED]`. */ redactPaths?: string[] /** Custom replacement string. @default '[REDACTED]' */ replacement?: string @@ -846,7 +851,7 @@ function stripIntegrity(event: WideEvent): WideEvent { * Strict redact preset for audit events. * * Combine with the user's existing redact configuration via spread: - * `initLogger({ redact: { keys: [...auditRedactPreset.keys!, ...mine] } })`. + * `initLogger({ redact: { paths: [...auditRedactPreset.paths!, ...mine] } })`. * * Hardens PII handling: * - Drops `Authorization` and `Cookie` headers anywhere they appear. @@ -859,7 +864,7 @@ function stripIntegrity(event: WideEvent): WideEvent { * enough signal to be useful. */ export const auditRedactPreset: RedactConfig = { - keys: [ + paths: [ 'password', 'passwordHash', 'token', diff --git a/packages/evlog/src/redact.ts b/packages/evlog/src/redact.ts index b948abe0..d424911e 100644 --- a/packages/evlog/src/redact.ts +++ b/packages/evlog/src/redact.ts @@ -1,41 +1,122 @@ import type { RedactConfig } from './types' +import { globToRegExp } from './utils' const DEFAULT_REPLACEMENT = '[REDACTED]' export type Masker = [RegExp, (match: string) => string] -/** Predicate that returns whether an object key name should be fully redacted. */ -export type KeyMatcher = (key: string) => boolean +/** Compiled matchers for {@link RedactConfig.paths} glob patterns. */ +export interface RedactPathMatchers { + exactPaths: Set + pathGlobs: RegExp[] + keyGlobs: RegExp[] + /** Single-segment shorthands (`password` → `**.password`) matched case-insensitively on leaf keys. */ + caseInsensitiveLeaves: Set +} /** - * Build a matcher from exact key names and regex patterns on key names. - * Returns `undefined` when both inputs are empty. + * Normalize a redact path pattern. + * Single segments without wildcards are shorthand for `**.`. */ -export function buildKeyMatcher(keys?: string[], keyPatterns?: RegExp[]): KeyMatcher | undefined { - const keySet = new Set(keys ?? []) - const patterns = (keyPatterns ?? []).map(cloneRegex) - if (keySet.size === 0 && patterns.length === 0) return undefined - - return (key: string) => { - if (keySet.has(key)) return true - for (const pattern of patterns) { - pattern.lastIndex = 0 - if (pattern.test(key)) return true +export function normalizeRedactPathPattern(pattern: string): string { + if (!pattern.includes('*') && !pattern.includes('.')) { + return `**.${pattern}` + } + return pattern +} + +/** + * Compile `RedactConfig.paths` into exact paths, path globs, and key globs. + * Returns `undefined` when `patterns` is empty. + */ +export function compileRedactPathMatchers(patterns?: string[]): RedactPathMatchers | undefined { + if (!patterns?.length) return undefined + + const exactPaths = new Set() + const pathGlobs: RegExp[] = [] + const keyGlobs: RegExp[] = [] + const caseInsensitiveLeaves = new Set() + + for (const raw of patterns) { + if (!raw.includes('*')) { + if (raw.includes('.')) { + exactPaths.add(raw) + } else { + addPathGlobPattern(normalizeRedactPathPattern(raw), exactPaths, pathGlobs, caseInsensitiveLeaves) + } + continue + } + + if (!raw.includes('.')) { + keyGlobs.push(globToRegExp(raw, '.')) + } else { + addPathGlobPattern(raw, exactPaths, pathGlobs, caseInsensitiveLeaves) } - return false } + + if (exactPaths.size === 0 && pathGlobs.length === 0 && keyGlobs.length === 0 && caseInsensitiveLeaves.size === 0) { + return undefined + } + + return { exactPaths, pathGlobs, keyGlobs, caseInsensitiveLeaves } +} + +/** `**.segment` also matches a top-level `segment` field. */ +function addPathGlobPattern( + pattern: string, + exactPaths: Set, + pathGlobs: RegExp[], + caseInsensitiveLeaves: Set, +): void { + pathGlobs.push(globToRegExp(pattern, '.')) + const leaf = pattern.match(/^\*\*\.([^.?*]+)$/) + if (leaf) { + exactPaths.add(leaf[1]!) + caseInsensitiveLeaves.add(leaf[1]!) + } +} + +/** + * Whether a field at `fullPath` (dot-notation from root) with leaf key `leafKey` + * should be fully redacted. + */ +export function matchesRedactPath(fullPath: string, leafKey: string, matchers: RedactPathMatchers): boolean { + if (matchers.exactPaths.has(fullPath)) return true + + const leafLower = leafKey.toLowerCase() + for (const name of matchers.caseInsensitiveLeaves) { + if (leafLower === name.toLowerCase()) return true + } + + for (const glob of matchers.pathGlobs) { + glob.lastIndex = 0 + if (glob.test(fullPath)) return true + } + + for (const glob of matchers.keyGlobs) { + glob.lastIndex = 0 + if (glob.test(leafKey)) return true + } + + return false } /** - * Redact values whose key names match `matcher`, recursively at any depth. - * Mutates `obj` in place (intended for use on a clone). + * Redact fields matching path globs recursively. Mutates `obj` in place (use on a clone). */ -export function redactKeysInTree(obj: unknown, matcher: KeyMatcher, replacement: string): void { +export function redactPathsInTree( + obj: unknown, + matchers: RedactPathMatchers, + replacement: string, + prefix = '', +): void { if (obj === null || obj === undefined) return if (Array.isArray(obj)) { - for (const item of obj) { - redactKeysInTree(item, matcher, replacement) + for (let i = 0; i < obj.length; i++) { + const segment = String(i) + const fullPath = prefix ? `${prefix}.${segment}` : segment + redactPathsInTree(obj[i], matchers, replacement, fullPath) } return } @@ -43,44 +124,50 @@ export function redactKeysInTree(obj: unknown, matcher: KeyMatcher, replacement: if (typeof obj === 'object') { const record = obj as Record for (const key in record) { - if (matcher(key)) { + const fullPath = prefix ? `${prefix}.${key}` : key + if (matchesRedactPath(fullPath, key, matchers)) { record[key] = replacement } else { - redactKeysInTree(record[key], matcher, replacement) + redactPathsInTree(record[key], matchers, replacement, fullPath) } } } } /** - * Return a copy of `value` with key-name matches replaced by `replacement`. + * Return a copy of `value` with path-pattern matches replaced by `replacement`. * Used by audit diffs; does not mutate the input. * - * When `value` is a scalar and `path` is provided, the last JSON Pointer - * segment is checked against `matcher` (for patch leaf values). + * `pointerPath` is a JSON Pointer (e.g. `/user/password`). */ -export function redactValueByKeys( +export function redactValueByPaths( value: unknown, - matcher: KeyMatcher, + matchers: RedactPathMatchers, replacement: string, - path?: string, + pointerPath = '', ): unknown { + const segments = pointerPath.split('/').filter(Boolean) + const dotPath = segments.join('.') + const leafKey = segments.at(-1) ?? '' + if (value === null || typeof value !== 'object') { - if (path) { - const last = path.split('/').filter(Boolean).at(-1) - if (last && matcher(last)) return replacement - } + if (dotPath && matchesRedactPath(dotPath, leafKey, matchers)) return replacement return value } if (Array.isArray(value)) { - return value.map((v, i) => redactValueByKeys(v, matcher, replacement, path ? `${path}/${i}` : undefined)) + return value.map((v, i) => redactValueByPaths(v, matchers, replacement, `${pointerPath}/${i}`)) } + if (!isPlainRecord(value)) return value + const out: Record = {} - for (const [k, v] of Object.entries(value as Record)) { - const childPath = path ? `${path}/${k}` : k - out[k] = matcher(k) ? replacement : redactValueByKeys(v, matcher, replacement, childPath) + for (const [k, v] of Object.entries(value)) { + const childPointer = pointerPath ? `${pointerPath}/${k}` : `/${k}` + const childDot = dotPath ? `${dotPath}.${k}` : k + out[k] = matchesRedactPath(childDot, k, matchers) + ? replacement + : redactValueByPaths(v, matchers, replacement, childPointer) } return out } @@ -229,11 +316,10 @@ function cloneForRedaction(event: Record): Record, config: RedactConfig const clone = cloneForRedaction(event) const replacement = config.replacement ?? DEFAULT_REPLACEMENT - const keyMatcher = buildKeyMatcher(config.keys, config.keyPatterns) - if (keyMatcher) { - redactKeysInTree(clone, keyMatcher, replacement) - } - - if (config.paths?.length) { - for (const path of config.paths) { - redactPath(clone, path.split('.'), replacement) - } + const pathMatchers = compileRedactPathMatchers(config.paths) + if (pathMatchers) { + redactPathsInTree(clone, pathMatchers, replacement) } if (config._maskers?.length) { @@ -265,21 +345,6 @@ export function redactEvent(event: Record, config: RedactConfig return clone } -function redactPath(obj: Record, segments: string[], replacement: string): void { - let current: unknown = obj - for (let i = 0; i < segments.length - 1; i++) { - if (current === null || current === undefined || typeof current !== 'object') return - current = (current as Record)[segments[i]!] - } - - if (current === null || current === undefined || typeof current !== 'object') return - - const leaf = segments[segments.length - 1]! - if (leaf in (current as Record)) { - (current as Record)[leaf] = replacement - } -} - function redactPatterns(obj: unknown, patterns: RegExp[], replacement: string): void { if (obj === null || obj === undefined) return @@ -367,10 +432,6 @@ export function normalizeRedactConfig(raw: boolean | Record | u config.paths = raw.paths as string[] } - if (Array.isArray(raw.keys)) { - config.keys = raw.keys as string[] - } - if (typeof raw.replacement === 'string') { config.replacement = raw.replacement } @@ -385,21 +446,36 @@ export function normalizeRedactConfig(raw: boolean | Record | u config.patterns = deserializeRegexList(raw.patterns) } - if (Array.isArray(raw.keyPatterns)) { - config.keyPatterns = deserializeRegexList(raw.keyPatterns) - } - return resolveRedactConfig(config) } +function isPlainRecord(value: unknown): value is Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) return false + const proto = Object.getPrototypeOf(value) + return proto === Object.prototype || proto === null +} + function deserializeRegexList(raw: unknown[]): RegExp[] { - return raw.map((p) => { - if (p instanceof RegExp) return p - if (typeof p === 'string') return new RegExp(p, 'g') - if (typeof p === 'object' && p !== null) { - const obj = p as Record - return new RegExp(obj.source, obj.flags ?? 'g') + const patterns: RegExp[] = [] + for (const p of raw) { + try { + if (p instanceof RegExp) { + patterns.push(cloneRegex(p)) + continue + } + if (typeof p === 'string') { + patterns.push(new RegExp(p, 'g')) + continue + } + if (typeof p === 'object' && p !== null && typeof (p as { source?: unknown }).source === 'string') { + const flags = typeof (p as { flags?: unknown }).flags === 'string' + ? (p as { flags: string }).flags + : 'g' + patterns.push(new RegExp((p as { source: string }).source, flags)) + } + } catch { + console.warn('[normalizeRedactConfig] Ignoring invalid redact regex entry') } - return null - }).filter((p): p is RegExp => p !== null) + } + return patterns } diff --git a/packages/evlog/src/types.ts b/packages/evlog/src/types.ts index 2bf7a0d7..de9e82ac 100644 --- a/packages/evlog/src/types.ts +++ b/packages/evlog/src/types.ts @@ -99,33 +99,26 @@ export interface IngestPayload { */ export interface RedactConfig { /** - * Exact dot-notation paths to redact (e.g. `'user.email'`, `'headers.x-forwarded-for'`). - * Only the leaf at that path is replaced — does not match the same key name elsewhere. + * Dot-notation paths to redact. Supports globs: + * - `'user.email'` — exact path only + * - `'password'` or `'**.password'` — key at any nesting depth + * - `'*_token'` — key-name glob at any depth + * - `'user.*'` — path glob under `user` */ paths?: string[] - /** - * Object key names to redact at any nesting depth (e.g. `'password'` redacts - * `user.password`, `data.a.b.password`, etc.). Replaces the entire value. - */ - keys?: string[] - /** - * RegExp patterns matched against object key names at any nesting depth. - * Replaces the entire value (e.g. `/.*_token$/` matches `access_token`, `refresh_token`). - */ - keyPatterns?: RegExp[] /** Additional regex patterns to match and replace string values anywhere in the event */ patterns?: RegExp[] /** * Control built-in PII patterns. * - `undefined` / omitted → all built-ins enabled (default) - * - `false` → no built-ins, only custom `paths`/`patterns` + * - `false` → no built-ins, only custom `paths` and `patterns` * - `['email', 'creditCard', ...]` → only the listed built-ins * * Available: `'creditCard'`, `'email'`, `'ipv4'`, `'phone'`, `'jwt'`, `'bearer'`, `'iban'` */ builtins?: false | Array<'creditCard' | 'email' | 'ipv4' | 'phone' | 'jwt' | 'bearer' | 'iban'> /** - * Replacement string used for path-, key-, and custom pattern redaction. + * Replacement string used for path- and custom pattern redaction. * Built-in patterns use smart partial masking instead (e.g. `****1111` for credit cards). * @default '[REDACTED]' */ diff --git a/packages/evlog/src/utils.ts b/packages/evlog/src/utils.ts index 657e0f18..2ce976bf 100644 --- a/packages/evlog/src/utils.ts +++ b/packages/evlog/src/utils.ts @@ -148,20 +148,34 @@ export function filterSafeHeaders(headers: Partial() /** - * Match a path against a glob pattern. - * Supports * (any chars except /) and ** (any chars including /). + * Compile a glob pattern to a anchored RegExp. + * + * - `*` matches any characters except the separator + * - `**` matches any characters including the separator + * - `?` matches one character except the separator */ -export function matchesPattern(path: string, pattern: string): boolean { - let regex = patternCache.get(pattern) +export function globToRegExp(pattern: string, separator: '/' | '.' = '/'): RegExp { + const cacheKey = `${separator}:${pattern}` + let regex = patternCache.get(cacheKey) if (!regex) { + const segment = separator === '/' ? '[^/]*' : '[^.]*' + const char = separator === '/' ? '[^/]' : '[^.]' const regexPattern = pattern .replace(/[.+^${}()|[\]\\]/g, '\\$&') .replace(/\*\*/g, '{{GLOBSTAR}}') - .replace(/\*/g, '[^/]*') + .replace(/\*/g, segment) .replace(/{{GLOBSTAR}}/g, '.*') - .replace(/\?/g, '[^/]') + .replace(/\?/g, char) regex = new RegExp(`^${regexPattern}$`) - patternCache.set(pattern, regex) + patternCache.set(cacheKey, regex) } - return regex.test(path) + return regex +} + +/** + * Match a path against a glob pattern. + * Supports * (any chars except /) and ** (any chars including /). + */ +export function matchesPattern(path: string, pattern: string): boolean { + return globToRegExp(pattern, '/').test(path) } diff --git a/packages/evlog/test/core/audit.test.ts b/packages/evlog/test/core/audit.test.ts index af5f80e7..03c21c09 100644 --- a/packages/evlog/test/core/audit.test.ts +++ b/packages/evlog/test/core/audit.test.ts @@ -209,6 +209,24 @@ describe('auditDiff()', () => { expect(diff.patch).toContainEqual({ op: 'replace', path: '/user/password', value: '[REDACTED]' }) }) + it('redacts exact dotted paths without matching other branches', () => { + const diff = auditDiff( + { user: { password: 'old' }, admin: { password: 'secret' } }, + { user: { password: 'new' }, admin: { password: 'secret' } }, + { redactPaths: ['user.password'] }, + ) + expect(diff.patch).toContainEqual({ op: 'replace', path: '/user/password', value: '[REDACTED]' }) + expect(diff.patch).not.toContainEqual({ op: 'replace', path: '/admin/password', value: '[REDACTED]' }) + }) + + it('preserves non-plain objects in snapshots', () => { + const before = { at: new Date('2026-01-01T00:00:00.000Z') } + const after = { at: new Date('2026-06-01T00:00:00.000Z') } + const diff = auditDiff(before, after, { includeBefore: true, includeAfter: true }) + expect((diff.before as { at: Date }).at).toBeInstanceOf(Date) + expect((diff.after as { at: Date }).at).toBeInstanceOf(Date) + }) + it('emits add and remove operations', () => { const diff = auditDiff({ a: 1 }, { b: 2 }) expect(diff.patch).toEqual(expect.arrayContaining([ @@ -508,21 +526,21 @@ describe('end-to-end: audit + auditOnly + global drain', () => { }) describe('auditRedactPreset', () => { - it('redacts authorization and cookie key names at any depth', () => { + it('redacts authorization and cookie path globs at any depth', () => { const config = defined(resolveRedactConfig(auditRedactPreset), 'audit redact preset') - expect(config.keys).toContain('authorization') - expect(config.keys).toContain('cookie') - expect(config.keys).toContain('set-cookie') + expect(config.paths).toContain('authorization') + expect(config.paths).toContain('cookie') + expect(config.paths).toContain('set-cookie') }) - it('redacts credential key names at any depth', () => { + it('redacts credential path globs at any depth', () => { const config = defined(resolveRedactConfig(auditRedactPreset), 'audit redact preset') - expect(config.keys).toContain('password') - expect(config.keys).toContain('token') - expect(config.keys).toContain('apiKey') + expect(config.paths).toContain('password') + expect(config.paths).toContain('token') + expect(config.paths).toContain('apiKey') }) - it('redacts nested audit.changes password fields via keys', () => { + it('redacts nested audit.changes password fields via path globs', () => { const config = defined(resolveRedactConfig(auditRedactPreset), 'audit redact preset') const event = redactEvent( { diff --git a/packages/evlog/test/core/redact.test.ts b/packages/evlog/test/core/redact.test.ts index a135ea1d..f63a6c7f 100644 --- a/packages/evlog/test/core/redact.test.ts +++ b/packages/evlog/test/core/redact.test.ts @@ -69,26 +69,24 @@ describe('redactEvent - path-based', () => { expect(user.settings).toBe('[REDACTED]') }) - it('does not redact nested password with top-level path only', () => { - const source: Record = { - password: 'top', - user: { password: 'nested' }, - } - const event = redactEvent(source, { paths: ['password'] }) - expect(event.password).toBe('[REDACTED]') - expect((event.user as Record).password).toBe('nested') - expect(source.user).toEqual({ password: 'nested' }) + it('does not redact nested email when exact path is user.email only', () => { + const event = redactEvent( + { user: { email: 'a@b.com' }, other: { email: 'c@d.com' } }, + { paths: ['user.email'] }, + ) + expect((event.user as Record).email).toBe('[REDACTED]') + expect((event.other as Record).email).toBe('c@d.com') }) }) -describe('redactEvent - key-based', () => { - it('redacts a key at any nesting depth', () => { +describe('redactEvent - path globs', () => { + it('redacts password at any nesting depth via shorthand', () => { const source: Record = { password: 'top', user: { name: 'Alice', password: 'secret' }, data: { a: { b: { password: 'deep' } } }, } - const event = redactEvent(source, { keys: ['password'] }) + const event = redactEvent(source, { paths: ['password'] }) expect(event.password).toBe('[REDACTED]') expect((event.user as Record).password).toBe('[REDACTED]') expect((event.user as Record).name).toBe('Alice') @@ -97,22 +95,32 @@ describe('redactEvent - key-based', () => { expect(source.user).toEqual({ name: 'Alice', password: 'secret' }) }) + it('treats password and **.password as equivalent', () => { + const data = { + user: { password: 'secret' }, + password: 'top', + } + const shorthand = redactEvent(data, { paths: ['password'] }) + const explicit = redactEvent(data, { paths: ['**.password'] }) + expect(shorthand).toEqual(explicit) + }) + it('replaces the entire matched value without recursing into children', () => { const event = redactEvent( { password: { nested: 'still-here' } }, - { keys: ['password'] }, + { paths: ['password'] }, ) expect(event.password).toBe('[REDACTED]') }) - it('redacts multiple key names', () => { + it('redacts multiple shorthand segments', () => { const event = redactEvent( { password: 'p', token: 't', user: { apiKey: 'k', name: 'Alice' }, }, - { keys: ['password', 'token', 'apiKey'] }, + { paths: ['password', 'token', 'apiKey'] }, ) expect(event.password).toBe('[REDACTED]') expect(event.token).toBe('[REDACTED]') @@ -120,29 +128,43 @@ describe('redactEvent - key-based', () => { expect((event.user as Record).name).toBe('Alice') }) - it('matches key names with keyPatterns', () => { + it('matches key-name globs without dots', () => { const event = redactEvent( { access_token: 'a', refresh_token: 'r', + nested: { access_token: 'n' }, username: 'alice', }, - { keyPatterns: [/.*_token$/] }, + { paths: ['*_token'] }, ) expect(event.access_token).toBe('[REDACTED]') expect(event.refresh_token).toBe('[REDACTED]') + expect((event.nested as Record).access_token).toBe('[REDACTED]') expect(event.username).toBe('alice') }) - it('combines keys and exact paths', () => { + it('matches path globs under a prefix', () => { + const event = redactEvent( + { + user: { email: 'a@b.com', password: 'secret' }, + admin: { email: 'c@d.com' }, + }, + { paths: ['user.*'] }, + ) + expect((event.user as Record).email).toBe('[REDACTED]') + expect((event.user as Record).password).toBe('[REDACTED]') + expect((event.admin as Record).email).toBe('c@d.com') + }) + + it('combines globs and exact paths', () => { const event = redactEvent( { password: 'everywhere', headers: { 'x-forwarded-for': '1.2.3.4', authorization: 'Bearer x' }, }, { - keys: ['password', 'authorization'], - paths: ['headers.x-forwarded-for'], + paths: ['password', 'authorization', 'headers.x-forwarded-for'], }, ) expect(event.password).toBe('[REDACTED]') @@ -153,10 +175,39 @@ describe('redactEvent - key-based', () => { it('uses custom replacement string', () => { const event = redactEvent( { user: { password: 'secret' } }, - { keys: ['password'], replacement: '***' }, + { paths: ['password'], replacement: '***' }, ) expect((event.user as Record).password).toBe('***') }) + + it('redacts header keys case-insensitively', () => { + const event = redactEvent( + { + headers: { Authorization: 'Bearer x', Cookie: 'sid=1' }, + }, + { paths: ['authorization', 'cookie'] }, + ) + const headers = event.headers as Record + expect(headers.Authorization).toBe('[REDACTED]') + expect(headers.Cookie).toBe('[REDACTED]') + }) + + it('redacts audit.changes nested passwords via segment shorthand', () => { + const event = redactEvent( + { + audit: { + changes: { + before: { password: 'old' }, + after: { password: 'new' }, + }, + }, + }, + { paths: ['password'] }, + ) + const changes = (event.audit as Record).changes as Record + expect((changes.before as Record).password).toBe('[REDACTED]') + expect((changes.after as Record).password).toBe('[REDACTED]') + }) }) describe('redactEvent - pattern-based', () => { @@ -396,17 +447,6 @@ describe('normalizeRedactConfig', () => { expect(config?.replacement).toBe('***') }) - it('preserves keys and deserializes keyPatterns', () => { - const config = defined(normalizeRedactConfig({ - keys: ['password', 'token'], - keyPatterns: ['.*_secret$'], - builtins: false, - }), 'redact config') - expect(config.keys).toEqual(['password', 'token']) - expect(config.keyPatterns).toHaveLength(1) - expect(defined(config.keyPatterns?.[0], 'keyPatterns[0]').source).toBe('.*_secret$') - }) - it('converts string patterns to RegExp separately from built-in maskers', () => { const config = defined(normalizeRedactConfig({ patterns: ['\\b\\d{4}\\b'], @@ -433,7 +473,9 @@ describe('normalizeRedactConfig', () => { builtins: false, patterns: [re], }), 'redact config') - expect(defined(config.patterns?.[0], 'patterns[0]')).toBe(re) + const pattern = defined(config.patterns?.[0], 'patterns[0]') + expect(pattern.source).toBe(re.source) + expect(pattern.flags).toBe(re.flags) }) it('filters out invalid pattern entries', () => { @@ -444,6 +486,17 @@ describe('normalizeRedactConfig', () => { expect(config?.patterns).toHaveLength(1) }) + it('ignores invalid regex sources without throwing', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const config = normalizeRedactConfig({ + builtins: false, + patterns: ['valid', '(unclosed', { source: '[', flags: 'g' }], + }) + expect(config?.patterns).toHaveLength(1) + expect(warn).toHaveBeenCalled() + warn.mockRestore() + }) + it('handles builtins field from deserialized JSON', () => { const config = normalizeRedactConfig({ builtins: ['email', 'creditCard'],