From bc88529933cc187e787dca768eabd0bd60720eb2 Mon Sep 17 00:00:00 2001 From: toolmanlab Date: Mon, 23 Mar 2026 16:11:17 +0800 Subject: [PATCH 1/2] feat(browser): human-like delay system for anti-detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a framework-level delay/jitter system using log-normal distribution to simulate natural browsing patterns, addressing issue #59 (P0). - New `HumanDelay` class with configurable profiles (none/fast/moderate/cautious/stealth) - Log-normal distribution for realistic delay variance (not uniform) - Periodic "breaks" that simulate reading/thinking pauses - Auto-injected between page.goto() navigations - Configurable via OPENCLI_DELAY_PROFILE env var - Boss search adapter migrated from hardcoded jitter to framework delay - 10 unit tests covering all profiles and edge cases Real-world validation against a major job board (cookie-authenticated, aggressive bot detection): | Scenario | Without jitter | With jitter | |-----------------------|--------------------|--------------------| | 50 detail pages | ✅ OK | ✅ OK | | 200 detail pages | ❌ Banned (code 32) | ✅ OK | | 850 requests over 5h | N/A (banned early) | ✅ Zero detection | | 4-day sustained crawl | N/A | ✅ 1800+ records | Closes #59 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/browser/page.ts | 19 ++++- src/clis/boss/search.ts | 5 +- src/human-delay.test.ts | 100 +++++++++++++++++++++++ src/human-delay.ts | 177 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 src/human-delay.test.ts create mode 100644 src/human-delay.ts diff --git a/src/browser/page.ts b/src/browser/page.ts index 79239548..1cc283cd 100644 --- a/src/browser/page.ts +++ b/src/browser/page.ts @@ -12,6 +12,7 @@ import { formatSnapshot } from '../snapshotFormatter.js'; import type { BrowserCookie, IPage, ScreenshotOptions, SnapshotOptions, WaitOptions } from '../types.js'; +import { HumanDelay, resolveProfile } from '../human-delay.js'; import { sendCommand } from './daemon-client.js'; import { wrapForEval } from './utils.js'; import { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js'; @@ -30,7 +31,14 @@ import { * Page — implements IPage by talking to the daemon via HTTP. */ export class Page implements IPage { - constructor(private readonly workspace: string = 'default') {} + /** Human-like delay generator for anti-detection. */ + private _delay: HumanDelay; + /** Whether goto() has been called at least once (skip delay on first navigation). */ + private _hasNavigated = false; + + constructor(private readonly workspace: string = 'default') { + this._delay = new HumanDelay(resolveProfile()); + } /** Active tab ID, set after navigate and used in all subsequent commands */ private _tabId: number | undefined; @@ -45,6 +53,15 @@ export class Page implements IPage { } async goto(url: string, options?: { waitUntil?: 'load' | 'none'; settleMs?: number }): Promise { + // Human-like delay between navigations (skip the very first one) + if (this._hasNavigated) { + const ms = await this._delay.sleep(); + if (ms > 0 && process.env.OPENCLI_VERBOSE) { + console.error(`[opencli] Human delay: ${(ms / 1000).toFixed(1)}s`); + } + } + this._hasNavigated = true; + const result = await sendCommand('navigate', { url, ...this._workspaceOpt(), diff --git a/src/clis/boss/search.ts b/src/clis/boss/search.ts index 69852573..59195a90 100644 --- a/src/clis/boss/search.ts +++ b/src/clis/boss/search.ts @@ -3,6 +3,7 @@ */ import { cli, Strategy } from '../../registry.js'; import { requirePage, navigateTo, bossFetch, assertOk, verbose } from './common.js'; +import { humanSleep } from '../../human-delay.js'; /** City name → BOSS Zhipin city code mapping */ const CITY_CODES: Record = { @@ -103,7 +104,9 @@ cli({ while (allJobs.length < limit) { if (allJobs.length > 0) { - await new Promise(r => setTimeout(r, 1000 + Math.random() * 2000)); + // Use framework-level human-like delay instead of fixed jitter + const ms = await humanSleep(); + verbose(`Human delay: ${(ms / 1000).toFixed(1)}s before page ${currentPage}`); } const qs = new URLSearchParams({ diff --git a/src/human-delay.test.ts b/src/human-delay.test.ts new file mode 100644 index 00000000..3734f4f1 --- /dev/null +++ b/src/human-delay.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from 'vitest'; +import { HumanDelay, PROFILES, jitterMs, resolveProfile } from './human-delay.js'; + +describe('HumanDelay', () => { + it('none profile returns 0', () => { + const delay = new HumanDelay('none'); + for (let i = 0; i < 20; i++) { + expect(delay.next()).toBe(0); + } + }); + + it('moderate profile stays within bounds', () => { + const delay = new HumanDelay('moderate'); + const values: number[] = []; + for (let i = 0; i < 100; i++) { + values.push(delay.next()); + } + // Separate normal delays from breaks (breaks are >= breakDurationMs[0]) + const breakMin = PROFILES.moderate.breakDurationMs[0]; + const nonBreaks = values.filter(v => v < breakMin); + for (const v of nonBreaks) { + expect(v).toBeGreaterThanOrEqual(PROFILES.moderate.minMs); + expect(v).toBeLessThanOrEqual(PROFILES.moderate.maxMs); + } + }); + + it('cautious profile produces breaks within interval range', () => { + const delay = new HumanDelay('cautious'); + const values: number[] = []; + // Run enough iterations to trigger at least one break (breakEveryMin=8) + for (let i = 0; i < 50; i++) { + values.push(delay.next()); + } + const breaks = values.filter( + v => v >= PROFILES.cautious.breakDurationMs[0] + ); + expect(breaks.length).toBeGreaterThan(0); + for (const b of breaks) { + expect(b).toBeGreaterThanOrEqual(PROFILES.cautious.breakDurationMs[0]); + expect(b).toBeLessThanOrEqual(PROFILES.cautious.breakDurationMs[1]); + } + }); + + it('reset() restarts action counter', () => { + const delay = new HumanDelay('cautious'); + for (let i = 0; i < 5; i++) delay.next(); + delay.reset(); + // After reset, next break should be at least breakEveryMin actions away + const values: number[] = []; + for (let i = 0; i < PROFILES.cautious.breakEveryMin - 1; i++) { + values.push(delay.next()); + } + // All should be normal delays (no breaks yet) + const breaks = values.filter(v => v >= PROFILES.cautious.breakDurationMs[0]); + expect(breaks.length).toBe(0); + }); + + it('distribution has reasonable variance (not uniform)', () => { + const delay = new HumanDelay('moderate'); + const values: number[] = []; + for (let i = 0; i < 200; i++) { + values.push(delay.next()); + } + const nonBreaks = values.filter(v => v < PROFILES.moderate.breakDurationMs[0]); + const mean = nonBreaks.reduce((a, b) => a + b, 0) / nonBreaks.length; + const variance = nonBreaks.reduce((a, b) => a + (b - mean) ** 2, 0) / nonBreaks.length; + const stddev = Math.sqrt(variance); + // Log-normal should have meaningful spread — stddev > 10% of mean + expect(stddev / mean).toBeGreaterThan(0.1); + }); +}); + +describe('jitterMs', () => { + it('returns 0 for none profile', () => { + expect(jitterMs('none')).toBe(0); + }); + + it('returns value within profile bounds', () => { + for (let i = 0; i < 50; i++) { + const ms = jitterMs('fast'); + expect(ms).toBeGreaterThanOrEqual(PROFILES.fast.minMs); + expect(ms).toBeLessThanOrEqual(PROFILES.fast.maxMs); + } + }); +}); + +describe('resolveProfile', () => { + it('returns moderate by default', () => { + expect(resolveProfile()).toBe(PROFILES.moderate); + }); + + it('resolves named profiles', () => { + expect(resolveProfile('stealth')).toBe(PROFILES.stealth); + expect(resolveProfile('none')).toBe(PROFILES.none); + }); + + it('falls back to moderate for unknown names', () => { + expect(resolveProfile('nonexistent')).toBe(PROFILES.moderate); + }); +}); diff --git a/src/human-delay.ts b/src/human-delay.ts new file mode 100644 index 00000000..d077bdd8 --- /dev/null +++ b/src/human-delay.ts @@ -0,0 +1,177 @@ +/** + * Human-like delay system for anti-detection. + * + * Uses log-normal distribution to simulate natural browsing patterns: + * - Most delays cluster around the median (feels natural) + * - Occasional longer delays (simulates reading/thinking) + * - Periodic "breaks" (simulates distraction or task switching) + * + * Addresses: https://github.com/jackwener/opencli/issues/59 (P0: request interval jitter) + */ + +// ─── Delay Profiles ───────────────────────────────────────────────── + +export interface DelayProfile { + /** Log-normal mu parameter (controls median delay) */ + mu: number; + /** Log-normal sigma parameter (controls spread) */ + sigma: number; + /** Minimum delay in ms */ + minMs: number; + /** Maximum delay in ms */ + maxMs: number; + /** Break interval: take a long pause every N actions (0 to disable) */ + breakEveryMin: number; + /** Break interval max (actual interval randomized between min..max) */ + breakEveryMax: number; + /** Break duration range in ms [min, max] */ + breakDurationMs: [number, number]; +} + +export const PROFILES: Record = { + /** No delays — for CI, testing, or trusted environments */ + none: { + mu: 0, sigma: 0, minMs: 0, maxMs: 0, + breakEveryMin: 0, breakEveryMax: 0, breakDurationMs: [0, 0], + }, + /** Light jitter — minimal detection risk, fast throughput */ + fast: { + mu: 6.9, sigma: 0.3, minMs: 500, maxMs: 3000, + breakEveryMin: 0, breakEveryMax: 0, breakDurationMs: [0, 0], + }, + /** Moderate — balanced speed and safety (default) */ + moderate: { + mu: 7.6, sigma: 0.4, minMs: 1000, maxMs: 8000, + breakEveryMin: 15, breakEveryMax: 25, breakDurationMs: [5000, 15000], + }, + /** Conservative — for aggressive anti-bot sites */ + cautious: { + mu: 8.3, sigma: 0.5, minMs: 3000, maxMs: 25000, + breakEveryMin: 8, breakEveryMax: 12, breakDurationMs: [15000, 60000], + }, + /** Stealth — maximum evasion, very slow */ + stealth: { + mu: 9.0, sigma: 0.5, minMs: 5000, maxMs: 40000, + breakEveryMin: 6, breakEveryMax: 10, breakDurationMs: [30000, 90000], + }, +}; + +// ─── Delay Generator ──────────────────────────────────────────────── + +/** + * Stateful delay generator that tracks action count for periodic breaks. + */ +export class HumanDelay { + private _profile: DelayProfile; + private _actionCount = 0; + private _nextBreakAt = 0; + + constructor(profile?: string | DelayProfile) { + if (typeof profile === 'string') { + this._profile = PROFILES[profile] ?? PROFILES.moderate; + } else { + this._profile = profile ?? PROFILES.moderate; + } + this._scheduleNextBreak(); + } + + get profileName(): string { + for (const [name, p] of Object.entries(PROFILES)) { + if (p === this._profile) return name; + } + return 'custom'; + } + + /** Generate a log-normal random delay in ms. */ + private _lognormalDelay(): number { + const { mu, sigma, minMs, maxMs } = this._profile; + if (maxMs <= 0) return 0; + + // Box-Muller transform for normal distribution + const u1 = Math.random(); + const u2 = Math.random(); + const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2); + const raw = Math.exp(mu / 1000 + sigma * z); // scale mu from "micro" to seconds-ish + + return Math.max(minMs, Math.min(raw * 1000, maxMs)); + } + + private _scheduleNextBreak(): void { + const { breakEveryMin, breakEveryMax } = this._profile; + if (breakEveryMin <= 0) { + this._nextBreakAt = Infinity; + return; + } + this._nextBreakAt = this._actionCount + + breakEveryMin + Math.floor(Math.random() * (breakEveryMax - breakEveryMin + 1)); + } + + /** Return the delay to wait (in ms) before the next action. */ + next(): number { + if (this._profile.maxMs <= 0) return 0; + + this._actionCount++; + + // Check if it's time for a break + if (this._actionCount >= this._nextBreakAt) { + this._scheduleNextBreak(); + const [minBreak, maxBreak] = this._profile.breakDurationMs; + return minBreak + Math.random() * (maxBreak - minBreak); + } + + return this._lognormalDelay(); + } + + /** Convenience: await this to sleep for the computed delay. */ + async sleep(): Promise { + const ms = this.next(); + if (ms > 0) { + await new Promise(resolve => setTimeout(resolve, ms)); + } + return ms; + } + + /** Reset action counter (e.g., between different command sessions). */ + reset(): void { + this._actionCount = 0; + this._scheduleNextBreak(); + } +} + +// ─── Module-level helpers ─────────────────────────────────────────── + +/** + * Resolve the active delay profile from environment or explicit name. + * + * Priority: + * 1. Explicit `profile` argument + * 2. OPENCLI_DELAY_PROFILE env var + * 3. 'moderate' default + */ +export function resolveProfile(profile?: string): DelayProfile { + const name = profile ?? process.env.OPENCLI_DELAY_PROFILE ?? 'moderate'; + return PROFILES[name] ?? PROFILES.moderate; +} + +/** + * One-shot jitter: returns a random delay in ms using the resolved profile. + * Stateless — does not track break intervals. + */ +export function jitterMs(profile?: string): number { + const p = resolveProfile(profile); + if (p.maxMs <= 0) return 0; + const u1 = Math.random(); + const u2 = Math.random(); + const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2); + const raw = Math.exp(p.mu / 1000 + p.sigma * z); + return Math.max(p.minMs, Math.min(raw * 1000, p.maxMs)); +} + +/** + * One-shot sleep with jitter. + */ +export async function humanSleep(profile?: string): Promise { + const ms = jitterMs(profile); + if (ms > 0) await new Promise(r => setTimeout(r, ms)); + return ms; +} From c043cb6fcf07c94aa645d1d42a1be64451e6eca9 Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 23 Mar 2026 17:37:52 +0800 Subject: [PATCH 2/2] fix: disable human delay in CI environment to prevent E2E timeouts In CI environments (CI=true), resolveProfile() now defaults to the 'none' profile instead of 'moderate'. This prevents the 1-8s per- navigation delay from causing E2E test timeouts (30s limit). Users can override this by setting OPENCLI_DELAY_PROFILE explicitly. --- src/human-delay.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/human-delay.ts b/src/human-delay.ts index d077bdd8..d943a647 100644 --- a/src/human-delay.ts +++ b/src/human-delay.ts @@ -146,10 +146,14 @@ export class HumanDelay { * Priority: * 1. Explicit `profile` argument * 2. OPENCLI_DELAY_PROFILE env var - * 3. 'moderate' default + * 3. 'none' when CI=true (to avoid test timeouts) + * 4. 'moderate' default */ export function resolveProfile(profile?: string): DelayProfile { - const name = profile ?? process.env.OPENCLI_DELAY_PROFILE ?? 'moderate'; + // In CI environments, default to 'none' to avoid test timeouts. + // Explicit OPENCLI_DELAY_PROFILE or profile argument always takes precedence. + const ciDefault = process.env.CI ? 'none' : 'moderate'; + const name = profile ?? process.env.OPENCLI_DELAY_PROFILE ?? ciDefault; return PROFILES[name] ?? PROFILES.moderate; }