From b43d5541e15e6d408a39203565e8d2e8a3535e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rabia=20Yazl=C4=B1?= Date: Wed, 3 Jun 2026 11:52:11 +0300 Subject: [PATCH 1/2] feat: rate-limiting & graceful degradation when an API key is missing (#61) Introduce a standard module result status enum (ok | skipped | rate_limited | error) so key-dependent modules degrade gracefully instead of failing hard. - Add modules/module_status.py with classify()/reason_for()/annotate() helpers - Shodan, VirusTotal, AbuseIPDB, Censys, Leak-Lookup/HIBP and Telegram now report 'skipped' when an API key is absent and 'rate_limited' on HTTP 429 - Scan engine propagates the status over WebSocket, persists it for the results view, and only caches successful (ok) results - Frontend renders a per-module status badge on progress chips and result cards, with the reason for skipped/rate-limited modules - Add tests for the status enum and per-module skip behaviour; update existing tests to the new (non-error) contract Closes #61 --- CHANGELOG.md | 12 + frontend/src/components/App.tsx | 30 ++- .../src/components/views/ScanProgress.tsx | 28 ++- frontend/src/components/views/ScanResults.tsx | 227 +++++++++++------- frontend/src/lib/types.ts | 23 +- modules/censys_lookup.py | 36 +-- modules/leak_lookup.py | 43 +++- modules/module_status.py | 124 ++++++++++ modules/shodan_lookup.py | 24 +- modules/telegram_lookup.py | 15 +- modules/threat_intel.py | 38 ++- tests/test_module_status.py | 138 +++++++++++ tests/test_modules_extended.py | 24 +- tests/test_v2_1_modules.py | 8 +- web/app.py | 31 ++- 15 files changed, 641 insertions(+), 160 deletions(-) create mode 100644 modules/module_status.py create mode 100644 tests/test_module_status.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dd87df3..44df6fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/). --- +## [Unreleased] + +### Added +- **Standard module status enum** (`ok` / `skipped` / `rate_limited` / `error`) in `modules/module_status.py`, with `classify()`, `reason_for()`, and `annotate()` helpers (#61). +- **Graceful degradation for key-dependent modules** — Shodan, VirusTotal, AbuseIPDB, Censys, Leak-Lookup/HIBP, and Telegram now report `skipped` when an API key is absent and `rate_limited` on HTTP 429, instead of failing with a hard error (#61). +- **Per-module status badges in the dashboard** — live progress chips and result cards now render the module status (skipped/rate-limited shown with their reason), and the scan engine propagates the status over WebSocket (#61). + +### Changed +- The scan engine (`web/app.py`) derives each module's status from its result, persists it for the results view, and only caches genuinely successful (`ok`) results so a missing key is not frozen in cache once configured (#61). + +--- + ## [2.3.0] — 2026-06-03 ### Added diff --git a/frontend/src/components/App.tsx b/frontend/src/components/App.tsx index 965c83a..ebbd528 100644 --- a/frontend/src/components/App.tsx +++ b/frontend/src/components/App.tsx @@ -9,10 +9,21 @@ import { ResultsSkeleton } from './ResultsSkeleton'; import { ToolPanels } from './tools/ToolPanels'; import { ScanComparison } from './views/ScanComparison'; import { startScan, getWsUrl, getScan } from '@/lib/api'; -import type { ScanType, ScanStatus, ToolMode, ScanResults as ScanResultsType, ScanMeta } from '@/lib/types'; +import type { ScanType, ScanStatus, ToolMode, ScanResults as ScanResultsType, ScanMeta, LiveModuleStatus } from '@/lib/types'; type View = 'idle' | 'tool' | 'scanning' | 'results' | 'compare'; +/** Format a `module_done` event into a progress-log line per status. */ +function moduleDoneLine(msg: { module: string; status?: string; reason?: string; error?: string }): string { + const detail = msg.reason || msg.error; + switch (msg.status) { + case 'skipped': return `⊘ ${msg.module}: skipped${detail ? ` — ${detail}` : ''}`; + case 'rate_limited': return `⏳ ${msg.module}: rate limited${detail ? ` — ${detail}` : ''}`; + case 'error': return `✗ ${msg.module}: ${detail || 'error'}`; + default: return `✓ ${msg.module}`; + } +} + export function App() { const [view, setView] = useState('idle'); const [sidebarOpen, setSidebarOpen] = useState(false); @@ -22,7 +33,7 @@ export function App() { const [scanMeta, setScanMeta] = useState<(ScanMeta & { results: ScanResultsType }) | null>(null); const [progressLog, setProgressLog] = useState([]); const [scanTarget, setScanTarget] = useState(''); - const [moduleStatuses, setModuleStatuses] = useState>({}); + const [moduleStatuses, setModuleStatuses] = useState>({}); const [totalModules, setTotalModules] = useState(0); const [compareIds, setCompareIds] = useState<[string, string] | null>(null); const wsRef = useRef(null); @@ -89,10 +100,7 @@ export function App() { const lines = [...prev]; for (const msg of newMsgs) { if (msg.type === 'module_start') lines.push(`→ ${msg.module}`); - else if (msg.type === 'module_done') { - if (msg.status === 'error') lines.push(`✗ ${msg.module}: ${msg.error || 'error'}`); - else lines.push(`✓ ${msg.module}`); - } + else if (msg.type === 'module_done') lines.push(moduleDoneLine(msg)); } return lines; }); @@ -100,7 +108,7 @@ export function App() { const next = { ...prev }; for (const msg of newMsgs) { if (msg.type === 'module_start') next[msg.module] = 'running'; - else if (msg.type === 'module_done') next[msg.module] = msg.status === 'error' ? 'error' : 'ok'; + else if (msg.type === 'module_done') next[msg.module] = (msg.status as LiveModuleStatus) || 'ok'; } return next; }); @@ -145,12 +153,8 @@ export function App() { setModuleStatuses(prev => ({ ...prev, [msg.module]: 'running' })); } else if (msg.type === 'module_done') { - setModuleStatuses(prev => ({ ...prev, [msg.module]: msg.status === 'error' ? 'error' : 'ok' })); - if (msg.status === 'error') { - setProgressLog(prev => [...prev, `✗ ${msg.module}: ${msg.error || 'error'}`]); - } else { - setProgressLog(prev => [...prev, `✓ ${msg.module}`]); - } + setModuleStatuses(prev => ({ ...prev, [msg.module]: (msg.status as LiveModuleStatus) || 'ok' })); + setProgressLog(prev => [...prev, moduleDoneLine(msg)]); } else if (msg.type === '_done') { done = true; diff --git a/frontend/src/components/views/ScanProgress.tsx b/frontend/src/components/views/ScanProgress.tsx index 5ce1d2f..184f071 100644 --- a/frontend/src/components/views/ScanProgress.tsx +++ b/frontend/src/components/views/ScanProgress.tsx @@ -1,18 +1,27 @@ 'use client'; -import { Terminal, Check, X, Loader2, Circle } from 'lucide-react'; +import { Terminal, Check, X, Loader2, Circle, Ban, Hourglass } from 'lucide-react'; import { useTranslations } from '@/lib/i18n'; +import type { LiveModuleStatus } from '@/lib/types'; interface Props { log: string[]; target: string; - moduleStatuses?: Record; + moduleStatuses?: Record; totalModules?: number; } +const STATUS_STYLE: Record = { + ok: { cls: 'text-green border-green/30 bg-green/5', Icon: Check }, + error: { cls: 'text-red border-red/30 bg-red/5', Icon: X }, + skipped: { cls: 'text-text-3 border-border-1 bg-surface-2', Icon: Ban }, + rate_limited: { cls: 'text-yellow border-yellow/30 bg-yellow/5', Icon: Hourglass }, + running: { cls: 'text-blue border-blue/30 bg-blue/5', Icon: Loader2, spin: true }, +}; + export function ScanProgress({ log, target, moduleStatuses = {}, totalModules = 0 }: Props) { const { t } = useTranslations(); - const entries = Object.entries(moduleStatuses); - const completed = entries.filter(([, s]) => s === 'ok' || s === 'error').length; + const entries = Object.entries(moduleStatuses) as [string, LiveModuleStatus][]; + const completed = entries.filter(([, s]) => s !== 'running').length; const cap = Math.max(totalModules, entries.length); const percent = cap > 0 ? Math.min(100, Math.round((completed / cap) * 100)) : 0; @@ -40,13 +49,12 @@ export function ScanProgress({ log, target, moduleStatuses = {}, totalModules = {entries.length > 0 && (
{entries.map(([mod, status]) => { - const cls = status === 'ok' ? 'text-green border-green/30 bg-green/5' - : status === 'error' ? 'text-red border-red/30 bg-red/5' - : 'text-blue border-blue/30 bg-blue/5'; - const Icon = status === 'ok' ? Check : status === 'error' ? X : status === 'running' ? Loader2 : Circle; + const style = STATUS_STYLE[status] ?? { cls: 'text-blue border-blue/30 bg-blue/5', Icon: Circle }; + const Icon = style.Icon; + const label = status === 'rate_limited' ? `${mod} (rate limited)` : status === 'skipped' ? `${mod} (skipped)` : mod; return ( - - + + {mod} ); diff --git a/frontend/src/components/views/ScanResults.tsx b/frontend/src/components/views/ScanResults.tsx index efed08d..4f7bedd 100644 --- a/frontend/src/components/views/ScanResults.tsx +++ b/frontend/src/components/views/ScanResults.tsx @@ -224,15 +224,84 @@ function DtRow({ label, value }: { label: string; value?: string | number | null ); } -function Card({ title, children }: { title?: string; children: React.ReactNode }) { +function Card({ title, extra, children }: { title?: string; extra?: React.ReactNode; children: React.ReactNode }) { return (
- {title &&
{title}
} + {title && ( +
+ {title} + {extra} +
+ )}
{children}
); } +type ModStatus = 'ok' | 'skipped' | 'rate_limited' | 'error'; + +/** Derive the standard status from a module result (honours explicit status). */ +function modStatus(m?: { status?: string; error?: string | null } | null): ModStatus { + if (!m) return 'ok'; + if (m.status === 'skipped' || m.status === 'rate_limited' || m.status === 'error') return m.status; + if (m.error) return 'error'; + return 'ok'; +} + +const STATUS_BADGE: Record, { label: string; color: string; hint: string }> = { + skipped: { label: 'SKIPPED', color: '#8b949e', hint: 'No API key configured' }, + rate_limited: { label: 'RATE LIMITED', color: '#d29922', hint: 'Provider rate limit reached' }, + error: { label: 'ERROR', color: '#f85149', hint: 'Module failed' }, +}; + +function ModuleStatusBadge({ status, label }: { status: ModStatus; label?: string }) { + if (status === 'ok') return null; + const b = STATUS_BADGE[status]; + return ( + + {label ?? b.label} + + ); +} + +function ModuleNotice({ status, reason }: { status: 'skipped' | 'rate_limited'; reason?: string }) { + const b = STATUS_BADGE[status]; + return ( +
+ {reason || b.hint} + {status === 'skipped' && ( + — add the key to .env to enable this module. + )} +
+ ); +} + +/** + * Render a card for a key-dependent module: the data when it ran OK, a clear + * "skipped"/"rate limited" notice when it degraded gracefully, and nothing on + * a hard error (kept hidden as before). + */ +function KeyModuleCard({ title, mod, children }: { + title: string; + mod?: { status?: string; error?: string | null; status_reason?: string } | null; + children: React.ReactNode; +}) { + if (!mod) return null; + const st = modStatus(mod); + if (st === 'ok') return {children}; + if (st === 'skipped' || st === 'rate_limited') { + return ( + }> + + + ); + } + return null; +} + function CopyIconButton({ onClick, label }: { onClick: () => void; label: string }) { return (