Skip to content

Commit 686535c

Browse files
feat(llm): add 15-minute overall timeout to ArticleFactChecker
ArticleFactChecker had no wall-clock timeout, risking unbounded execution on articles with many claims. Add asyncio.wait_for-based overall timeout (default 900s) with input validation, range clamping [30s, 7200s], and graceful error reporting on both normal and Jupyter fallback paths. Also add missing arxiv>=2.4.0 dependency to requirements/agent.txt. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent a43db21 commit 686535c

2 files changed

Lines changed: 94 additions & 15 deletions

File tree

dingo/model/llm/agent/agent_article_fact_checker.py

Lines changed: 81 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ class ArticleFactChecker(BaseAgent):
346346
"parameters": {
347347
"agent_config": {
348348
"max_iterations": 10,
349+
"overall_timeout": 900,
350+
"max_concurrent_claims": 5,
349351
"tools": {
350352
"claims_extractor": {
351353
"api_key": "your-openai-api-key",
@@ -372,6 +374,9 @@ class ArticleFactChecker(BaseAgent):
372374
]
373375
max_iterations = 10 # Allow more iterations for comprehensive checking
374376
max_concurrent_claims = 5 # Default parallel claim verification slots
377+
overall_timeout = 900 # 15-minute wall-clock timeout for entire evaluation
378+
_MIN_OVERALL_TIMEOUT = 30 # Floor: 30 seconds
379+
_MAX_OVERALL_TIMEOUT = 7200 # Ceiling: 2 hours
375380

376381
_required_fields = [RequiredField.CONTENT] # Article text
377382

@@ -823,17 +828,36 @@ def eval(cls, input_data: Data) -> EvalDetail:
823828
if output_dir and input_data.content:
824829
cls._save_article_content(output_dir, input_data.content)
825830

831+
timeout = cls._get_overall_timeout()
832+
833+
async def _run_with_timeout() -> EvalDetail:
834+
return await asyncio.wait_for(
835+
cls._async_eval(input_data, start_time, output_dir),
836+
timeout=timeout,
837+
)
838+
826839
try:
827-
return asyncio.run(cls._async_eval(input_data, start_time, output_dir))
840+
return asyncio.run(_run_with_timeout())
841+
except asyncio.TimeoutError:
842+
elapsed = time.time() - start_time
843+
log.warning(f"ArticleFactChecker: overall timeout exceeded ({elapsed:.1f}s / {timeout:.0f}s limit)")
844+
return cls._create_overall_timeout_result(elapsed, timeout)
828845
except RuntimeError as e:
829846
# Fallback when called inside an already-running event loop (e.g. Jupyter, tests)
830847
if "cannot run" in str(e).lower() or "already running" in str(e).lower():
831848
import concurrent.futures
832849
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
833-
future = pool.submit(
834-
lambda: asyncio.run(cls._async_eval(input_data, start_time, output_dir))
835-
)
836-
return future.result()
850+
future = pool.submit(lambda: asyncio.run(_run_with_timeout()))
851+
try:
852+
# Extra margin so asyncio.wait_for fires before this outer timeout
853+
return future.result(timeout=timeout + 30)
854+
except (asyncio.TimeoutError, concurrent.futures.TimeoutError):
855+
elapsed = time.time() - start_time
856+
log.warning(
857+
f"ArticleFactChecker: overall timeout exceeded "
858+
f"({elapsed:.1f}s / {timeout:.0f}s limit, fallback path)"
859+
)
860+
return cls._create_overall_timeout_result(elapsed, timeout)
837861
raise
838862

839863
# --- Two-Phase Async Architecture Methods ---
@@ -1023,6 +1047,26 @@ def _get_max_concurrent_claims(cls) -> int:
10231047
agent_cfg = params.get('agent_config') or {}
10241048
return agent_cfg.get('max_concurrent_claims', cls.max_concurrent_claims)
10251049

1050+
@classmethod
1051+
def _get_overall_timeout(cls) -> float:
1052+
"""Read overall_timeout from agent_config or use class default (900s).
1053+
1054+
Returns:
1055+
Positive timeout in seconds, clamped to [30, 7200].
1056+
"""
1057+
params = cls.dynamic_config.parameters or {}
1058+
agent_cfg = params.get('agent_config') or {}
1059+
raw = agent_cfg.get('overall_timeout', cls.overall_timeout)
1060+
try:
1061+
timeout = float(raw)
1062+
except (TypeError, ValueError):
1063+
log.warning(f"Invalid overall_timeout={raw!r}, using default {cls.overall_timeout}s")
1064+
return float(cls.overall_timeout)
1065+
clamped = max(cls._MIN_OVERALL_TIMEOUT, min(timeout, cls._MAX_OVERALL_TIMEOUT))
1066+
if clamped != timeout:
1067+
log.warning(f"overall_timeout={timeout} out of range, clamped to {clamped}s")
1068+
return float(clamped)
1069+
10261070
@classmethod
10271071
def _parse_claim_json_robust(cls, output: Optional[str]) -> Dict[str, Any]:
10281072
"""
@@ -1795,6 +1839,38 @@ def _create_error_result(cls, error_message: str) -> EvalDetail:
17951839
]
17961840
return result
17971841

1842+
@classmethod
1843+
def _create_overall_timeout_result(cls, elapsed: float, timeout: float) -> EvalDetail:
1844+
"""
1845+
Create error result when overall wall-clock timeout is exceeded.
1846+
1847+
Args:
1848+
elapsed: Actual elapsed time in seconds
1849+
timeout: Configured timeout limit in seconds
1850+
1851+
Returns:
1852+
EvalDetail with timeout error status
1853+
"""
1854+
minutes, seconds = divmod(int(timeout), 60)
1855+
limit_str = f"{minutes}m{seconds}s" if minutes else f"{int(timeout)}s"
1856+
result = EvalDetail(metric=cls.__name__)
1857+
result.status = True
1858+
result.label = [f"{QualityLabel.QUALITY_BAD_PREFIX}AGENT_OVERALL_TIMEOUT"]
1859+
result.reason = [
1860+
"Article Fact-Checking Failed: Overall Timeout Exceeded",
1861+
"=" * 70,
1862+
f"Execution exceeded the {int(timeout)}s ({limit_str}) wall-clock limit.",
1863+
f"Elapsed time: {elapsed:.1f}s",
1864+
"",
1865+
"Recommendations:",
1866+
f" 1. Increase overall_timeout (current: {int(timeout)}s) in agent_config",
1867+
" 2. Reduce max_claims in claims_extractor config (e.g., 50 -> 20)",
1868+
" 3. Use a faster model (e.g., gpt-4o-mini instead of gpt-4o)",
1869+
" 4. Reduce max_concurrent_claims to lower API rate-limit pressure",
1870+
" 5. Split long articles into shorter sections",
1871+
]
1872+
return result
1873+
17981874
@classmethod
17991875
def plan_execution(cls, input_data: Data) -> List[Dict[str, Any]]:
18001876
"""

requirements/agent.txt

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1-
# Agent-specific dependencies (optional)
2-
# Install with: pip install -r requirements/agent.txt
3-
# Or: pip install dingo-python[agent]
4-
5-
# LangChain 1.0 for agent-based evaluation
6-
langchain>=1.0.0
7-
langchain-openai>=1.0.0
8-
9-
# Tavily for web search tool
10-
tavily-python>=0.3.0
1+
# Agent-specific dependencies (optional)
2+
# Install with: pip install -r requirements/agent.txt
3+
# Or: pip install dingo-python[agent]
4+
5+
# LangChain 1.0 for agent-based evaluation
6+
langchain>=1.0.0
7+
langchain-openai>=1.0.0
8+
9+
# Tavily for web search tool
10+
tavily-python>=0.3.0
11+
12+
# ArXiv for academic paper search tool
13+
arxiv>=2.4.0

0 commit comments

Comments
 (0)