LLM4AD enforce the same timeout from any thread

benchmarks/LLM4AD/llm4ad_loader.py

@@ -229,6 +229,30 @@ def _apply_llm4ad_transforms(code: str, entry: str, cfg: dict) -> str:
     return code
 
 
+def _eval_with_thread_timeout(fn: Callable, timeout: float):
+    """Run ``fn()`` enforcing ``timeout`` seconds from ANY thread.
+
+    ``signal.alarm`` only works in the main thread. Trace trainers evaluate
+    rollouts in worker threads by default (``num_threads > 1``), where the alarm
+    is skipped — so a hot-looping candidate previously ran with NO timeout and
+    stalled training forever. This fallback converts that hang into the existing
+    ``TimeoutError`` -> -1e6 feedback path. The runaway worker is leaked as a
+    daemon thread by design: Python threads cannot be killed, and a leaked
+    daemon is strictly better than a stalled run.
+    """
+    from concurrent.futures import ThreadPoolExecutor
+    from concurrent.futures import TimeoutError as _FutureTimeout
+
+    executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="llm4ad-eval")
+    future = executor.submit(fn)
+    try:
+        return future.result(timeout=timeout)
+    except _FutureTimeout:
+        raise TimeoutError(f"Evaluation timed out after {timeout}s (thread-timeout fallback)")
+    finally:
+        executor.shutdown(wait=False, cancel_futures=True)
+
+
 class LLM4ADEvaluatorGuide(Guide):
     """Trace Guide that uses LLM4AD evaluators for feedback."""
 
@@ -286,7 +310,16 @@ def timeout_handler(signum, frame):
 
             # Use LLM4AD's evaluate_program method
             try:
-                score = self.evaluator_loader.evaluate_program(full_code, func, entry_name=self._entry)
+                if use_signal:
+                    score = self.evaluator_loader.evaluate_program(full_code, func, entry_name=self._entry)
+                else:
+                    # worker thread: signal.alarm unavailable -> enforce the same
+                    # timeout via the thread fallback (fixes silent infinite evals)
+                    score = _eval_with_thread_timeout(
+                        lambda: self.evaluator_loader.evaluate_program(
+                            full_code, func, entry_name=self._entry),
+                        timeout,
+                    )
                 if use_signal:
                     signal.alarm(0)
                 elapsed = time.time() - start

tests/test_llm4ad_thread_timeout.py

@@ -0,0 +1,40 @@
+"""Regression: LLM4AD evaluation must time out in worker threads too.
+
+signal.alarm-based timeouts are silently skipped outside the main thread, which
+is exactly where Trace trainers run rollouts (num_threads > 1). Before the
+thread-timeout fallback, a hot-looping candidate stalled training forever.
+"""
+import sys
+import threading
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "benchmarks" / "LLM4AD"))
+from llm4ad_loader import _eval_with_thread_timeout  # noqa: E402
+
+import pytest  # noqa: E402
+
+
+def test_returns_value_under_timeout():
+    assert _eval_with_thread_timeout(lambda: 42, timeout=2.0) == 42
+
+
+def test_raises_timeout_instead_of_hanging():
+    start = time.time()
+    with pytest.raises(TimeoutError):
+        _eval_with_thread_timeout(lambda: time.sleep(30), timeout=0.5)
+    assert time.time() - start < 5.0  # returned promptly, no stall
+
+
+def test_enforced_from_worker_thread():
+    result = {}
+
+    def worker():
+        try:
+            _eval_with_thread_timeout(lambda: time.sleep(30), timeout=0.5)
+        except TimeoutError:
+            result["timed_out"] = True
+
+    t = threading.Thread(target=worker)
+    t.start(); t.join(timeout=10)
+    assert result.get("timed_out") is True