From 1c24875008a9d07476de561be7c97e195b2b131d Mon Sep 17 00:00:00 2001 From: doxav <> Date: Sat, 13 Jun 2026 13:33:11 +0200 Subject: [PATCH 1/2] Enhance existing benchmarks for recursive optimization - Introduced `TraceCodeAgent` for code synthesis tasks, integrating unit test scoring. - Added `code_exec.py` for handling code generation tasks with execution feedback. - Implemented `strategy_qa.py` for implicit multi-hop yes/no reasoning tasks. - Updated `hf_qa_loader.py` to include new task modules and improve guide instantiation. - Enhanced YAML configuration with new diagnostic families for recursive optimization. - Created tests for new task functionalities and ensured compatibility with existing frameworks. --- benchmarks/hf_qa/agents/dspy_agent.py | 5 + benchmarks/hf_qa/agents/trace_agent.py | 60 ++++++ benchmarks/hf_qa/bbeh.py | 4 +- benchmarks/hf_qa/code_exec.py | 175 ++++++++++++++++ benchmarks/hf_qa/hf_qa_loader.py | 31 ++- benchmarks/hf_qa/hf_tasks.yaml | 123 ++++++++++++ benchmarks/hf_qa/strategy_qa.py | 89 +++++++++ tests/m1/test_recursive_diagnostic_tasks.py | 210 ++++++++++++++++++++ tests/test_lite_optimize_llm4ad.py | 69 +++++-- 9 files changed, 742 insertions(+), 24 deletions(-) create mode 100644 benchmarks/hf_qa/code_exec.py create mode 100644 benchmarks/hf_qa/strategy_qa.py create mode 100644 tests/m1/test_recursive_diagnostic_tasks.py diff --git a/benchmarks/hf_qa/agents/dspy_agent.py b/benchmarks/hf_qa/agents/dspy_agent.py index 0a2b3f5..9aff6c8 100644 --- a/benchmarks/hf_qa/agents/dspy_agent.py +++ b/benchmarks/hf_qa/agents/dspy_agent.py @@ -167,4 +167,9 @@ def make_agent(agent_class: str, **kwargs: Any) -> Any: ) if agent_class == "bbeh": return DSPyBBEHAgent() + if agent_class == "code": + raise NotImplementedError( + "Code-synthesis tasks (agent_class: code) currently have a Trace " + "agent only; run them with framework: trace (the default)." + ) return DSPyHotpotQAAgent(**kwargs) diff --git a/benchmarks/hf_qa/agents/trace_agent.py b/benchmarks/hf_qa/agents/trace_agent.py index 5f47ed1..69cf515 100644 --- a/benchmarks/hf_qa/agents/trace_agent.py +++ b/benchmarks/hf_qa/agents/trace_agent.py @@ -133,6 +133,7 @@ def __init__(self) -> None: self.llm = LLM() self.prompt_template = ParameterNode( _PROMPT_TEMPLATE, + name="prompt_template", trainable=True, description=_PROMPT_DESCRIPTION, ) @@ -169,6 +170,63 @@ class TraceBBEHAgent: # type: ignore[no-redef] pass +# --------------------------------------------------------------------------- +# TraceCodeAgent — code-generation agent for unit-test-scored tasks +# --------------------------------------------------------------------------- + +_CODE_PROMPT = dedent("""\ +You are an expert Python programmer. Implement the requested function. +Return ONLY the complete function definition (you may use a ```python block). +Do not include explanations, examples, or test code. + +""") + +_CODE_PROMPT_DESCRIPTION = ( + "[ParameterNode] Code-generation instructions prepended to each problem " + "spec. Should steer the model to emit a single complete, correct Python " + "function definition (no prose, no tests) that satisfies the spec and its " + "hidden unit tests." +) + +if _OPTO_AVAILABLE: + class TraceCodeAgent(Module): + """Agent for code-synthesis tasks scored by unit-test execution. + + Differs from the BBEH agent in two ways that matter for code: + - it traces a code-generation ``prompt_template`` (the optimizer rewrites + the instructions that produce the code), and + - it passes the raw LLM response THROUGH unchanged — the code itself is + the answer, so there is no ``extract_answer`` step to mangle it. The + code_exec guide extracts the function body and runs the tests. + """ + + llm: Any # replaced with DummyLLM in mode: stub + + def __init__(self) -> None: + super().__init__() + self.llm = LLM() + self.prompt_template = ParameterNode( + _CODE_PROMPT, + name="prompt_template", + trainable=True, + description=_CODE_PROMPT_DESCRIPTION, + ) + + def forward(self, question: str) -> Any: + # call_llm keeps the call (and the trainable prompt) on the trace + # graph so the guide's feedback is attributed to prompt_template. + return trace_ops.call_llm(self.llm, self.prompt_template, question) + + @classmethod + def to_examples(cls, inputs: Any, infos: Any): + """No-op: Trace agents consume raw inputs/infos as-is.""" + return inputs, infos + +else: + class TraceCodeAgent: # type: ignore[no-redef] + pass + + # --------------------------------------------------------------------------- # Factory # --------------------------------------------------------------------------- @@ -191,4 +249,6 @@ def make_agent(agent_class: str, **kwargs: Any) -> Any: ) if agent_class == "bbeh": return TraceBBEHAgent() + if agent_class == "code": + return TraceCodeAgent() return TraceHotpotQAAgent(**kwargs) diff --git a/benchmarks/hf_qa/bbeh.py b/benchmarks/hf_qa/bbeh.py index 2329dcf..1173937 100644 --- a/benchmarks/hf_qa/bbeh.py +++ b/benchmarks/hf_qa/bbeh.py @@ -78,8 +78,10 @@ def get_feedback( response_str = response_text(response) if eval_metric(info, response_str): return 1.0, "The answer is correct." + task_excerpt = " ".join(str(task).split())[:240] return 0.0, ( - f'The answer is wrong. Expected: "{info}", Got: "{response_str}". ' + f'The answer is wrong for task: "{task_excerpt}". ' + f'Expected: "{info}", Got: "{response_str}". ' "Improve the agent to produce the correct answer in the expected format: " "(A)/(B) for multiple-choice, or an exact string/number for free-form questions." ) diff --git a/benchmarks/hf_qa/code_exec.py b/benchmarks/hf_qa/code_exec.py new file mode 100644 index 0000000..914fd26 --- /dev/null +++ b/benchmarks/hf_qa/code_exec.py @@ -0,0 +1,175 @@ +"""Code-generation tasks scored by executing unit tests (HumanEval / MBPP). + +This is the one genuinely-new task semantics in this batch: the response is a +Python function body, and the guide SCORES IT BY RUNNING the dataset's own unit +tests in an isolated subprocess. That gives the recursive optimizer a +``CodeArtifactLevel`` surface with a real, non-LLM, deterministic pass/fail +signal (like the validator-style component in recursive_opt example B) — no +judge model required. + +Safety: candidate code is executed, so each evaluation runs in a separate +``python -c`` subprocess with a wall-clock timeout and is never ``exec``'d in the +host process. The timeout/inability-to-run cases score 0.0 with feedback rather +than raising, so a single bad candidate cannot abort a sweep. +""" +from __future__ import annotations + +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Tuple + +try: + from .common import HFQATask, response_text +except ImportError: + from common import HFQATask, response_text + +try: + from opto.trainer.guide import Guide +except Exception: + Guide = object # type: ignore[assignment,misc] + +# Default per-candidate execution budget (seconds). Overridable via cfg. +_DEFAULT_TIMEOUT = 10 + + +def _extract_code(response: Any) -> str: + """Pull a code body out of an LLM response, tolerating ``` fences.""" + text = response_text(response) + if "```" in text: + # take the content of the first fenced block + parts = text.split("```") + if len(parts) >= 2: + block = parts[1] + # drop an optional language tag on the first line + if "\n" in block: + first, rest = block.split("\n", 1) + if first.strip().lower() in {"python", "py", ""}: + return rest + return block + return text + + +def row_to_tasks(row: Dict[str, Any], cfg: Dict[str, Any]) -> List[HFQATask]: + """Convert one code row into a code task. + + Supports both schemas via cfg fields (defaults target HumanEval): + - prompt_field : the function signature + docstring (the question) + - test_field : unit-test source string + - entry_field : name of the function under test (for the test harness) + - setup_field : optional import/setup preamble (MBPP: 'test_setup_code') + """ + prompt_field = cfg.get("question_field", cfg.get("prompt_field", "prompt")) + test_field = cfg.get("test_field", "test") + entry_field = cfg.get("entry_field", "entry_point") + setup_field = cfg.get("setup_field") + if prompt_field not in row: + raise KeyError(f"Code row missing prompt field {prompt_field!r}") + if test_field not in row: + raise KeyError(f"Code row missing test field {test_field!r}") + + # Test source may be a single string (HumanEval) or a list of assert + # strings (MBPP 'test_list'); join lists into a runnable test block. + raw_test = row[test_field] + test_src = "\n".join(str(t) for t in raw_test) if isinstance(raw_test, (list, tuple)) else str(raw_test) + + meta = { + "test": test_src, + "entry_point": str(row.get(entry_field, "")), + "setup": str(row.get(setup_field, "")) if setup_field else "", + } + return [HFQATask(question=str(row[prompt_field]), context="", answer=meta["entry_point"], meta=meta)] + + +def format_context(raw: Any) -> str: + """Code tasks carry their spec in the prompt; no separate context.""" + return "" + + +def make_dataset(tasks: List[HFQATask]) -> Dict[str, List[Any]]: + """Question-only dataset shape.""" + return {"inputs": [task.question for task in tasks], "infos": list(tasks)} + + +def _run_unit_tests(candidate: str, info: HFQATask, timeout: int) -> Tuple[bool, str]: + """Execute candidate + dataset tests in an isolated subprocess. + + On failure, return a DIRECTIONAL diagnostic — the exception type, message, + and the specific failing assertion/line where available — because that text + becomes the optimizer's entire learning signal (``optimizer.backward(target, + feedback)``). A bare "tests failed" gives the optimizer nothing to act on. + """ + entry = info.meta.get("entry_point", "") + setup = info.meta.get("setup", "") + test_src = info.meta.get("test", "") + # HumanEval tests define check(candidate); MBPP tests are bare asserts. + harness = "\n".join([ + "import traceback", + setup, + candidate, + test_src, + # Call check(entry) when the test file defines it (HumanEval); MBPP's + # module-level asserts have already run on import of test_src above. + f"\nif 'check' in dir():\n check({entry})\n" if entry else "", + "print('PASS')", + ]) + # Wrap execution so we can surface the precise failing line, not just the + # last stderr line (which often loses the assertion text). + runner = ( + "import traceback, sys\n" + "try:\n" + + "\n".join(" " + line for line in harness.splitlines()) + + "\nexcept Exception as exc:\n" + " tb = traceback.extract_tb(sys.exc_info()[2])\n" + " frame = tb[-1] if tb else None\n" + " loc = f' at: {frame.line}' if frame and frame.line else ''\n" + " print(f'FAIL: {type(exc).__name__}: {exc}{loc}', file=sys.stderr)\n" + " sys.exit(1)\n" + ) + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "candidate_test.py" + script.write_text(runner) + try: + proc = subprocess.run( + [sys.executable, str(script)], + capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired: + return False, (f"timed out after {timeout}s — the function likely has an " + "infinite loop or is far too slow; ensure it terminates") + if proc.returncode == 0 and "PASS" in proc.stdout: + return True, "all unit tests passed" + # Prefer our structured FAIL line; fall back to raw stderr/stdout. + for line in (proc.stderr or "").splitlines(): + if line.startswith("FAIL:"): + return False, line[len("FAIL:"):].strip() + tail = (proc.stderr or proc.stdout or "no output").strip().splitlines() + return False, (tail[-1][:200] if tail else "unit tests failed") + + +class CodeExecGuide(Guide): # type: ignore[misc] + """Guide that scores generated code by unit-test pass/fail (1.0 / 0.0).""" + + def __init__(self, timeout: int = _DEFAULT_TIMEOUT) -> None: + self._timeout = int(timeout) + + def get_feedback( + self, task: Any, response: Any, info: HFQATask, **kwargs: Any + ) -> Tuple[float, str]: + """Run the dataset's tests against the extracted candidate code.""" + candidate = _extract_code(response) + if not candidate.strip(): + return 0.0, "Empty code response; produce a complete function definition." + passed, detail = _run_unit_tests(candidate, info, self._timeout) + if passed: + return 1.0, "The implementation passes all unit tests." + return 0.0, ( + f"The implementation fails: {detail}. Revise the function so this " + "specific failure is resolved while keeping all other cases correct." + ) + + +def make_guide(timeout: int = _DEFAULT_TIMEOUT) -> Any: + """Return an instantiated code-execution guide.""" + return CodeExecGuide(timeout=timeout) diff --git a/benchmarks/hf_qa/hf_qa_loader.py b/benchmarks/hf_qa/hf_qa_loader.py index 7c3f06c..6427c7d 100644 --- a/benchmarks/hf_qa/hf_qa_loader.py +++ b/benchmarks/hf_qa/hf_qa_loader.py @@ -52,21 +52,25 @@ try: from . import bbeh as _bbeh_mod + from . import code_exec as _code_exec_mod from . import drop_qa as _drop_qa_mod from . import gsm8k as _gsm8k_mod from . import hotpot_qa as _hotpot_qa_mod from . import mcqa as _mcqa_mod from . import musique as _musique_mod from . import qasper as _qasper_mod + from . import strategy_qa as _strategy_qa_mod from .common import HFQATask except ImportError: import bbeh as _bbeh_mod + import code_exec as _code_exec_mod import drop_qa as _drop_qa_mod import gsm8k as _gsm8k_mod import hotpot_qa as _hotpot_qa_mod import mcqa as _mcqa_mod import musique as _musique_mod import qasper as _qasper_mod + import strategy_qa as _strategy_qa_mod from common import HFQATask # Registry mapping task_type (from hf_tasks.yaml) → task module. @@ -80,6 +84,8 @@ "mcqa": _mcqa_mod, "musique": _musique_mod, "qasper": _qasper_mod, + "strategy_qa": _strategy_qa_mod, + "code_exec": _code_exec_mod, } # Registry mapping framework name → agent module (lazily populated). @@ -95,11 +101,17 @@ def _get_framework_module(framework: str) -> Any: """Return (and cache) the agent module for *framework*.""" if framework not in _FRAMEWORK_MODULES: if framework == "trace": - from agents import trace_agent + try: + from .agents import trace_agent + except ImportError: + from agents import trace_agent _FRAMEWORK_MODULES["trace"] = trace_agent elif framework == "dspy": - from agents import dspy_agent + try: + from .agents import dspy_agent + except ImportError: + from agents import dspy_agent _FRAMEWORK_MODULES["dspy"] = dspy_agent else: @@ -317,8 +329,19 @@ def _make_dataset_from_cfg( def _make_guide(cfg: Dict[str, Any]) -> Any: - """Return an instantiated guide for the configured task semantics.""" - return _get_task_module(_task_type_key(cfg)).make_guide() + """Return an instantiated guide for the configured task semantics. + + Most task modules expose ``make_guide()``; modules that accept a ``timeout`` + kwarg (e.g. code_exec) receive it from the task config. Introspection keeps + a single call site without forcing every module to take the same signature. + """ + import inspect + + make_guide = _get_task_module(_task_type_key(cfg)).make_guide + params = inspect.signature(make_guide).parameters + if "timeout" in params and "timeout" in cfg: + return make_guide(timeout=cfg["timeout"]) + return make_guide() def _make_agent(agent_class: str, framework: str, **kwargs: Any) -> Any: diff --git a/benchmarks/hf_qa/hf_tasks.yaml b/benchmarks/hf_qa/hf_tasks.yaml index e535830..2e7af15 100644 --- a/benchmarks/hf_qa/hf_tasks.yaml +++ b/benchmarks/hf_qa/hf_tasks.yaml @@ -284,3 +284,126 @@ description: "Qasper scholarly long-document QA" objective: | Optimize instructions for information-seeking QA over full research papers. + +# =========================================================================== +# DIAGNOSTIC FAMILIES for recursive_opt (added alongside the QA suite) +# +# These exist to give recursive_opt's causal-effect knobs a measurable surface: +# - bbeh_horizon : long reasoning chains -> exercises credit_horizon +# - aqua_rat : algebraic 5-way MCQ -> starting_artifact headroom (reuses mcqa) +# - strategy_qa : implicit multi-hop Y/N -> starting_artifact headroom +# - humaneval : code + unit tests -> CodeArtifactLevel surface (code_exec) +# - mbpp : code + unit tests -> CodeArtifactLevel surface (code_exec) +# All are additive: they only append entries / new task_type modules, so existing +# task ids and other branches are unaffected. +# =========================================================================== + +# bbeh_horizon — the long-chain BBEH subtasks where step-level vs episode-level +# feedback (credit_horizon) plausibly changes update quality. Pure curation of +# existing bbeh semantics: no new code, just a focused subtask list. +- id: bbeh_horizon + dataset_id: hubert233/BigBenchExtraHard + question_field: input + answer_field: target + context_format: none + agent_class: bbeh + task_type: bbeh + description: "BBEH long-horizon reasoning subset (for credit_horizon diagnostics)" + objective: | + Optimize the prompt_template and extraction logic for multi-step reasoning + chains where intermediate steps must be tracked to reach the final answer. + + #Variables: + - prompt_template: Instructions and format template given to the LLM. + - extract_answer: Code that parses the LLM response into a final answer. + subtasks: + - multistep_arithmetic + - web_of_lies + - dyck_languages + - shuffled_objects + - temporal_sequence + - object_counting + - time_arithmetic + - zebra_puzzles + +# AQuA-RAT — algebraic word problems as 5-way MCQ. Reuses mcqa semantics +# entirely (options list + letter answer); YAML-only. +- id: aqua_rat + dataset_id: deepmind/aqua_rat + dataset_config: raw + train_split: train + test_split: test + num_train: 128 + num_validate: 128 + num_test: 128 + question_field: question + answer_field: correct + options_field: options + context_format: none + task_type: mcqa + agent_class: bbeh + description: "AQuA-RAT algebraic multiple-choice reasoning" + objective: | + Optimize the prompt template and extraction logic for algebraic word-problem + reasoning with multiple-choice answers. + +# StrategyQA — implicit multi-hop yes/no reasoning (boolean shaping module). +- id: strategy_qa + dataset_id: ChilleD/StrategyQA + train_split: train + test_split: test + num_train: 128 + num_validate: 128 + num_test: 128 + question_field: question + answer_field: answer + context_format: none + task_type: strategy_qa + agent_class: bbeh + description: "StrategyQA implicit multi-hop yes/no reasoning" + objective: | + Optimize the agent's instructions for implicit multi-step decomposition that + yields a correct final Yes/No answer. + +# HumanEval — function synthesis scored by executing the provided unit tests. +- id: humaneval + dataset_id: openai/openai_humaneval + split: test + test_split: test + num_train: 41 + num_validate: 41 + num_test: 82 + question_field: prompt + answer_field: entry_point + test_field: test + entry_field: entry_point + context_format: none + task_type: code_exec + agent_class: code + timeout: 10 + description: "HumanEval Python function synthesis (unit-test scored)" + objective: | + Optimize the prompt template and code-generation instructions so the produced + Python function passes all provided unit tests. + +# MBPP — basic Python problems; tests are bare asserts (+ optional setup code). +- id: mbpp + dataset_id: google-research-datasets/mbpp + dataset_config: sanitized + train_split: train + test_split: test + num_train: 120 + num_validate: 90 + num_test: 257 + question_field: prompt + answer_field: entry_point + test_field: test_list + setup_field: test_setup_code + context_format: none + task_type: code_exec + agent_class: code + timeout: 10 + description: "MBPP basic Python programming problems (unit-test scored)" + objective: | + Optimize the prompt template and code-generation instructions so the produced + Python function passes all provided unit tests. diff --git a/benchmarks/hf_qa/strategy_qa.py b/benchmarks/hf_qa/strategy_qa.py new file mode 100644 index 0000000..3ecd4bf --- /dev/null +++ b/benchmarks/hf_qa/strategy_qa.py @@ -0,0 +1,89 @@ +"""StrategyQA — implicit multi-hop yes/no reasoning. + +StrategyQA rows carry a boolean ``answer`` and no options field, so they need a +small shaping step (boolean -> Yes/No) rather than the MCQ path. Everything else +(prompt building, response extraction) reuses ``common`` helpers, keeping this +module DRY. This task gives the recursive optimizer a *reasoning-artifact +headroom* surface: the gold answer is one bit, but reaching it requires implicit +decomposition, so a better ``starting_artifact`` / prompt measurably helps. +""" +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +try: + from .common import HFQATask, normalize_text, response_text +except ImportError: + from common import HFQATask, normalize_text, response_text + +try: + from opto.trainer.guide import Guide +except Exception: + Guide = object # type: ignore[assignment,misc] + + +def _gold_bool(raw: Any) -> str: + """Normalize StrategyQA's answer (bool / 'yes' / 'true' / 1) to 'yes'|'no'.""" + if isinstance(raw, bool): + return "yes" if raw else "no" + text = normalize_text(str(raw)) + if text in {"yes", "true", "1"}: + return "yes" + if text in {"no", "false", "0"}: + return "no" + return text # leave unexpected values visible rather than guessing + + +def row_to_tasks(row: Dict[str, Any], cfg: Dict[str, Any]) -> List[HFQATask]: + """Convert one StrategyQA row into a yes/no QA task.""" + question_field = cfg.get("question_field", "question") + answer_field = cfg.get("answer_field", "answer") + if question_field not in row: + raise KeyError(f"StrategyQA row missing question field {question_field!r}") + if answer_field not in row: + raise KeyError(f"StrategyQA row missing answer field {answer_field!r}") + + gold = _gold_bool(row[answer_field]) + prompt = ( + f"Question: {row[question_field]}\n" + "Answer with exactly 'Yes' or 'No'.\nAnswer:" + ) + return [HFQATask(question=prompt, context="", answer=gold)] + + +def format_context(raw: Any) -> str: + """StrategyQA is question-only.""" + return "" + + +def make_dataset(tasks: List[HFQATask]) -> Dict[str, List[Any]]: + """Question-only dataset shape used by BBEH-style agents.""" + return {"inputs": [task.question for task in tasks], "infos": list(tasks)} + + +class StrategyQAGuide(Guide): # type: ignore[misc] + """Guide for exact yes/no StrategyQA scoring.""" + + def get_feedback( + self, task: Any, response: Any, info: HFQATask, **kwargs: Any + ) -> Tuple[float, str]: + """Score by the first yes/no token in the response.""" + text = normalize_text(response_text(response)) + # first explicit yes/no token wins (robust to trailing rationale) + prediction = "" + for token in text.replace(".", " ").split(): + if token in {"yes", "no"}: + prediction = token + break + if prediction and prediction == info.answer: + return 1.0, "The yes/no answer is correct." + return 0.0, ( + f"The yes/no answer is wrong. Expected '{info.answer}', got " + f"'{prediction or text[:40]}'. Decompose the implicit reasoning steps " + "and state the final Yes/No explicitly." + ) + + +def make_guide() -> Any: + """Return an instantiated StrategyQA guide.""" + return StrategyQAGuide() diff --git a/tests/m1/test_recursive_diagnostic_tasks.py b/tests/m1/test_recursive_diagnostic_tasks.py new file mode 100644 index 0000000..9dd8ee3 --- /dev/null +++ b/tests/m1/test_recursive_diagnostic_tasks.py @@ -0,0 +1,210 @@ +"""Offline tests for the recursive_opt diagnostic task families. + +All tests are row-level: they feed synthetic rows to ``row_to_tasks`` and +synthetic responses to the guides, so they run without any HuggingFace network +access (mirroring the existing tests/m1/test_hf_qa_ext.py convention). +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +import benchmarks.hf_qa.hf_qa_loader as loader +from benchmarks.hf_qa import bbeh, code_exec, strategy_qa + + +# --------------------------------------------------------------------------- # +# YAML wiring +# --------------------------------------------------------------------------- # +def test_new_yaml_entries_resolve_to_task_types() -> None: + """New task ids map to the expected task semantics.""" + expected = { + "bbeh_horizon": "bbeh", # reuses bbeh semantics (curated subtasks) + "aqua_rat": "mcqa", # reuses mcqa semantics + "strategy_qa": "strategy_qa", + "humaneval": "code_exec", + "mbpp": "code_exec", + } + for task_id, task_type in expected.items(): + assert loader._load_task_config(task_id)["task_type"] == task_type + + +def test_bbeh_horizon_expands_subtasks() -> None: + """bbeh_horizon carries the long-chain subtask list for credit_horizon.""" + cfg = loader._load_task_config("bbeh_horizon") + assert "web_of_lies" in cfg["subtasks"] + assert "multistep_arithmetic" in cfg["subtasks"] + + +def test_bbeh_failure_feedback_names_task_context() -> None: + """Long-horizon feedback should identify the concrete failed task.""" + guide = bbeh.make_guide() + score, feedback = guide.get_feedback( + "Question: What is 2 + 2?\nAnswer:", + "5", + "4", + ) + + assert score == 0.0 + assert "What is 2 + 2" in feedback + assert 'Expected: "4"' in feedback + + +# --------------------------------------------------------------------------- # +# StrategyQA boolean shaping +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("raw,expected", [ + (True, "yes"), (False, "no"), ("yes", "yes"), ("False", "no"), (1, "yes"), +]) +def test_strategy_qa_gold_normalization(raw, expected) -> None: + task = strategy_qa.row_to_tasks( + {"question": "Is water wet?", "answer": raw}, + {"question_field": "question", "answer_field": "answer"})[0] + assert task.answer == expected + assert "Yes" in task.question # the Yes/No instruction is injected + + +def test_strategy_qa_guide_scores_first_boolean_token() -> None: + guide = strategy_qa.make_guide() + task = strategy_qa.row_to_tasks( + {"question": "Q?", "answer": True}, + {"question_field": "question", "answer_field": "answer"})[0] + assert guide.get_feedback(task.question, "Yes, because ...", task)[0] == 1.0 + assert guide.get_feedback(task.question, "No.", task)[0] == 0.0 + + +def test_strategy_qa_missing_field_raises() -> None: + with pytest.raises(KeyError, match="question field"): + strategy_qa.row_to_tasks({"answer": True}, {}) + + +# --------------------------------------------------------------------------- # +# Code execution (HumanEval / MBPP semantics) +# --------------------------------------------------------------------------- # +def _humaneval_row(correct: bool) -> dict: + body = " return a + b\n" if correct else " return a - b\n" + return { + "prompt": "def add(a, b):\n '''add'''\n", + "entry_point": "add", + "test": "def check(candidate):\n assert candidate(2, 3) == 5\n", + "_body": body, + } + + +def test_code_exec_passes_correct_humaneval_solution() -> None: + row = _humaneval_row(correct=True) + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + guide = code_exec.make_guide(timeout=10) + response = "def add(a, b):\n return a + b\n" + score, feedback = guide.get_feedback(task.question, response, task) + assert score == 1.0 and "passes" in feedback + + +def test_code_exec_fails_wrong_solution_without_raising() -> None: + row = _humaneval_row(correct=True) + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + guide = code_exec.make_guide(timeout=10) + score, feedback = guide.get_feedback(task.question, "def add(a, b):\n return a - b\n", task) + assert score == 0.0 and "fails" in feedback + + +def test_code_exec_handles_fenced_code_blocks() -> None: + row = _humaneval_row(correct=True) + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + guide = code_exec.make_guide(timeout=10) + fenced = "Here is the solution:\n```python\ndef add(a, b):\n return a + b\n```" + assert guide.get_feedback(task.question, fenced, task)[0] == 1.0 + + +def test_code_exec_joins_mbpp_test_list() -> None: + """MBPP-style list-valued tests are joined into a runnable block.""" + row = {"prompt": "Write add.", "entry_point": "add", + "test_list": ["assert add(1, 2) == 3", "assert add(0, 0) == 0"]} + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test_list", "entry_field": "entry_point"})[0] + assert "assert add(1, 2) == 3" in task.meta["test"] + guide = code_exec.make_guide(timeout=10) + # MBPP asserts run at module import; no check() needed + assert guide.get_feedback(task.question, "def add(a, b):\n return a + b\n", task)[0] == 1.0 + + +def test_code_exec_times_out_safely() -> None: + row = {"prompt": "loop", "entry_point": "f", + "test": "def check(candidate):\n candidate()\n"} + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + guide = code_exec.make_guide(timeout=1) + score, feedback = guide.get_feedback(task.question, "def f():\n while True:\n pass\n", task) + assert score == 0.0 and "timed out" in feedback + + +def test_code_exec_empty_response_scores_zero() -> None: + row = _humaneval_row(correct=True) + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + assert code_exec.make_guide().get_feedback(task.question, "", task)[0] == 0.0 + + +def test_make_guide_passes_timeout_from_cfg() -> None: + """Loader introspection wires cfg['timeout'] into code_exec.make_guide.""" + guide = loader._make_guide({"task_type": "code_exec", "timeout": 3}) + assert guide._timeout == 3 + # modules without a timeout param are unaffected + assert loader._make_guide({"task_type": "gsm8k"}) is not None + + +# --------------------------------------------------------------------------- # +# Guide efficiency: directional feedback (the optimizer's learning signal) +# --------------------------------------------------------------------------- # +def test_code_exec_feedback_names_the_failing_assertion() -> None: + """A wrong solution must yield a DIRECTIONAL diagnostic, not 'tests failed'. + + The feedback string is what optimizer.backward() consumes, so it must point + at the specific failure (exception + failing line) for efficient repair. + """ + row = {"prompt": "def add(a,b):\n '''add'''\n", "entry_point": "add", + "test": "def check(candidate):\n assert candidate(2, 3) == 5, 'add must sum'\n"} + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + guide = code_exec.make_guide(timeout=10) + score, feedback = guide.get_feedback(task.question, "def add(a, b):\n return a - b\n", task) + assert score == 0.0 + # names the exception type and surfaces the assertion (not just "failed") + assert "AssertionError" in feedback + assert "add must sum" in feedback or "assert candidate(2, 3) == 5" in feedback + + +def test_code_exec_timeout_feedback_is_actionable() -> None: + row = {"prompt": "loop", "entry_point": "f", + "test": "def check(candidate):\n candidate()\n"} + task = code_exec.row_to_tasks(row, {"question_field": "prompt", + "test_field": "test", "entry_field": "entry_point"})[0] + score, feedback = code_exec.make_guide(timeout=1).get_feedback( + task.question, "def f():\n while True:\n pass\n", task) + assert score == 0.0 and "terminate" in feedback + + +# --------------------------------------------------------------------------- # +# Trace surface: code tasks bind to the code agent (traces prompt, passes code) +# --------------------------------------------------------------------------- # +def test_code_tasks_use_code_agent_class() -> None: + """humaneval/mbpp must use agent_class: code (not bbeh) so the response + code is passed through to the unit-test guide unmangled.""" + for tid in ("humaneval", "mbpp"): + assert loader._load_task_config(tid)["agent_class"] == "code" + + +def test_code_agent_traces_prompt_and_passes_response_through() -> None: + pytest.importorskip("opto", reason="OpenTrace required for agent wiring") + from benchmarks.hf_qa.agents import trace_agent + + agent = trace_agent.make_agent("code") + # trainable code-generation prompt is exposed for the optimizer + assert agent.prompt_template.trainable + # there is no extract_answer step that would split on "Answer:" and mangle code + assert not hasattr(agent, "extract_answer") diff --git a/tests/test_lite_optimize_llm4ad.py b/tests/test_lite_optimize_llm4ad.py index 03994ca..0532edb 100644 --- a/tests/test_lite_optimize_llm4ad.py +++ b/tests/test_lite_optimize_llm4ad.py @@ -5,6 +5,8 @@ import importlib import importlib.util from pathlib import Path +from types import ModuleType +from typing import Any REPO_ROOT = Path(__file__).resolve().parents[1] @@ -41,25 +43,30 @@ ] -def _import_llm4ad_loader(): +def _import_llm4ad_loader() -> ModuleType: """Import llm4ad_loader with fallbacks for different repo layouts.""" - try: - return importlib.import_module("llm4ad_loader") - except ModuleNotFoundError: - pass - - try: - module = importlib.import_module("LLM4AD.llm4ad_loader") - sys.modules.setdefault("llm4ad_loader", module) - return module - except ModuleNotFoundError: - pass + for module_name in ( + "llm4ad_loader", + "LLM4AD.llm4ad_loader", + "benchmarks.LLM4AD.llm4ad_loader", + ): + try: + module = importlib.import_module(module_name) + sys.modules.setdefault("llm4ad_loader", module) + sys.modules.setdefault("LLM4AD.llm4ad_loader", module) + return module + except ModuleNotFoundError: + pass repo_root = Path(__file__).resolve().parents[1] - loader_path = repo_root / "LLM4AD" / "llm4ad_loader.py" - if not loader_path.exists(): - raise + loader_paths = ( + repo_root / "LLM4AD" / "llm4ad_loader.py", + repo_root / "benchmarks" / "LLM4AD" / "llm4ad_loader.py", + ) + loader_path = next((path for path in loader_paths if path.exists()), None) + if loader_path is None: + raise ModuleNotFoundError("Could not find llm4ad_loader in known layouts") spec = importlib.util.spec_from_file_location("llm4ad_loader", loader_path) module = importlib.util.module_from_spec(spec) @@ -71,7 +78,7 @@ def _import_llm4ad_loader(): return module -def _get_param_value(param): +def _get_param_value(param: Any) -> Any: """Try to extract a raw value/string from a Parameter-like object.""" for attr in ("value", "get", "get_value", "_value", "initial"): @@ -88,8 +95,23 @@ def _get_param_value(param): return getattr(param, "__dict__", None) +def _import_task_module(module_path: str) -> ModuleType: + """Import an LLM4AD task module from supported repository layouts.""" + + module_names = (module_path, f"benchmarks.{module_path}") + last_error: Exception | None = None + for module_name in module_names: + try: + module = importlib.import_module(module_name) + if getattr(module, "__file__", None) is not None: + return module + except Exception as exc: + last_error = exc + raise ModuleNotFoundError(f"Could not import task module '{module_path}'") from last_error + + @pytest.mark.parametrize("task", TASKS) -def test_lite_optimize_llm4ad_task(task): +def test_lite_optimize_llm4ad_task(task: dict[str, Any]) -> None: if not os.environ.get("OPENAI_API_KEY"): pytest.skip("OPENAI_API_KEY not set; skipping LLM-backed optimizer test.") @@ -106,7 +128,7 @@ def test_lite_optimize_llm4ad_task(task): module_path = task["module"] try: - mod = importlib.import_module(module_path) + mod = _import_task_module(module_path) except Exception as exc: pytest.skip(f"Could not import LLM4AD task module '{module_path}': {exc!r}") @@ -129,7 +151,7 @@ def test_lite_optimize_llm4ad_task(task): try: problem = llm4ad_loader.build_trace_problem_from_config( llm4ad_root=str(repo_root), - eval_module_path=module_path, + eval_module_path=mod.__name__, eval_class_name=task["eval_class"], eval_file_path=eval_file_path, entry_name=entry_name, @@ -148,6 +170,15 @@ def test_lite_optimize_llm4ad_task(task): param = problem["param"] guide = problem["guide"] optimizer_kwargs = problem["optimizer_kwargs"] + optimizer_kwargs = dict(optimizer_kwargs) + try: + from opto.utils.llm import LiteLLM + from trace_bench.runner import _make_optimizer_litellm + + model_name = os.environ.get("TRACE_LITELLM_MODEL", "gpt-5.4-nano") + optimizer_kwargs.setdefault("llm", _make_optimizer_litellm(LiteLLM, model_name)) + except Exception: + pass try: opt = OptoPrime(parameters=[param], **optimizer_kwargs) From 716cf82674d0dcb6a62c3f7b72a87b571bf35aa1 Mon Sep 17 00:00:00 2001 From: doxav <> Date: Sat, 13 Jun 2026 15:03:56 +0200 Subject: [PATCH 2/2] provide a .env template --- .env.template | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .env.template diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..fa7e16d --- /dev/null +++ b/.env.template @@ -0,0 +1,4 @@ +# Copy this file to .env for local live runs. Do not commit .env. +OPENAI_API_KEY= +TRACE_LITELLM_MODEL=gpt-5.4-nano +RECURSIVE_OPT_MODEL=gpt-5.4-nano