diff --git a/README.md b/README.md index 62da05d..3cc6bdd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A benchmarking framework for evaluating LLM-as-optimizer algorithms, built on [OpenTrace](https://github.com/AgentOpt/Trace). -Trace-Bench provides a **CLI**, **Gradio UI**, and **notebook workflows** to run reproducible experiments across LLM4AD, VeriBench, KernelBench, and HuggingFace QA tasks with fair comparisons between trainers, optimizers, and LLM backends. +Trace-Bench provides a **CLI**, **Gradio UI**, and **notebook workflows** to run reproducible experiments across LLM4AD, VeriBench, KernelBench, HuggingFace QA, and Terminal-Bench via Harbor with fair comparisons between trainers, optimizers, and LLM backends. **Full documentation:** [docs/](docs/README.md) @@ -28,9 +28,12 @@ pip install -e . # Optional extras can be combined: pip install -e ".[hf]" # HuggingFace QA benchmarks pip install -e ".[dspy]" # DSPy trainer adapter +pip install -e ".[terminal-bench]" # Terminal-Bench 2.0 via Harbor pip install -e ".[all-external]" # everything ``` +Note: the `terminal-bench` extra installs Harbor, which currently requires Python >=3.12. + ## Quick Start ```bash @@ -71,6 +74,70 @@ Then run with a real-mode config: trace-bench run --config configs/smoke_real.yaml --runs-dir runs ``` + +## Terminal-Bench / Harbor Support + +Trace-Bench can optimize Harbor's Terminus-2 prompt template against Terminal-Bench 2.0 tasks. Task IDs use the `terminal_bench:` namespace, for example `terminal_bench:regex-log`. + +### Install and configure + +```bash +# Install Trace-Bench plus Harbor support. +pip install -e ".[terminal-bench]" + +# For local Docker execution: +export HARBOR_MODEL="openai/gpt-5-nano" +export OPENAI_API_KEY="sk-..." + +# For hosted Daytona execution, recommended in Colab or environments without Docker: +export DAYTONA_API_KEY="dtn_..." +export HARBOR_MODEL="openai/gpt-5-nano" +export OPENAI_API_KEY="sk-..." +``` + +Do not put API keys in config files or commit them to the repository. Trace-Bench reads `HARBOR_MODEL`, `OPENAI_API_KEY`, and `DAYTONA_API_KEY` from the process environment at runtime. If you install Harbor manually instead of using the extra, use `pip install "harbor[daytona]>=0.6.0"` for Daytona-backed runs or `pip install "harbor>=0.6.0"` for Docker-only runs. + +### Offline smoke run + +The demo config defaults to `mode: stub`, so Harbor is not invoked and no API keys are required: + +```bash +trace-bench validate --config configs/terminal_bench_demo.yaml --runs-dir runs --strict +trace-bench run --config configs/terminal_bench_demo.yaml --runs-dir runs +``` + +### Real Terminal-Bench run + +Switch the config to `mode: real`, or create a small real-mode config with a Terminal-Bench task. In real mode, each guide evaluation shells out to `harbor run` and stores Harbor command/stdout/stderr/results/trajectory artifacts under the Trace-Bench job artifacts directory (`runs//jobs//artifacts/terminal_bench/`). + +```yaml +mode: real +tasks: + - id: terminal_bench:regex-log + eval_kwargs: + harbor_dataset: terminal-bench@2.0 + harbor_env: daytona # or docker + harbor_model: null # read HARBOR_MODEL from the environment + harbor_cli_timeout_seconds: 1800 +``` + +Then run: + +```bash +trace-bench run --config path/to/terminal_bench_real.yaml --runs-dir runs +``` + +### Limits and operational notes + +- Terminal-Bench execution is external: real mode requires the Harbor CLI, a supported environment backend (`docker`, `daytona`, etc.), and model credentials accepted by Harbor/LiteLLM. The model must be compatible with Harbor/LiteLLM's chat-completion path; if a model reports a Responses API `messages`/`input` mismatch, use a compatible model alias or upgrade Harbor/LiteLLM. +- The built-in task discovery intentionally lists only a curated smoke/demo subset. You can still run any Harbor Terminal-Bench 2.0 slug directly as `terminal_bench:`. +- Real runs can be slow and cost-bearing because optimizers may evaluate the trainable prompt multiple times; keep `max_workers`, `ps_steps`, `ps_candidates`, and `ps_proposals` small while testing. +- Daytona quotas/API limits are outside Trace-Bench's control. Use `harbor_cli_timeout_seconds` and low concurrency when keys are limited. +- Stub mode verifies Trace-Bench wiring only; it does not prove Harbor, Daytona, Docker, credentials, or a specific Terminal-Bench task works. +- Trace-Bench currently optimizes the Terminus-2 prompt template, not Harbor environment definitions or benchmark tests. + +See `configs/terminal_bench_demo.yaml` and `notebooks/05_terminal_bench_demo.ipynb` for an end-to-end walkthrough. + ## Repository Layout ``` @@ -131,6 +198,7 @@ See [docs/running-experiments.md](docs/running-experiments.md) for full CLI usag |-------|----------|---------| | `hf` | `datasets>=2.0` | `hf:*` QA tasks | | `dspy` | `dspy-ai>=2.0` | `DSPyTrainer` in `trace_bench/trainers/` | +| `terminal-bench` | `harbor[daytona]>=0.6.0` | `terminal_bench:*` tasks through Harbor, including Daytona env support | | `all-external` | all of the above | everything | ## Tests diff --git a/configs/09_harbor_bench_colab.json b/configs/09_harbor_bench_colab.json new file mode 100644 index 0000000..51ffcc8 --- /dev/null +++ b/configs/09_harbor_bench_colab.json @@ -0,0 +1,42 @@ +{ + "runs_dir": "/home/user/code/Trace-Bench/notebooks/runs/harbor_trace_bench", + "mode": "real", + "seeds": [ + 123 + ], + "max_workers": 1, + "fail_fast": false, + "tasks": [ + { + "id": "terminal_bench:regex-log", + "eval_kwargs": { + "harbor_dataset": "terminal-bench@2.0", + "harbor_env": "daytona", + "harbor_model": "openrouter/openai/gpt-5-nano", + "harbor_cli_timeout_seconds": 1800, + "harbor_agent_kwargs": { + "max_turns": 3, + "parser_name": "json" + } + } + } + ], + "trainers": [ + { + "id": "PrioritySearch", + "params_variants": [ + { + "threads": 1, + "ps_steps": 1, + "ps_batches": 1, + "ps_candidates": 1, + "ps_proposals": 1, + "ps_mem_update": 1 + } + ], + "optimizer_kwargs": { + "memory_size": 5 + } + } + ] +} \ No newline at end of file diff --git a/configs/terminal_bench_demo.yaml b/configs/terminal_bench_demo.yaml new file mode 100644 index 0000000..1ddec46 --- /dev/null +++ b/configs/terminal_bench_demo.yaml @@ -0,0 +1,27 @@ +runs_dir: runs +mode: stub +seeds: [123] +max_workers: 1 +fail_fast: false + +# Terminal-Bench tasks require the Harbor CLI for real execution. +# In stub mode (default here) Harbor is not invoked. + +tasks: + - id: terminal_bench:regex-log + eval_kwargs: + harbor_dataset: terminal-bench@2.0 + # In Colab, prefer daytona (set DAYTONA_API_KEY). Locally, docker is fine. + harbor_env: daytona + # Real mode requires this (or env var HARBOR_MODEL), e.g. "openai/gpt-4o-mini". + harbor_model: null + +trainers: + - id: PrioritySearch + params_variants: + - threads: 1 + ps_steps: 1 + ps_batches: 1 + ps_candidates: 2 + ps_proposals: 2 + ps_mem_update: 1 diff --git a/notebooks/05_terminal_bench_demo.ipynb b/notebooks/05_terminal_bench_demo.ipynb new file mode 100644 index 0000000..48557ed --- /dev/null +++ b/notebooks/05_terminal_bench_demo.ipynb @@ -0,0 +1,175 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Terminal\u2011Bench 2.0 demo (Trace\u2011Bench)\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/AgentOpt/Trace-Bench/blob/main/notebooks/05_terminal_bench_demo.ipynb)\n", + "\n", + "This notebook demonstrates how to run **Terminal\u2011Bench 2.0** tasks via **Harbor** from Trace\u2011Bench.\n", + "\n", + "Key points:\n", + "- Tasks are referenced as `terminal_bench:` (e.g. `terminal_bench:regex-log`).\n", + "- In Colab, Terminal\u2011Bench typically requires **Daytona** (`DAYTONA_API_KEY`) because Docker is not available.\n", + "- The optimization target is the **Terminus\u20112 prompt template** (a trainable Trace node).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# If running from a cloned repo, install Trace-Bench in editable mode.\n", + "!pip -q install -e .\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "\n", + "# Pick a persistent runs directory.\n", + "# In Colab you likely want to mount Drive first and set this under /content/drive/...\n", + "RUNS_DIR = Path(os.environ.get('TRACE_BENCH_RUNS_DIR', 'runs')).resolve()\n", + "RUNS_DIR.mkdir(parents=True, exist_ok=True)\n", + "print('RUNS_DIR =', RUNS_DIR)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# List available trainers (from Trace/OpenTrace) and the curated TB task subset.\n", + "!trace-bench list-trainers | head -n 40\n", + "print('\\n--- terminal_bench sample tasks ---')\n", + "!trace-bench list-tasks --bench terminal_bench --root .\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1) Stub smoke (offline)\n", + "\n", + "The shipped config `configs/terminal_bench_demo.yaml` defaults to `mode: stub`, meaning Harbor is **not** invoked.\n", + "This is useful for CI and quick sanity checks.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!trace-bench validate --config configs/terminal_bench_demo.yaml --runs-dir \"$RUNS_DIR\" --strict\n", + "!trace-bench run --config configs/terminal_bench_demo.yaml --runs-dir \"$RUNS_DIR\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Real mode (Harbor + Daytona)\n", + "\n", + "Requirements:\n", + "- `harbor` CLI available (`pip install harbor` typically provides it)\n", + "- `DAYTONA_API_KEY` exported (Colab recommended)\n", + "- A Harbor model name provided via `HARBOR_MODEL` (e.g. `openai/gpt-4o-mini`)\n", + "\n", + "If you don't have these, keep using stub mode.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import shutil, textwrap\n", + "\n", + "if shutil.which('harbor') is None:\n", + " print('harbor CLI not found. Install it first (e.g. pip install harbor).')\n", + "else:\n", + " print('harbor CLI found:', shutil.which('harbor'))\n", + "\n", + "print('DAYTONA_API_KEY set:', bool(os.environ.get('DAYTONA_API_KEY')))\n", + "print('HARBOR_MODEL:', os.environ.get('HARBOR_MODEL'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a small real-mode config on the fly.\n", + "# NOTE: replace trainer settings / timeouts as needed.\n", + "real_cfg = f\"\"\"\n", + "runs_dir: {RUNS_DIR}\n", + "mode: real\n", + "seeds: [123]\n", + "max_workers: 1\n", + "fail_fast: false\n", + "\n", + "tasks:\n", + " - id: terminal_bench:regex-log\n", + " eval_kwargs:\n", + " harbor_dataset: terminal-bench@2.0\n", + " harbor_env: daytona\n", + " # read from env in the guide if omitted\n", + " harbor_model: null\n", + " harbor_cli_timeout_seconds: 1800\n", + "\n", + "trainers:\n", + " - id: PrioritySearch\n", + " params_variants:\n", + " - threads: 1\n", + " ps_steps: 1\n", + " ps_batches: 1\n", + " ps_candidates: 2\n", + " ps_proposals: 2\n", + " ps_mem_update: 1\n", + "\"\"\"\n", + "\n", + "cfg_path = Path('configs/terminal_bench_real_tmp.yaml')\n", + "cfg_path.write_text(textwrap.dedent(real_cfg))\n", + "print('Wrote', cfg_path)\n", + "print(cfg_path.read_text()[:400])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run a single TB task in real mode.\n", + "# This will write Harbor artifacts under runs//jobs//artifacts/terminal_bench/.\n", + "!trace-bench validate --config configs/terminal_bench_real_tmp.yaml --runs-dir \"$RUNS_DIR\" --strict\n", + "!trace-bench run --config configs/terminal_bench_real_tmp.yaml --runs-dir \"$RUNS_DIR\"\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/09_harbor_bench.ipynb b/notebooks/09_harbor_bench.ipynb new file mode 100644 index 0000000..47dc053 --- /dev/null +++ b/notebooks/09_harbor_bench.ipynb @@ -0,0 +1,631 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8baf49cc", + "metadata": {}, + "source": [ + "# Trace-Bench × Harbor / Terminal-Bench 2.0 demo\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/doxav/Trace-Bench/blob/harbor/notebooks/09_harbor_bench.ipynb)\n", + "\n", + "This notebook demonstrates the new **Harbor benchmark adapter** in Trace-Bench.\n", + "\n", + "It is designed to run in **Google Colab** from the `doxav/Trace-Bench` `harbor` branch and documents the full workflow:\n", + "\n", + "1. install Trace-Bench from the `harbor` branch;\n", + "2. install Harbor CLI support;\n", + "3. read Colab secrets for `DAYTONA_API_KEY`, `OPENROUTER_API_KEY`, and `OPENAI_API_KEY`;\n", + "4. list Terminal-Bench tasks and Trace-Bench trainers;\n", + "5. build a Trace-Bench config for `terminal_bench:`;\n", + "6. run a safe **stub** validation when keys are missing;\n", + "7. run a **real Harbor/Daytona** job when the required keys are available;\n", + "8. inspect `results.csv`, `summary.json`, `job_meta.json`, and Harbor artifacts such as trajectory files.\n", + "\n", + "## Execution modes\n", + "\n", + "- **Stub mode**: always runnable in Colab; Harbor is not invoked.\n", + "- **Real mode**: requires `DAYTONA_API_KEY` and either `OPENROUTER_API_KEY` or `OPENAI_API_KEY`.\n", + "\n", + "The optimized object is the **Terminus-2 agent prompt template** exposed as a Trace trainable node by the Harbor adapter.\n" + ] + }, + { + "cell_type": "markdown", + "id": "cc3f11db", + "metadata": {}, + "source": [ + "## 0. Runtime options\n", + "\n", + "Edit these values before running if you want a different task, trainer, or model. The defaults are intentionally small so the notebook is suitable for a first validation pass.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d0f0f0f", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import os, sys, json, shutil, subprocess, textwrap, time, glob\n", + "\n", + "# --- Repository / notebook source ---\n", + "TRACE_BENCH_REPO = \"https://github.com/doxav/Trace-Bench.git\"\n", + "TRACE_BENCH_BRANCH = \"harbor\"\n", + "TRACE_BENCH_DIR = Path(\"/content/Trace-Bench\")\n", + "\n", + "# --- Harbor / Terminal-Bench task configuration ---\n", + "# Use a curated sample task from the adapter. Change after listing tasks below.\n", + "TASK_NAME = os.environ.get(\"TBENCH_TASK\", \"regex-log\")\n", + "TRAINER_ID = os.environ.get(\"TRACE_BENCH_TRAINER\", \"PrioritySearch\")\n", + "HARBOR_DATASET = os.environ.get(\"HARBOR_DATASET\", \"terminal-bench@2.0\")\n", + "\n", + "# Keep real runs bounded. Raise this only after a smoke run succeeds.\n", + "HARBOR_MAX_TURNS = int(os.environ.get(\"HARBOR_MAX_TURNS\", \"3\"))\n", + "HARBOR_TIMEOUT_SECONDS = int(os.environ.get(\"HARBOR_TIMEOUT_SECONDS\", \"1800\"))\n", + "\n", + "# Model selection. Can be overridden via Colab secret/env HARBOR_MODEL or OPENROUTER_MODEL.\n", + "DEFAULT_OPENROUTER_MODEL = os.environ.get(\"OPENROUTER_MODEL\", \"openai/gpt-4o-mini\")\n", + "DEFAULT_OPENAI_MODEL = os.environ.get(\"OPENAI_MODEL\", \"openai/gpt-4o-mini\")\n", + "\n", + "# Where Trace-Bench run artifacts will be written.\n", + "RUN_DATE = time.strftime(\"%Y-%m-%d\")\n", + "DEFAULT_DRIVE_RUNS = Path(f\"/content/drive/MyDrive/bench/{RUN_DATE}/harbor_trace_bench\")\n", + "DEFAULT_LOCAL_RUNS = Path(f\"/content/harbor_trace_bench_runs/{RUN_DATE}\")\n", + "\n", + "print(\"Configured task:\", TASK_NAME)\n", + "print(\"Configured trainer:\", TRAINER_ID)\n", + "print(\"Harbor dataset:\", HARBOR_DATASET)\n" + ] + }, + { + "cell_type": "markdown", + "id": "993f69f5", + "metadata": {}, + "source": [ + "## 1. Mount Google Drive for persistent artifacts\n", + "\n", + "The notebook writes run outputs to Drive if available. If Drive mount is skipped or unavailable, it falls back to `/content`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef59abe9", + "metadata": {}, + "outputs": [], + "source": [ + "IN_COLAB = False\n", + "try:\n", + " import google.colab # type: ignore\n", + " IN_COLAB = True\n", + "except Exception:\n", + " IN_COLAB = False\n", + "\n", + "if IN_COLAB:\n", + " try:\n", + " from google.colab import drive # type: ignore\n", + " drive.mount(\"/content/drive\")\n", + " RUNS_DIR = DEFAULT_DRIVE_RUNS\n", + " except Exception as exc:\n", + " print(\"Drive mount failed or was skipped; using local /content runs dir.\")\n", + " print(type(exc).__name__, exc)\n", + " RUNS_DIR = DEFAULT_LOCAL_RUNS\n", + "else:\n", + " RUNS_DIR = Path.cwd() / \"runs\" / \"harbor_trace_bench\"\n", + "\n", + "RUNS_DIR.mkdir(parents=True, exist_ok=True)\n", + "os.environ[\"TRACE_BENCH_RUNS_DIR\"] = str(RUNS_DIR)\n", + "print(\"RUNS_DIR =\", RUNS_DIR)\n" + ] + }, + { + "cell_type": "markdown", + "id": "a8176b26", + "metadata": {}, + "source": [ + "## 2. Clone and install Trace-Bench from the `harbor` branch\n", + "\n", + "Colab opens the notebook file, but it does not automatically clone the repository. This cell clones the target branch, checks out the exact code, and installs it in editable mode.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70296d97", + "metadata": {}, + "outputs": [], + "source": [ + "def run_cmd(cmd, *, cwd=None, check=True, env=None):\n", + " print(\"$\", \" \".join(map(str, cmd)))\n", + " proc = subprocess.run(\n", + " list(map(str, cmd)),\n", + " cwd=str(cwd) if cwd else None,\n", + " env=env,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.STDOUT,\n", + " )\n", + " print(proc.stdout[-4000:])\n", + " if check and proc.returncode != 0:\n", + " raise RuntimeError(f\"Command failed with exit code {proc.returncode}: {' '.join(map(str, cmd))}\")\n", + " return proc\n", + "\n", + "if IN_COLAB:\n", + " if not TRACE_BENCH_DIR.exists():\n", + " run_cmd([\"git\", \"clone\", \"--depth\", \"1\", \"--branch\", TRACE_BENCH_BRANCH, TRACE_BENCH_REPO, TRACE_BENCH_DIR])\n", + " else:\n", + " print(f\"Repo already exists at {TRACE_BENCH_DIR}; refreshing {TRACE_BENCH_BRANCH}.\")\n", + " run_cmd([\"git\", \"-C\", TRACE_BENCH_DIR, \"fetch\", \"origin\", TRACE_BENCH_BRANCH, \"--depth\", \"1\"], check=False)\n", + " run_cmd([\"git\", \"-C\", TRACE_BENCH_DIR, \"checkout\", TRACE_BENCH_BRANCH], check=False)\n", + " run_cmd([\"git\", \"-C\", TRACE_BENCH_DIR, \"pull\", \"--ff-only\", \"origin\", TRACE_BENCH_BRANCH], check=False)\n", + " os.chdir(TRACE_BENCH_DIR)\n", + "else:\n", + " # Local notebook: assume it is being run from inside the repo or a checkout.\n", + " if (Path.cwd() / \"trace_bench\").exists():\n", + " TRACE_BENCH_DIR = Path.cwd()\n", + " os.chdir(TRACE_BENCH_DIR)\n", + "\n", + "print(\"CWD:\", Path.cwd())\n", + "run_cmd([\"git\", \"rev-parse\", \"--abbrev-ref\", \"HEAD\"], check=False)\n", + "run_cmd([\"git\", \"rev-parse\", \"HEAD\"], check=False)\n", + "\n", + "# Install Trace-Bench and lightweight notebook dependencies.\n", + "run_cmd([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-U\", \"pip\"])\n", + "run_cmd([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"-e\", \".\"])\n", + "run_cmd([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"pandas\", \"pyyaml\"])\n" + ] + }, + { + "cell_type": "markdown", + "id": "b9427669", + "metadata": {}, + "source": [ + "## 3. Install Harbor CLI support\n", + "\n", + "Real Harbor runs need a working `harbor` command. This cell tries common package names. If your environment requires a different Harbor install command, replace this cell with your internal install line.\n", + "\n", + "Stub mode does **not** require Harbor.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4ddc3546", + "metadata": {}, + "outputs": [], + "source": [ + "def install_harbor_best_effort():\n", + " candidates = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"harbor[daytona]>=0.6.0\"],\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"harbor>=0.6.0\"],\n", + " ]\n", + " for cmd in candidates:\n", + " proc = run_cmd(cmd, check=False)\n", + " if proc.returncode == 0 and shutil.which(\"harbor\"):\n", + " return True\n", + " return shutil.which(\"harbor\") is not None\n", + "\n", + "HARBOR_AVAILABLE = install_harbor_best_effort()\n", + "print(\"harbor available:\", HARBOR_AVAILABLE, shutil.which(\"harbor\"))\n", + "if HARBOR_AVAILABLE:\n", + " run_cmd([\"harbor\", \"--help\"], check=False)\n", + "else:\n", + " print(\"Harbor CLI not found. Real mode will be skipped; stub mode can still run.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "28cec35e", + "metadata": {}, + "source": [ + "## 4. Load Colab secrets / environment keys\n", + "\n", + "This cell uses Colab secrets if available:\n", + "\n", + "- `DAYTONA_API_KEY` — required for real Harbor runs in Colab.\n", + "- `OPENROUTER_API_KEY` — preferred if present; exposed through OpenAI-compatible env vars.\n", + "- `OPENAI_API_KEY` — used if OpenRouter is not present.\n", + "\n", + "If neither model key is set, the notebook automatically stays in **stub mode**.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "708ae180", + "metadata": {}, + "outputs": [], + "source": [ + "def get_secret_or_env(name: str) -> str:\n", + " # Prefer existing env var, then Colab secret.\n", + " if os.environ.get(name):\n", + " return os.environ[name]\n", + " if IN_COLAB:\n", + " try:\n", + " from google.colab import userdata # type: ignore\n", + " value = userdata.get(name)\n", + " if value:\n", + " return value\n", + " except Exception:\n", + " pass\n", + " return \"\"\n", + "\n", + "DAYTONA_API_KEY = get_secret_or_env(\"DAYTONA_API_KEY\")\n", + "OPENROUTER_API_KEY = get_secret_or_env(\"OPENROUTER_API_KEY\")\n", + "OPENAI_API_KEY = get_secret_or_env(\"OPENAI_API_KEY\")\n", + "\n", + "if DAYTONA_API_KEY:\n", + " os.environ[\"DAYTONA_API_KEY\"] = DAYTONA_API_KEY\n", + "\n", + "# Provider policy:\n", + "# - Prefer OpenRouter when OPENROUTER_API_KEY is present.\n", + "# - Otherwise use OpenAI when OPENAI_API_KEY is present.\n", + "if OPENROUTER_API_KEY:\n", + " os.environ[\"OPENROUTER_API_KEY\"] = OPENROUTER_API_KEY\n", + " os.environ[\"OPENAI_API_KEY\"] = OPENROUTER_API_KEY\n", + " os.environ[\"OPENAI_BASE_URL\"] = \"https://openrouter.ai/api/v1\"\n", + " PROVIDER = \"openrouter\"\n", + " HARBOR_MODEL = os.environ.get(\"HARBOR_MODEL\") or DEFAULT_OPENROUTER_MODEL\n", + "elif OPENAI_API_KEY:\n", + " os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n", + " os.environ.pop(\"OPENAI_BASE_URL\", None)\n", + " PROVIDER = \"openai\"\n", + " HARBOR_MODEL = os.environ.get(\"HARBOR_MODEL\") or DEFAULT_OPENAI_MODEL\n", + "else:\n", + " PROVIDER = \"none\"\n", + " HARBOR_MODEL = os.environ.get(\"HARBOR_MODEL\", \"\")\n", + "\n", + "if HARBOR_MODEL:\n", + " os.environ[\"HARBOR_MODEL\"] = HARBOR_MODEL\n", + "\n", + "REAL_READY = bool(DAYTONA_API_KEY and HARBOR_MODEL and (OPENROUTER_API_KEY or OPENAI_API_KEY) and HARBOR_AVAILABLE)\n", + "MODE = \"real\" if REAL_READY else \"stub\"\n", + "HARBOR_ENV = \"daytona\" if DAYTONA_API_KEY else \"docker\"\n", + "\n", + "print(\"Provider:\", PROVIDER)\n", + "print(\"DAYTONA_API_KEY set:\", bool(DAYTONA_API_KEY))\n", + "print(\"OPENROUTER_API_KEY set:\", bool(OPENROUTER_API_KEY))\n", + "print(\"OPENAI_API_KEY set:\", bool(OPENAI_API_KEY if not OPENROUTER_API_KEY else OPENROUTER_API_KEY))\n", + "print(\"HARBOR_MODEL:\", HARBOR_MODEL or \"\")\n", + "print(\"Harbor env:\", HARBOR_ENV)\n", + "print(\"Selected Trace-Bench mode:\", MODE)\n", + "\n", + "if MODE == \"stub\":\n", + " print(\"\\nRunning in STUB mode: Harbor/Daytona will not be invoked. Add DAYTONA_API_KEY + model key for a real run.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3d120bf7", + "metadata": {}, + "source": [ + "## 5. List Terminal-Bench tasks and Trace-Bench trainers\n", + "\n", + "The Harbor adapter exposes a curated subset of Terminal-Bench 2.0 tasks for smoke/demo runs. You can still run any task by setting `TASK_NAME` to a valid Harbor task slug.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d62bce3e", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Trace-Bench trainers:\")\n", + "run_cmd([sys.executable, \"-m\", \"trace_bench\", \"list-trainers\", \"--all\"], check=False)\n", + "\n", + "print(\"\\nTerminal-Bench sample tasks:\")\n", + "proc = run_cmd([sys.executable, \"-m\", \"trace_bench\", \"list-tasks\", \"--bench\", \"terminal_bench\"], check=False)\n", + "listed_tasks = [line.strip() for line in proc.stdout.splitlines() if line.strip().startswith(\"terminal_bench:\")]\n", + "print(\"Parsed terminal_bench tasks:\", listed_tasks)\n", + "\n", + "if TASK_NAME not in [t.split(\":\", 1)[1] for t in listed_tasks] and listed_tasks:\n", + " print(f\"Configured TASK_NAME={TASK_NAME!r} is not in the curated list; this is allowed if Harbor knows it.\")\n", + "\n", + "print(\"\\nChosen TASK_NAME:\", TASK_NAME)\n", + "print(\"Chosen TRAINER_ID:\", TRAINER_ID)\n" + ] + }, + { + "cell_type": "markdown", + "id": "a26ea07a", + "metadata": {}, + "source": [ + "## 6. Create a Trace-Bench config\n", + "\n", + "The config uses the same Trace-Bench runner surface as LLM4AD/VeriBench-style tasks:\n", + "\n", + "- `tasks[].id = terminal_bench:`\n", + "- Harbor parameters live under `tasks[].eval_kwargs`\n", + "- trainer parameters live under `trainers[].params_variants`\n", + "\n", + "In **real mode**, this invokes Harbor with Daytona. In **stub mode**, Harbor is not invoked.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ef355e9", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "CONFIG_DIR = Path(\"configs\")\n", + "CONFIG_DIR.mkdir(exist_ok=True)\n", + "CONFIG_PATH = CONFIG_DIR / \"09_harbor_bench_colab.json\"\n", + "\n", + "config = {\n", + " \"runs_dir\": str(RUNS_DIR),\n", + " \"mode\": MODE,\n", + " \"seeds\": [123],\n", + " \"max_workers\": 1,\n", + " \"fail_fast\": False,\n", + " \"tasks\": [\n", + " {\n", + " \"id\": f\"terminal_bench:{TASK_NAME}\",\n", + " \"eval_kwargs\": {\n", + " \"harbor_dataset\": HARBOR_DATASET,\n", + " \"harbor_env\": HARBOR_ENV,\n", + " \"harbor_model\": HARBOR_MODEL or None,\n", + " \"harbor_cli_timeout_seconds\": HARBOR_TIMEOUT_SECONDS,\n", + " \"harbor_agent_kwargs\": {\n", + " \"max_turns\": HARBOR_MAX_TURNS,\n", + " \"parser_name\": \"json\",\n", + " },\n", + " },\n", + " }\n", + " ],\n", + " \"trainers\": [\n", + " {\n", + " \"id\": TRAINER_ID,\n", + " \"params_variants\": [\n", + " {\n", + " \"threads\": 1,\n", + " \"ps_steps\": 1,\n", + " \"ps_batches\": 1,\n", + " \"ps_candidates\": 1,\n", + " \"ps_proposals\": 1,\n", + " \"ps_mem_update\": 1,\n", + " }\n", + " ],\n", + " \"optimizer_kwargs\": {\"memory_size\": 5},\n", + " }\n", + " ],\n", + "}\n", + "\n", + "CONFIG_PATH.write_text(json.dumps(config, indent=2), encoding=\"utf-8\")\n", + "print(\"Wrote config:\", CONFIG_PATH)\n", + "print(CONFIG_PATH.read_text())\n" + ] + }, + { + "cell_type": "markdown", + "id": "1da5a1dd", + "metadata": {}, + "source": [ + "## 7. Validate the config\n", + "\n", + "Validation should succeed and write any validation artifacts under the same `RUNS_DIR`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5170ac1", + "metadata": {}, + "outputs": [], + "source": [ + "validate_cmd = [\n", + " sys.executable, \"-m\", \"trace_bench\", \"validate\",\n", + " \"--config\", str(CONFIG_PATH),\n", + " \"--strict\",\n", + " \"--runs-dir\", str(RUNS_DIR),\n", + "]\n", + "proc = run_cmd(validate_cmd, check=False)\n", + "if proc.returncode != 0:\n", + " raise RuntimeError(\"Validation failed. See output above.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "421b98e7", + "metadata": {}, + "source": [ + "## 8. Run the Harbor benchmark through Trace-Bench\n", + "\n", + "This uses `trace-bench run` exactly like the other benchmark integrations.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df768f91", + "metadata": {}, + "outputs": [], + "source": [ + "run_cmd([\n", + " sys.executable, \"-m\", \"trace_bench\", \"run\",\n", + " \"--config\", str(CONFIG_PATH),\n", + " \"--runs-dir\", str(RUNS_DIR),\n", + "], check=True)\n" + ] + }, + { + "cell_type": "markdown", + "id": "b732fcc1", + "metadata": {}, + "source": [ + "## 9. Inspect run-level artifacts\n", + "\n", + "The run should contain:\n", + "\n", + "- `meta/config.snapshot.yaml` or `.json` depending on current writer\n", + "- `meta/manifest.json`\n", + "- `meta/env.json`\n", + "- `results.csv`\n", + "- `summary.json`\n", + "- `jobs//job_meta.json`\n", + "- `jobs//results.json`\n", + "- `jobs//events.jsonl`\n", + "- in real mode: Harbor artifacts under `jobs//artifacts/terminal_bench/`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4831492f", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "\n", + "def find_run_dirs(root: Path):\n", + " root = Path(root)\n", + " dirs = []\n", + " if root.exists():\n", + " dirs.extend([p for p in root.iterdir() if p.is_dir() and (p / \"results.csv\").exists()])\n", + " nested = root / \"runs\"\n", + " if nested.exists():\n", + " dirs.extend([p for p in nested.iterdir() if p.is_dir() and (p / \"results.csv\").exists()])\n", + " return sorted(set(dirs), key=lambda p: p.stat().st_mtime)\n", + "\n", + "run_dirs = find_run_dirs(RUNS_DIR)\n", + "assert run_dirs, f\"No Trace-Bench run dirs found under {RUNS_DIR}\"\n", + "latest_run = run_dirs[-1]\n", + "print(\"Latest run:\", latest_run)\n", + "print(\"Files:\", sorted([p.name for p in latest_run.iterdir()]))\n", + "\n", + "summary_path = latest_run / \"summary.json\"\n", + "results_path = latest_run / \"results.csv\"\n", + "manifest_path = latest_run / \"meta\" / \"manifest.json\"\n", + "\n", + "print(\"summary.json exists:\", summary_path.exists())\n", + "print(\"results.csv exists:\", results_path.exists())\n", + "print(\"manifest.json exists:\", manifest_path.exists())\n", + "\n", + "summary = json.loads(summary_path.read_text()) if summary_path.exists() else {}\n", + "print(\"Summary:\")\n", + "print(json.dumps(summary, indent=2)[:3000])\n", + "\n", + "df = pd.read_csv(results_path)\n", + "display(df)\n" + ] + }, + { + "cell_type": "markdown", + "id": "13ca94bc", + "metadata": {}, + "source": [ + "## 10. Inspect job-level artifacts and Harbor trajectory output\n", + "\n", + "In stub mode, Harbor artifacts may be absent or minimal. In real mode, you should see Harbor job/trial outputs and a trajectory conversion file.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "471c19f1", + "metadata": {}, + "outputs": [], + "source": [ + "jobs_dir = latest_run / \"jobs\"\n", + "job_dirs = sorted([p for p in jobs_dir.iterdir() if p.is_dir()])\n", + "assert job_dirs, \"No job dirs found\"\n", + "\n", + "for job_dir in job_dirs:\n", + " print(\"\\n=== JOB\", job_dir.name, \"===\")\n", + " for rel in [\"job_meta.json\", \"results.json\", \"events.jsonl\"]:\n", + " p = job_dir / rel\n", + " print(rel, \"exists:\", p.exists())\n", + " if p.exists() and p.stat().st_size < 12000:\n", + " txt = p.read_text(encoding=\"utf-8\", errors=\"replace\")\n", + " print(txt[:3000])\n", + "\n", + " tb_art = job_dir / \"artifacts\" / \"terminal_bench\"\n", + " print(\"terminal_bench artifact dir:\", tb_art, \"exists:\", tb_art.exists())\n", + " if tb_art.exists():\n", + " all_files = sorted([p for p in tb_art.rglob(\"*\") if p.is_file()])\n", + " print(\"Terminal-Bench artifact files:\")\n", + " for p in all_files[:50]:\n", + " print(\" -\", p.relative_to(tb_art), f\"({p.stat().st_size} bytes)\")\n", + " trajectory_files = [p for p in all_files if \"trajectory\" in p.name.lower()]\n", + " if trajectory_files:\n", + " print(\"\\nTrajectory-related files:\")\n", + " for p in trajectory_files:\n", + " print(\" -\", p)\n", + " sample = trajectory_files[-1]\n", + " print(\"\\nSample trajectory artifact:\", sample)\n", + " print(sample.read_text(encoding=\"utf-8\", errors=\"replace\")[:4000])\n" + ] + }, + { + "cell_type": "markdown", + "id": "4f100dea", + "metadata": {}, + "source": [ + "## 11. Optional: switch task/trainer/model and rerun\n", + "\n", + "To try a different task:\n", + "\n", + "1. change `TASK_NAME` in Cell 0, or set `TBENCH_TASK` as an environment variable;\n", + "2. re-run cells 6 onward.\n", + "\n", + "For real Harbor runs in Colab:\n", + "\n", + "- add Colab secret `DAYTONA_API_KEY`;\n", + "- add either `OPENROUTER_API_KEY` or `OPENAI_API_KEY`;\n", + "- optionally set `HARBOR_MODEL` to a model string supported by your Harbor/LLM setup.\n", + "\n", + "Suggested first real run settings:\n", + "\n", + "```python\n", + "TASK_NAME = \"regex-log\"\n", + "HARBOR_MAX_TURNS = 3\n", + "HARBOR_TIMEOUT_SECONDS = 1800\n", + "```\n", + "\n", + "Then re-run cells 6–10.\n" + ] + }, + { + "cell_type": "markdown", + "id": "6dd08738", + "metadata": {}, + "source": [ + "## 12. Validation checklist\n", + "\n", + "After running all cells, check:\n", + "\n", + "- [ ] `trace-bench list-tasks --bench terminal_bench` lists curated Harbor/Terminal-Bench tasks.\n", + "- [ ] `trace-bench list-trainers --all` lists available Trace trainers.\n", + "- [ ] `validate --strict --runs-dir ` succeeds.\n", + "- [ ] `trace-bench run` writes a run under `RUNS_DIR`.\n", + "- [ ] `results.csv` contains `terminal_bench:`.\n", + "- [ ] `summary.json` reports exactly one planned job for this demo config.\n", + "- [ ] In real mode, Harbor artifacts exist under `jobs//artifacts/terminal_bench/`.\n", + "- [ ] In real mode, trajectory artifacts exist and can be inspected.\n" + ] + } + ], + "metadata": { + "colab": { + "name": "09_harbor_bench.ipynb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/setup.py b/setup.py index 769805e..1c3bae0 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,11 @@ # External trainer adapters (trace_bench/trainers/) "dspy": ["dspy-ai>=2.0"], + + # Harbor CLI / Terminal-Bench 2.0 adapter. + # Harbor 0.6.x currently requires Python >=3.12. Keep this optional and + # guarded so core Trace-Bench installs remain stable on older Python. + "terminal-bench": ["harbor[daytona]>=0.6.0; python_version >= '3.12'"], } extras_require["all-external"] = sorted( {pkg for pkgs in extras_require.values() for pkg in pkgs} diff --git a/tests/test_terminal_bench_atif.py b/tests/test_terminal_bench_atif.py new file mode 100644 index 0000000..a3cbf8c --- /dev/null +++ b/tests/test_terminal_bench_atif.py @@ -0,0 +1,34 @@ +from trace_bench.terminal_bench.atif import ( + summarize_atif, + atif_to_trace_like, + atif_feedback_snippet, +) + + +def test_atif_helpers_smoke(): + payload = { + "steps": [ + { + "role": "assistant", + "content": "Plan: list files, then run tests.", + "tool_calls": [{"name": "bash", "args": {"cmd": "ls -la"}}], + }, + { + "role": "tool", + "content": "file1\nfile2\n", + }, + ] + } + + summary = summarize_atif(payload, last_n=1) + assert summary.n_steps == 2 + assert len(summary.last_steps) == 1 + + trace_like = atif_to_trace_like(payload, last_n=10) + assert trace_like["format"] == "trace_bench.atif.v1" + assert trace_like["n_steps"] == 2 + assert len(trace_like["events"]) == 2 + + snippet = atif_feedback_snippet(payload, max_chars=2000, last_n=2) + assert "ATIF" in snippet + assert "steps=2" in snippet diff --git a/tests/test_terminal_bench_harbor_runner.py b/tests/test_terminal_bench_harbor_runner.py new file mode 100644 index 0000000..40fe023 --- /dev/null +++ b/tests/test_terminal_bench_harbor_runner.py @@ -0,0 +1,220 @@ +from pathlib import Path + +import pytest + +from trace_bench.terminal_bench.harbor_runner import build_harbor_cmd + + +def test_build_harbor_cmd_requires_dataset_or_path(tmp_path: Path): + with pytest.raises(ValueError): + build_harbor_cmd( + job_name="x", + jobs_dir=tmp_path, + model_name="m", + env_type="docker", + task_name="t", + dataset=None, + task_path=None, + ) + + with pytest.raises(ValueError): + build_harbor_cmd( + job_name="x", + jobs_dir=tmp_path, + model_name="m", + env_type="docker", + task_name="t", + dataset="terminal-bench@2.0", + task_path=tmp_path, + ) + + +def test_build_harbor_cmd_dataset(tmp_path: Path): + cmd, _env = build_harbor_cmd( + job_name="job", + jobs_dir=tmp_path, + model_name="m", + env_type="daytona", + task_name="regex-log", + dataset="terminal-bench@2.0", + task_path=None, + agent_kwargs={"prompt_template_path": "/tmp/prompt.txt"}, + n_concurrent=1, + quiet=True, + ) + assert cmd[:2] == ["harbor", "run"] + assert "--dataset" in cmd + assert "--task-name" in cmd + assert "regex-log" in cmd + assert "--env" in cmd + assert "daytona" in cmd + + +def test_build_harbor_cmd_path(tmp_path: Path): + task_path = tmp_path / "task" + task_path.mkdir() + cmd, _env = build_harbor_cmd( + job_name="job", + jobs_dir=tmp_path, + model_name="m", + env_type="docker", + task_name="ignored", + dataset=None, + task_path=task_path, + agent_kwargs={}, + n_concurrent=1, + quiet=False, + ) + assert "--path" in cmd + assert str(task_path) in cmd + assert "--quiet" not in cmd + + +def test_run_harbor_eval_marks_trial_exception_failed(tmp_path: Path, monkeypatch): + from types import SimpleNamespace + + from trace_bench.terminal_bench import harbor_runner + + jobs_dir = tmp_path / "jobs" + job_dir = jobs_dir / "job" + trial_dir = job_dir / "regex-log__abc123" + trial_dir.mkdir(parents=True) + (job_dir / "result.json").write_text( + '{"stats":{"n_errored_trials":1}}', encoding="utf-8" + ) + (trial_dir / "result.json").write_text( + '{"exception_info":{"exception_type":"BadRequestError"}}', encoding="utf-8" + ) + + monkeypatch.setattr(harbor_runner, "harbor_available", lambda: True) + monkeypatch.setattr( + harbor_runner.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="", stderr=""), + ) + + result = harbor_runner.run_harbor_eval( + job_name="job", + jobs_dir=jobs_dir, + model_name="m", + env_type="daytona", + task_name="regex-log", + dataset="terminal-bench@2.0", + ) + + assert result.status == "failed" + assert result.trial_dir == trial_dir + assert result.trial_result["exception_info"]["exception_type"] == "BadRequestError" + + +def test_build_harbor_cmd_agent_kwargs_are_forwarded(tmp_path: Path): + cmd, _env = build_harbor_cmd( + job_name="job", + jobs_dir=tmp_path, + model_name="m", + env_type="daytona", + task_name="regex-log", + dataset="terminal-bench@2.0", + task_path=None, + agent_kwargs={"max_turns": 3, "parser_name": "json"}, + n_concurrent=1, + quiet=True, + ) + joined = " ".join(cmd) + assert "--agent-kwarg max_turns=3" in joined + assert "--agent-kwarg parser_name=json" in joined + + +def test_run_harbor_eval_without_job_result_returns_failure_payload( + tmp_path: Path, monkeypatch +): + from types import SimpleNamespace + + from trace_bench.terminal_bench import harbor_runner + + monkeypatch.setattr(harbor_runner, "harbor_available", lambda: True) + monkeypatch.setattr( + harbor_runner.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=2, stdout="", stderr="bad"), + ) + + result = harbor_runner.run_harbor_eval( + job_name="job", + jobs_dir=tmp_path / "jobs", + model_name="m", + env_type="daytona", + task_name="regex-log", + dataset="terminal-bench@2.0", + ) + + assert result.status == "failed" + assert ( + result.job_result["trace_bench"]["failure_reason"] + == "harbor_failed_without_job_result" + ) + assert result.job_result["trace_bench"]["returncode"] == 2 + + +def test_run_harbor_eval_retries_include_task_name_for_older_harbor( + tmp_path: Path, monkeypatch +): + from types import SimpleNamespace + + from trace_bench.terminal_bench import harbor_runner + + jobs_dir = tmp_path / "jobs" + job_dir = jobs_dir / "job" + job_dir.mkdir(parents=True) + (job_dir / "result.json").write_text('{"score": 1}', encoding="utf-8") + seen_cmds = [] + + def fake_run(cmd, **kwargs): + seen_cmds.append(cmd) + if "--task-name" in cmd: + return SimpleNamespace(returncode=2, stdout="", stderr="No such option") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(harbor_runner, "harbor_available", lambda: True) + monkeypatch.setattr(harbor_runner.subprocess, "run", fake_run) + + result = harbor_runner.run_harbor_eval( + job_name="job", + jobs_dir=jobs_dir, + model_name="m", + env_type="daytona", + task_name="regex-log", + dataset="terminal-bench@2.0", + ) + + assert result.status == "ok" + assert "--task-name" in seen_cmds[0] + assert "--include-task-name" in seen_cmds[1] + + +def test_run_harbor_eval_timeout_returns_failure_payload(tmp_path: Path, monkeypatch): + from trace_bench.terminal_bench import harbor_runner + + def raise_timeout(cmd, **kwargs): + raise harbor_runner.subprocess.TimeoutExpired(cmd=cmd, timeout=1, output="out") + + monkeypatch.setattr(harbor_runner, "harbor_available", lambda: True) + monkeypatch.setattr(harbor_runner.subprocess, "run", raise_timeout) + + result = harbor_runner.run_harbor_eval( + job_name="job", + jobs_dir=tmp_path / "jobs", + model_name="m", + env_type="daytona", + task_name="regex-log", + dataset="terminal-bench@2.0", + timeout_seconds=1, + ) + + assert result.status == "failed" + assert ( + result.job_result["trace_bench"]["failure_reason"] + == "harbor_failed_without_job_result" + ) + assert result.job_result["trace_bench"]["returncode"] == 124 + assert "timed out" in result.stderr diff --git a/tests/test_terminal_bench_registry.py b/tests/test_terminal_bench_registry.py new file mode 100644 index 0000000..212ab6b --- /dev/null +++ b/tests/test_terminal_bench_registry.py @@ -0,0 +1,53 @@ +from pathlib import Path + +import pytest + +from trace_bench.registry import _parse_bench, discover_tasks, load_task_bundle + + +def test_parse_bench_allows_terminal_bench(): + assert _parse_bench("terminal_bench") == {"terminal_bench"} + + +def test_terminal_bench_is_explicit_not_default(): + assert "terminal_bench" not in _parse_bench(None) + + +def test_discover_terminal_bench_subset_smoke(): + repo_root = Path(__file__).resolve().parents[1] + tasks = discover_tasks(repo_root, bench="terminal_bench") + ids = [t.id for t in tasks] + assert any(tid.startswith("terminal_bench:") for tid in ids) + + +def test_load_terminal_bench_bundle_stub_feedback(monkeypatch): + monkeypatch.setenv("TRACE_BENCH_MODE", "stub") + bundle = load_task_bundle( + "terminal_bench:regex-log", Path(__file__).resolve().parents[1] + ) + assert bundle["metadata"]["benchmark"] == "terminal_bench" + score, feedback = bundle["guide"].get_feedback("regex-log", "prompt", None) + assert score == 0.0 + assert "harbor not invoked" in feedback + + +def test_terminal_bench_real_mode_requires_model(monkeypatch): + from trace_bench.terminal_bench.task import TerminalBenchGuide + + monkeypatch.setenv("TRACE_BENCH_MODE", "real") + monkeypatch.delenv("HARBOR_MODEL", raising=False) + guide = TerminalBenchGuide(eval_kwargs={}) + with pytest.raises(RuntimeError) as exc: + guide.get_feedback("regex-log", "prompt", None) + assert "requires a Harbor model" in str(exc.value) + + +def test_terminal_bench_default_prompt_uses_harbor_placeholders(monkeypatch): + monkeypatch.setenv("TRACE_BENCH_MODE", "stub") + bundle = load_task_bundle( + "terminal_bench:regex-log", Path(__file__).resolve().parents[1] + ) + rendered = str(bundle["param"]("regex-log")) + assert "{instruction}" in rendered + assert "{terminal_state}" in rendered + assert "{{task_instruction}}" not in rendered diff --git a/trace_bench/cli.py b/trace_bench/cli.py index 8813326..466647e 100644 --- a/trace_bench/cli.py +++ b/trace_bench/cli.py @@ -360,7 +360,7 @@ def build_parser() -> argparse.ArgumentParser: "--dataset-name", dest="bench", default=None, - help="Bench selection: llm4ad,trace_examples,internal,veribench", + help="Bench selection: llm4ad,trace_examples,internal,veribench,hf,terminal_bench", ) list_t = sub.add_parser("list-trainers", help="List discoverable trainers") @@ -374,7 +374,7 @@ def build_parser() -> argparse.ArgumentParser: "--dataset-name", dest="bench", default=None, - help="Bench selection: llm4ad,trace_examples,internal,veribench", + help="Bench selection: llm4ad,trace_examples,internal,veribench,hf,terminal_bench", ) val_p.add_argument("--strict", action="store_true") val_p.add_argument("--runs-dir", "--output-dir", dest="runs_dir", default=None) diff --git a/trace_bench/registry.py b/trace_bench/registry.py index 1e62e43..a8a3ae3 100644 --- a/trace_bench/registry.py +++ b/trace_bench/registry.py @@ -155,6 +155,21 @@ def discover_hf_tasks() -> List[TaskSpec]: return specs +def discover_terminal_bench() -> List[TaskSpec]: + """Curated subset of Terminal-Bench tasks. + + Terminal-Bench task inventory is maintained externally and may evolve. + Trace-Bench intentionally ships a small, stable subset for demos/smoke. + + Users can still run any task by specifying `terminal_bench:` + directly in configs. + """ + + from trace_bench.terminal_bench import sample_task_ids + + return [TaskSpec(id=tid, suite="terminal_bench", module="terminal_bench") for tid in sample_task_ids()] + + def discover_trace_examples() -> List[TaskSpec]: return [ TaskSpec(id="trace_examples:greeting_stub", suite="trace_examples", module="greeting_stub"), @@ -370,17 +385,25 @@ def discover_trainers() -> List[TrainerSpec]: return sorted(specs.values(), key=lambda spec: spec.id) +_DEFAULT_BENCHES: Set[str] = {"llm4ad", "trace_examples", "internal", "veribench", "hf"} +_ALLOWED_BENCHES: Set[str] = _DEFAULT_BENCHES | {"terminal_bench"} + + def _parse_bench(bench: Optional[str]) -> Set[str]: if not bench: - return {"llm4ad", "trace_examples", "internal", "veribench", "hf"} + # Keep external Harbor-backed benchmarks opt-in. This avoids changing + # default task discovery behavior for users who did not request it. + return set(_DEFAULT_BENCHES) normalized = bench.replace("+", ",") parts = [p.strip() for p in normalized.split(",") if p.strip()] if not parts: - return {"llm4ad", "trace_examples", "internal", "veribench", "hf"} - allowed = {"llm4ad", "trace_examples", "internal", "veribench", "hf"} - unknown = [p for p in parts if p not in allowed] + return set(_DEFAULT_BENCHES) + unknown = [p for p in parts if p not in _ALLOWED_BENCHES] if unknown: - raise ValueError(f"Unknown bench selector(s): {unknown}. Allowed: {sorted(allowed)}") + raise ValueError( + f"Unknown bench selector(s): {unknown}. " + f"Allowed: {sorted(_ALLOWED_BENCHES)}" + ) return set(parts) @@ -398,6 +421,8 @@ def discover_tasks(tasks_root: str | Path, bench: Optional[str] = None) -> List[ specs.extend(discover_veribench()) if "hf" in selected: specs.extend(discover_hf_tasks()) + if "terminal_bench" in selected: + specs.extend(discover_terminal_bench()) return specs @@ -472,6 +497,10 @@ def load_task_module(task_id: str, tasks_root: str | Path): def load_task_bundle(task_id: str, tasks_root: str | Path, eval_kwargs: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: task_id = _normalize_task_id(task_id) + if task_id.startswith("terminal_bench:"): + from trace_bench.terminal_bench.task import build_terminal_bench_bundle + + return build_terminal_bench_bundle(task_id, eval_kwargs=eval_kwargs) if task_id.startswith("veribench:"): if task_id in {"veribench:all", _VERIBENCH_PLACEHOLDER}: raise NotImplementedError(_VERIBENCH_UNAVAILABLE) diff --git a/trace_bench/runner.py b/trace_bench/runner.py index 2751aef..4104f8a 100644 --- a/trace_bench/runner.py +++ b/trace_bench/runner.py @@ -920,6 +920,8 @@ def _write_atomic(path: str, payload_value: Dict[str, Any]) -> None: os.replace(tmp, target) start = _time.time() + os.environ["TRACE_BENCH_MODE"] = str(mode) + # Re-apply LLM config in the spawned subprocess (spawn context does not # inherit in-memory dspy.settings state from the parent process). if llm_cfg: @@ -964,6 +966,25 @@ def _write_atomic(path: str, payload_value: Dict[str, Any]) -> None: eval_kwargs = dict(eval_kwargs) eval_kwargs["framework"] = trainer_framework bundle = load_task_bundle(task_id, tasks_root, eval_kwargs=eval_kwargs) + # Provide job-scoped paths to context-aware guides. + try: + job_dir = Path(stdout_log).parent + guide_obj = bundle.get("guide") + if guide_obj is not None and hasattr(guide_obj, "set_trace_bench_context"): + guide_obj.set_trace_bench_context( + run_id="", + job_id=str(job_dir.name), + task_id=str(task_id), + suite=str(task_id.split(":", 1)[0] if ":" in task_id else ""), + mode=str(mode), + job_dir=str(job_dir), + artifacts_dir=str(job_dir / "artifacts"), + tb_dir=str(job_dir / "tb"), + runs_dir=str(job_dir.parent.parent if job_dir.parent.name == "jobs" else job_dir.parent), + ) + except Exception: + pass + _stub_bundle(bundle, mode) runtime = _resolve_runtime_from_bundle(bundle, trainer_spec, params) payload.update( @@ -1452,7 +1473,7 @@ def _run_job(self, job: JobSpec, timeout: Optional[float] = None) -> Tuple[Dict[ else: # ---- In-process path: no timeout overhead ---- with job_artifacts.stdout_log.open("a", encoding="utf-8", errors="ignore") as _logf, redirect_stdout(_logf), redirect_stderr(_logf): - payload = self._run_job_inprocess(job) + payload = self._run_job_inprocess(job, job_artifacts) status = payload.get("status", "failed") feedback = payload.get("feedback") @@ -1592,9 +1613,12 @@ def _run_job(self, job: JobSpec, timeout: Optional[float] = None) -> Tuple[Dict[ # In-process job execution (no timeout) # ------------------------------------------------------------------ - def _run_job_inprocess(self, job: JobSpec) -> Dict[str, Any]: + def _run_job_inprocess(self, job: JobSpec, job_artifacts: JobArtifacts) -> Dict[str, Any]: """Execute a job in the current process. Uses bundle cache.""" start_time = time.time() + # Expose the run mode to guides/evaluators (useful for offline CI). + os.environ["TRACE_BENCH_MODE"] = str(self.config.mode) + status = "ok" feedback: Optional[str] = None @@ -1624,6 +1648,24 @@ def _run_job_inprocess(self, job: JobSpec) -> Dict[str, Any]: if bundle is not None and status == "ok": runtime = _resolve_runtime_from_bundle(bundle, job.trainer, job.params) + # Allow guides to write job-scoped artifacts (e.g., external benchmarks). + guide_obj = bundle.get("guide") + if guide_obj is not None and hasattr(guide_obj, "set_trace_bench_context"): + try: + guide_obj.set_trace_bench_context( + run_id=str(self.config.run_id or ""), + job_id=str(job.job_id), + task_id=str(job.task_id), + suite=str(job.suite), + mode=str(self.config.mode), + job_dir=str(job_artifacts.job_dir), + artifacts_dir=str(job_artifacts.artifacts_dir), + tb_dir=str(job_artifacts.tb_dir), + runs_dir=str(self.artifacts.run_dir if self.artifacts else ""), + ) + except Exception: + pass + resolved_optimizer = runtime["resolved_optimizer"] resolved_guide = runtime["resolved_guide"] resolved_logger = runtime["resolved_logger"] diff --git a/trace_bench/terminal_bench/__init__.py b/trace_bench/terminal_bench/__init__.py new file mode 100644 index 0000000..bf57802 --- /dev/null +++ b/trace_bench/terminal_bench/__init__.py @@ -0,0 +1,35 @@ +"""Terminal-Bench integration helpers. + +This module intentionally keeps hard dependencies optional: +- The Harbor CLI ("harbor") is required only when running Terminal-Bench tasks. +- The Harbor Python package is required only when using the custom Terminus-2 wrapper. + +Trace-Bench exposes Terminal-Bench tasks under the task-id namespace: + terminal_bench: + +Where is the Terminal-Bench 2.x task slug (e.g. "regex-log"). +""" + +from __future__ import annotations + +from typing import List + +DEFAULT_HARBOR_DATASET = "terminal-bench@2.0" + +SAMPLE_TASK_SLUGS: List[str] = [ + "regex-log", + "log-summary-date-ranges", + "multi-source-data-merger", + "financial-document-processor", +] + + +def sample_task_ids() -> List[str]: + return [f"terminal_bench:{slug}" for slug in SAMPLE_TASK_SLUGS] + + +__all__ = [ + "DEFAULT_HARBOR_DATASET", + "SAMPLE_TASK_SLUGS", + "sample_task_ids", +] diff --git a/trace_bench/terminal_bench/agent.py b/trace_bench/terminal_bench/agent.py new file mode 100644 index 0000000..7dab918 --- /dev/null +++ b/trace_bench/terminal_bench/agent.py @@ -0,0 +1,50 @@ +"""Harbor agent wrapper(s) used by Trace-Bench. + +The main purpose is to make Terminus-2 prompt templates overridable from Trace-Bench. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Optional + +try: # Harbor is optional; only required for real Terminal-Bench runs. + from harbor.agents.terminus_2.terminus_2 import Terminus2 +except Exception as _exc: # pragma: no cover + Terminus2 = None # type: ignore + _HARBOR_IMPORT_ERROR: Exception = _exc + + +if Terminus2 is None: # pragma: no cover + + class TraceBenchTerminus2: # type: ignore + """Placeholder when Harbor is not installed.""" + + def __init__(self, *_: Any, **__: Any) -> None: + raise ImportError( + "Harbor is required to use TraceBenchTerminus2. " + "Install harbor and retry. Original import error: " + f"{_HARBOR_IMPORT_ERROR}" + ) + +else: + + class TraceBenchTerminus2(Terminus2): + """Terminus2 with an overridable prompt template.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + prompt_template_path: Optional[str] = kwargs.pop( + "prompt_template_path", None + ) + prompt_template: Optional[str] = kwargs.pop("prompt_template", None) + super().__init__(*args, **kwargs) + + if prompt_template_path: + self._prompt_template = Path(prompt_template_path).read_text( + encoding="utf-8" + ) + elif prompt_template is not None: + self._prompt_template = str(prompt_template) + + +__all__ = ["TraceBenchTerminus2"] diff --git a/trace_bench/terminal_bench/atif.py b/trace_bench/terminal_bench/atif.py new file mode 100644 index 0000000..dfd2933 --- /dev/null +++ b/trace_bench/terminal_bench/atif.py @@ -0,0 +1,131 @@ +"""Utilities for Harbor ATIF trajectories.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List +import json + + +@dataclass +class ATIFSummary: + """Compact view of an ATIF trajectory.""" + + n_steps: int + last_steps: List[Dict[str, Any]] + + +def load_atif(path: str | Path) -> Dict[str, Any]: + p = Path(path) + return json.loads(p.read_text(encoding="utf-8")) + + +def _get_steps(payload: Dict[str, Any]) -> List[Dict[str, Any]]: + """Best-effort extraction of step list across possible versions.""" + + steps = payload.get("steps") + if isinstance(steps, list): + return [s for s in steps if isinstance(s, dict)] + + nested = payload.get("trajectory") + if isinstance(nested, dict): + steps = nested.get("steps") + if isinstance(steps, list): + return [s for s in steps if isinstance(s, dict)] + + return [] + + +def summarize_atif(payload: Dict[str, Any], last_n: int = 6) -> ATIFSummary: + steps = _get_steps(payload) + n_steps = len(steps) + if last_n <= 0: + return ATIFSummary(n_steps=n_steps, last_steps=[]) + return ATIFSummary(n_steps=n_steps, last_steps=steps[-last_n:]) + + +def atif_to_trace_like(payload: Dict[str, Any], last_n: int = 12) -> Dict[str, Any]: + """Convert ATIF to a Trace-ish, stable JSON schema.""" + + steps = _get_steps(payload) + steps = steps[-last_n:] if last_n > 0 else steps + + def norm_tool_calls(step: Dict[str, Any]) -> List[Dict[str, Any]]: + tool_calls = step.get("tool_calls") or step.get("toolCalls") + if not isinstance(tool_calls, list): + return [] + out: List[Dict[str, Any]] = [] + for tc in tool_calls: + if not isinstance(tc, dict): + continue + out.append( + { + "name": tc.get("name") or tc.get("tool_name") or tc.get("tool"), + "args": tc.get("args") or tc.get("arguments"), + "id": tc.get("id") or tc.get("call_id"), + } + ) + return out + + events: List[Dict[str, Any]] = [] + for i, step in enumerate(steps): + events.append( + { + "i": i, + "role": step.get("role") or step.get("type") or step.get("speaker"), + "content": step.get("content") + or step.get("message") + or step.get("text"), + "tool_calls": norm_tool_calls(step), + "observation": step.get("observation") or step.get("observation_text"), + "timestamp": step.get("timestamp"), + } + ) + + return { + "format": "trace_bench.atif.v1", + "n_steps": len(_get_steps(payload)), + "events": events, + "final_metrics": payload.get("final_metrics") or payload.get("finalMetrics"), + } + + +def atif_feedback_snippet( + payload: Dict[str, Any], max_chars: int = 4000, last_n: int = 6 +) -> str: + """Generate a compact, human/LLM-readable snippet.""" + + summary = summarize_atif(payload, last_n=last_n) + lines: List[str] = [] + lines.append(f"ATIF: steps={summary.n_steps}") + start_idx = max(0, summary.n_steps - len(summary.last_steps)) + for idx, step in enumerate(summary.last_steps, start=start_idx): + role = step.get("role") or step.get("type") or "?" + content = step.get("content") or step.get("message") or step.get("text") or "" + content = str(content).replace("\n", " ") + if len(content) > 280: + content = content[:280] + "…" + lines.append(f"[{idx}] {role}: {content}") + + tool_calls = step.get("tool_calls") + if isinstance(tool_calls, list) and tool_calls: + for tc in tool_calls[:3]: + if not isinstance(tc, dict): + continue + name = tc.get("name") or tc.get("tool_name") or tc.get("tool") + lines.append(f" tool_call: {name}") + + out = "\n".join(lines) + if len(out) > max_chars: + out = out[: max_chars - 1] + "…" + return out + + +__all__ = [ + "ATIFSummary", + "load_atif", + "summarize_atif", + "atif_to_trace_like", + "atif_feedback_snippet", +] diff --git a/trace_bench/terminal_bench/harbor_runner.py b/trace_bench/terminal_bench/harbor_runner.py new file mode 100644 index 0000000..9700945 --- /dev/null +++ b/trace_bench/terminal_bench/harbor_runner.py @@ -0,0 +1,374 @@ +"""Harbor CLI wrapper for Terminal-Bench evaluation.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +import json +import os +import shutil +import subprocess +import time + + +@dataclass +class HarborEvalResult: + score: float + status: str + job_dir: Path + trial_dir: Optional[Path] + job_result: Dict[str, Any] + trial_result: Dict[str, Any] + trajectory_path: Optional[Path] + stdout: str + stderr: str + cmd: List[str] + + +def harbor_available() -> bool: + return shutil.which("harbor") is not None + + +def default_env_type() -> str: + """Prefer Daytona when an API key is present (useful for Colab).""" + + return "daytona" if os.environ.get("DAYTONA_API_KEY") else "docker" + + +def build_harbor_cmd( + *, + job_name: str, + jobs_dir: Path, + model_name: str, + env_type: str, + task_name: str, + dataset: Optional[str] = None, + task_path: Optional[Path] = None, + agent_name: Optional[str] = None, + agent_import_path: Optional[ + str + ] = "trace_bench.terminal_bench.agent:TraceBenchTerminus2", + agent_kwargs: Optional[Dict[str, Any]] = None, + n_concurrent: int = 1, + quiet: bool = True, + extra_env: Optional[Dict[str, str]] = None, +) -> Tuple[List[str], Dict[str, str]]: + """Return (cmd, env) for `harbor run`. + + Exactly one of (dataset, task_path) must be provided. + """ + + if (dataset is None) == (task_path is None): + raise ValueError("Provide exactly one of dataset or task_path") + + cmd: List[str] = [ + "harbor", + "run", + "--job-name", + job_name, + "--jobs-dir", + str(jobs_dir), + ] + + if task_path is not None: + cmd += ["--path", str(task_path)] + else: + cmd += [ + "--dataset", + str(dataset), + # Current Harbor docs expose this as --task-name / -t. + # Older internal builds may have accepted --include-task-name, but + # using the public CLI name makes this adapter less brittle. + "--task-name", + task_name, + "--n-tasks", + "1", + ] + + if agent_name: + cmd += ["--agent", agent_name] + cmd += ["--model", model_name] + if agent_import_path: + cmd += ["--agent-import-path", agent_import_path] + + for k, v in (agent_kwargs or {}).items(): + cmd += ["--agent-kwarg", f"{k}={v}"] + + cmd += ["--env", env_type] + cmd += ["--n-concurrent", str(int(n_concurrent))] + + if quiet: + cmd += ["--quiet"] + + env = dict(os.environ) + env.update(extra_env or {}) + return cmd, env + + +def _latest_subdir(path: Path) -> Optional[Path]: + subs = [p for p in path.iterdir() if p.is_dir()] + if not subs: + return None + subs.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return subs[0] + + +def _find_job_dir(jobs_dir: Path, job_name: str) -> Path: + direct = jobs_dir / job_name + if direct.exists(): + return direct + + latest = _latest_subdir(jobs_dir) + if latest is None: + raise FileNotFoundError(f"No job directory created under {jobs_dir}") + return latest + + +def _find_trial_dir(job_dir: Path) -> Optional[Path]: + trials = [p for p in job_dir.iterdir() if p.is_dir() and p.name.startswith("trial")] + if not trials: + troot = job_dir / "trials" + if troot.exists() and troot.is_dir(): + trials = [p for p in troot.iterdir() if p.is_dir()] + if not trials: + # Harbor 0.6 writes trial directories as __. + trials = [ + p for p in job_dir.iterdir() if p.is_dir() and (p / "result.json").exists() + ] + if not trials: + return None + trials.sort(key=lambda p: p.stat().st_mtime, reverse=True) + return trials[0] + + +def _read_json(path: Path) -> Dict[str, Any]: + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _run_harbor_subprocess( + cmd: List[str], + *, + env: Dict[str, str], + timeout_seconds: Optional[float], +) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run( + cmd, text=True, capture_output=True, env=env, timeout=timeout_seconds + ) + except subprocess.TimeoutExpired as exc: + stdout = ( + exc.stdout.decode("utf-8", errors="replace") + if isinstance(exc.stdout, bytes) + else (exc.stdout or "") + ) + stderr = ( + exc.stderr.decode("utf-8", errors="replace") + if isinstance(exc.stderr, bytes) + else (exc.stderr or "") + ) + if stderr: + stderr += "\n" + stderr += f"Harbor CLI timed out after {timeout_seconds} seconds" + return subprocess.CompletedProcess( + cmd, returncode=124, stdout=stdout, stderr=stderr + ) + + +def _extract_score(job_result: Dict[str, Any], trial_result: Dict[str, Any]) -> float: + candidates: List[Any] = [] + for d in (trial_result, job_result): + if not isinstance(d, dict): + continue + for key in ( + "score", + "reward", + "resolved", + "success", + "passed", + "verified", + "ok", + ): + if key in d: + candidates.append(d[key]) + summary = d.get("summary") + if isinstance(summary, dict): + for key in ("score", "reward", "resolved", "success"): + if key in summary: + candidates.append(summary[key]) + + for val in candidates: + if isinstance(val, bool): + return 1.0 if val else 0.0 + if isinstance(val, (int, float)): + return float(val) + if isinstance(val, str): + if val.lower() in {"true", "ok", "pass", "passed", "success"}: + return 1.0 + if val.lower() in {"false", "fail", "failed"}: + return 0.0 + try: + return float(val) + except Exception: + continue + + return 0.0 + + +def run_harbor_eval( + *, + job_name: str, + jobs_dir: Path, + model_name: str, + env_type: str, + task_name: str, + dataset: Optional[str] = None, + task_path: Optional[Path] = None, + agent_name: Optional[str] = None, + agent_import_path: Optional[ + str + ] = "trace_bench.terminal_bench.agent:TraceBenchTerminus2", + agent_kwargs: Optional[Dict[str, Any]] = None, + n_concurrent: int = 1, + quiet: bool = True, + timeout_seconds: Optional[float] = None, + extra_env: Optional[Dict[str, str]] = None, +) -> HarborEvalResult: + if not harbor_available(): + raise RuntimeError( + "Harbor CLI not found on PATH. Install Harbor or ensure the `harbor` " + "binary is available before running Terminal-Bench tasks." + ) + + jobs_dir.mkdir(parents=True, exist_ok=True) + + cmd, env = build_harbor_cmd( + job_name=job_name, + jobs_dir=jobs_dir, + model_name=model_name, + env_type=env_type, + task_name=task_name, + dataset=dataset, + task_path=task_path, + agent_name=agent_name, + agent_import_path=agent_import_path, + agent_kwargs=agent_kwargs, + n_concurrent=n_concurrent, + quiet=quiet, + extra_env=extra_env, + ) + + start = time.time() + proc = _run_harbor_subprocess(cmd, env=env, timeout_seconds=timeout_seconds) + + # Compatibility fallback for Harbor 0.6.x builds that still expose dataset + # filtering as --include-task-name instead of --task-name. Keep + # build_harbor_cmd() on the documented/public flag while still allowing + # older CLIs to run real smoke tests. + if ( + proc.returncode != 0 + and dataset is not None + and task_path is None + and "--task-name" in cmd + ): + compat_cmd = list(cmd) + compat_cmd[compat_cmd.index("--task-name")] = "--include-task-name" + proc2 = _run_harbor_subprocess( + compat_cmd, env=env, timeout_seconds=timeout_seconds + ) + if proc2.returncode == 0: + cmd, proc = compat_cmd, proc2 + + if proc.returncode != 0 and agent_import_path and ":" in agent_import_path: + mod, cls = agent_import_path.split(":", 1) + fallback_agent_name = cls if agent_name is None else agent_name + fallback_job_name = f"{job_name}-retry1" + try: + cmd2, env2 = build_harbor_cmd( + job_name=fallback_job_name, + jobs_dir=jobs_dir, + model_name=model_name, + env_type=env_type, + task_name=task_name, + dataset=dataset, + task_path=task_path, + agent_name=fallback_agent_name, + agent_import_path=mod, + agent_kwargs=agent_kwargs, + n_concurrent=n_concurrent, + quiet=quiet, + extra_env=extra_env, + ) + proc2 = _run_harbor_subprocess( + cmd2, env=env2, timeout_seconds=timeout_seconds + ) + if proc2.returncode == 0: + cmd, env, proc, job_name = cmd2, env2, proc2, fallback_job_name + except Exception: + pass + + elapsed = time.time() - start + + try: + job_dir = _find_job_dir(jobs_dir, job_name) + except FileNotFoundError: + job_dir = jobs_dir / job_name + trial_dir = _find_trial_dir(job_dir) if job_dir.exists() else None + + job_result = _read_json(job_dir / "result.json") + trial_result: Dict[str, Any] = {} + trajectory_path: Optional[Path] = None + if trial_dir is not None: + trial_result = _read_json(trial_dir / "result.json") + cand = list(trial_dir.glob("**/trajectory.json")) + trajectory_path = cand[0] if cand else None + + score = _extract_score(job_result, trial_result) + has_trial_exception = bool( + isinstance(trial_result, dict) and trial_result.get("exception_info") + ) + stats = job_result.get("stats") if isinstance(job_result, dict) else None + has_job_errors = bool( + isinstance(stats, dict) and int(stats.get("n_errored_trials") or 0) > 0 + ) + status = ( + "ok" + if proc.returncode == 0 and not has_trial_exception and not has_job_errors + else "failed" + ) + + if status == "failed" and not job_result: + job_result = { + "trace_bench": { + "failure_reason": "harbor_failed_without_job_result", + "returncode": proc.returncode, + } + } + + if isinstance(job_result, dict) and "trace_bench" not in job_result: + job_result["trace_bench"] = {"wall_seconds": elapsed} + + return HarborEvalResult( + score=score, + status=status, + job_dir=job_dir, + trial_dir=trial_dir, + job_result=job_result, + trial_result=trial_result, + trajectory_path=trajectory_path, + stdout=proc.stdout, + stderr=proc.stderr, + cmd=cmd, + ) + + +__all__ = [ + "HarborEvalResult", + "harbor_available", + "default_env_type", + "build_harbor_cmd", + "run_harbor_eval", +] diff --git a/trace_bench/terminal_bench/task.py b/trace_bench/terminal_bench/task.py new file mode 100644 index 0000000..7c15abe --- /dev/null +++ b/trace_bench/terminal_bench/task.py @@ -0,0 +1,259 @@ +"""Terminal-Bench task adapter for Trace-Bench.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, Optional +import json +import os +import shutil + +from trace_bench.terminal_bench import DEFAULT_HARBOR_DATASET +from trace_bench.terminal_bench.atif import ( + atif_feedback_snippet, + atif_to_trace_like, + load_atif, +) +from trace_bench.terminal_bench.harbor_runner import default_env_type, run_harbor_eval + +try: + from opto import trace + from opto.trainer.guide import Guide +except Exception: # pragma: no cover + trace = None # type: ignore + Guide = object # type: ignore + + +_DEFAULT_PROMPT_TEMPLATE = """You are an autonomous terminal agent solving command-line tasks in a sandboxed Linux environment. + +Task Description: +{instruction} + +Current terminal state: +{terminal_state} + +Rules: +- Be precise and avoid unnecessary commands. +- Inspect files before editing. +- Run the task's tests or verifier when appropriate. +- Respond as JSON matching this schema: + +{{ + "analysis": "What you observed and what remains to do.", + "plan": "Your next steps.", + "commands": [ + {{"keystrokes": "ls -la\n", "duration": 0.1}} + ], + "task_complete": false +}} + +Every command must include a trailing newline in `keystrokes`. Use an empty +commands array if you only need to wait or if the task is complete. +""" + + +if trace is not None: + + @trace.model # type: ignore[misc] + class PromptTemplateAgent: + """A trivial model whose output is a trainable prompt template string.""" + + def __init__(self, initial_template: str = _DEFAULT_PROMPT_TEMPLATE): + self.template = trace.node(initial_template, trainable=True) # type: ignore[attr-defined] + + def __call__(self, _task_name: str) -> str: + return self.emit(self.template) + + @trace.bundle(trainable=True) # type: ignore[attr-defined] + def emit(self, template: str) -> str: + return template + +else: # pragma: no cover + + class PromptTemplateAgent: # type: ignore[misc] + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError("opto.trace is required to build Terminal-Bench tasks") + + +class TerminalBenchGuide(Guide): # type: ignore[misc] + """Guide that evaluates prompt templates by running Harbor on a TB task.""" + + def __init__(self, *, eval_kwargs: Optional[Dict[str, Any]] = None): + self.eval_kwargs = eval_kwargs or {} + self._context: Dict[str, Any] = {} + self._eval_count = 0 + + def set_trace_bench_context(self, **ctx: Any) -> None: + self._context.update(ctx) + self._eval_count = 0 + + @staticmethod + def _write_json(path: Path, payload: Dict[str, Any]) -> None: + """Write stable machine-readable JSON. + + Avoid json.dumps(..., default=str): it can silently serialize Python + objects with memory addresses and make artifacts non-reproducible. + """ + from trace_bench.artifacts import sanitize_for_json + + path.write_text( + json.dumps(sanitize_for_json(payload), indent=2, sort_keys=True), + encoding="utf-8", + ) + + def _mode(self) -> str: + return ( + self._context.get("mode") + or os.environ.get("TRACE_BENCH_MODE") + or self.eval_kwargs.get("mode") + or "real" + ).lower() + + def _job_dir(self) -> Optional[Path]: + jd = self._context.get("job_dir") + if jd is None: + return None + return Path(jd) + + def _artifact_dir(self) -> Optional[Path]: + ad = self._context.get("artifacts_dir") + if ad is not None: + return Path(ad) + jd = self._job_dir() + return (jd / "artifacts") if jd else None + + def get_feedback(self, query, response, reference, **_kwargs): # type: ignore[override] + task_name = str(query) + prompt_template = str(response) + + if self._mode() == "stub": + score = 0.0 + feedback = f"STUB: terminal_bench task={task_name} (harbor not invoked)" + return score, feedback + + model_name = self.eval_kwargs.get("harbor_model") or os.environ.get( + "HARBOR_MODEL" + ) + if not model_name: + raise RuntimeError( + "Terminal-Bench real mode requires a Harbor model name. " + "Set eval_kwargs.harbor_model or env var HARBOR_MODEL." + ) + + env_type = self.eval_kwargs.get("harbor_env") or default_env_type() + dataset = self.eval_kwargs.get("harbor_dataset", DEFAULT_HARBOR_DATASET) + task_path = self.eval_kwargs.get("harbor_task_path") + + artifact_dir = self._artifact_dir() + if artifact_dir is None: + artifact_dir = Path.cwd() / "artifacts" + tb_art_dir = artifact_dir / "terminal_bench" + tb_art_dir.mkdir(parents=True, exist_ok=True) + + template_path = tb_art_dir / "prompt_template.txt" + template_path.write_text(prompt_template, encoding="utf-8") + + self._eval_count += 1 + job_name = ( + f"tracebench-{self._context.get('job_id','job')}-eval{self._eval_count:04d}" + ) + jobs_dir = tb_art_dir / "harbor_jobs" + + agent_kwargs = dict(self.eval_kwargs.get("harbor_agent_kwargs") or {}) + agent_kwargs.setdefault("prompt_template_path", str(template_path)) + extra_env = dict(self.eval_kwargs.get("harbor_extra_env") or {}) + timeout_seconds = self.eval_kwargs.get("harbor_cli_timeout_seconds") + + result = run_harbor_eval( + job_name=job_name, + jobs_dir=jobs_dir, + model_name=model_name, + env_type=env_type, + task_name=task_name, + dataset=None if task_path else dataset, + task_path=Path(task_path) if task_path else None, + agent_kwargs=agent_kwargs, + timeout_seconds=timeout_seconds, + extra_env=extra_env, + ) + + (tb_art_dir / f"{job_name}.cmd.txt").write_text( + " ".join(result.cmd), encoding="utf-8" + ) + (tb_art_dir / f"{job_name}.stdout.txt").write_text( + result.stdout or "", encoding="utf-8" + ) + (tb_art_dir / f"{job_name}.stderr.txt").write_text( + result.stderr or "", encoding="utf-8" + ) + self._write_json(tb_art_dir / f"{job_name}.job_result.json", result.job_result) + if result.trial_result: + self._write_json( + tb_art_dir / f"{job_name}.trial_result.json", result.trial_result + ) + + snippet = "" + if result.trajectory_path and result.trajectory_path.exists(): + dst = tb_art_dir / f"{job_name}.trajectory.json" + shutil.copyfile(result.trajectory_path, dst) + atif_payload = load_atif(dst) + trace_like = atif_to_trace_like(atif_payload) + self._write_json( + tb_art_dir / f"{job_name}.trajectory_trace.json", trace_like + ) + snippet = atif_feedback_snippet(atif_payload) + + feedback_parts = [ + f"terminal_bench task={task_name}", + f"score={result.score}", + f"env={env_type}", + ] + if snippet: + feedback_parts.append("\n" + snippet) + feedback = " ".join(feedback_parts) + + return float(result.score), feedback + + +def build_trace_problem(task_slug: str, **override_eval_kwargs: Any) -> Dict[str, Any]: + """Build a Trace problem bundle for a single Terminal-Bench task.""" + + if trace is None: + raise RuntimeError("opto.trace is required to build Terminal-Bench tasks") + + guide = TerminalBenchGuide(eval_kwargs=override_eval_kwargs) + agent = PromptTemplateAgent() + + train_dataset = dict(inputs=[task_slug], infos=[None]) + optimizer_kwargs = dict( + objective=( + "Optimize the Terminus-2 prompt template to solve the given Terminal-Bench task " + "with minimal errors and maximal success." + ), + memory_size=5, + ) + + return dict( + param=agent, + guide=guide, + train_dataset=train_dataset, + optimizer_kwargs=optimizer_kwargs, + metadata=dict(benchmark="terminal_bench", entry=task_slug), + ) + + +def build_terminal_bench_bundle( + task_id: str, eval_kwargs: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + if not task_id.startswith("terminal_bench:"): + raise ValueError(f"Not a terminal_bench task id: {task_id}") + task_slug = task_id.split(":", 1)[1] + return build_trace_problem(task_slug, **(eval_kwargs or {})) + + +__all__ = [ + "PromptTemplateAgent", + "TerminalBenchGuide", + "build_trace_problem", + "build_terminal_bench_bundle", +]