Add Langfuse tracing for LLM call observability

.env.template

@@ -29,6 +29,12 @@ LLM_QUEST_CLAUDE_EXEC_PATH=claude
 LLM_QUEST_CLAUDE_EXEC_MODEL=
 LLM_QUEST_CLAUDE_EXEC_ARGS=
 
+# Langfuse tracing (optional). When set, all LLM calls are traced automatically.
+# Sign up at https://cloud.langfuse.com or self-host.
+LANGFUSE_SECRET_KEY=
+LANGFUSE_PUBLIC_KEY=
+LANGFUSE_BASE_URL=https://cloud.langfuse.com
+
 # Optional token pricing overrides (USD per 1M tokens) used for run cost estimates.
 # If omitted, built-in defaults are used for known models.
 # Format:

llm_quest_benchmark/executors/benchmark.py

@@ -12,6 +12,7 @@
 from llm_quest_benchmark.core.logging import DEFAULT_DB_PATH
 from llm_quest_benchmark.core.runner import run_quest_with_timeout
 from llm_quest_benchmark.environments.state import QuestOutcome
+from llm_quest_benchmark.llm import tracing
 from llm_quest_benchmark.schemas.config import BenchmarkConfig
 
 # Configure logging
@@ -304,6 +305,7 @@ def run_benchmark(config: BenchmarkConfig, progress_callback=None) -> list[dict[
     if artifact_dir:
         logger.info("Benchmark artifacts saved to %s", artifact_dir)
 
+    tracing.flush()
     return results
 
 

llm_quest_benchmark/executors/cli/commands.py

@@ -40,6 +40,7 @@
     print_summary,
     run_benchmark,
 )
+from llm_quest_benchmark.llm import tracing
 from llm_quest_benchmark.renderers.terminal import RichRenderer
 from llm_quest_benchmark.schemas.config import AgentConfig, BenchmarkConfig
 
@@ -433,6 +434,8 @@ def close_callback(event, data):
     except Exception as e:
         log.exception(f"Error during quest run: {e}")
         raise typer.Exit(code=2)
+    finally:
+        tracing.flush()
 
 
 @app.command()

llm_quest_benchmark/llm/client.py

@@ -17,22 +17,35 @@
 
 import anthropic
 from dotenv import load_dotenv
-from openai import OpenAI
 
-from llm_quest_benchmark.constants import (
+# Load .env BEFORE checking Langfuse (needs LANGFUSE_SECRET_KEY from .env).
+load_dotenv(dotenv_path=Path(__file__).resolve().parents[2] / ".env", override=False)
+
+from llm_quest_benchmark.llm.tracing import is_enabled as _langfuse_enabled  # noqa: E402
+
+if _langfuse_enabled():
+    from langfuse import observe  # noqa: E402
+    from langfuse.openai import OpenAI  # noqa: E402
+else:
+    from openai import OpenAI  # noqa: E402
+
+    def observe(**_kwargs):  # type: ignore[misc]
+        def _noop(fn):
+            return fn
+
+        return _noop
+
+
+from llm_quest_benchmark.constants import (  # noqa: E402
     DEFAULT_TEMPERATURE,
     MODEL_ALIASES,
     MODEL_PROVIDER_CONFIG,
 )
-from llm_quest_benchmark.llm.cost import UsageStats, estimate_cost_usd
+from llm_quest_benchmark.llm.cost import UsageStats, estimate_cost_usd  # noqa: E402
 
 logger = logging.getLogger(__name__)
-# Configure httpx logger to only show in debug mode
 logging.getLogger("httpx").setLevel(logging.WARNING)
 
-# Load local .env when running directly from repository root.
-load_dotenv(dotenv_path=Path(__file__).resolve().parents[2] / ".env", override=False)
-
 
 @dataclass(frozen=True)
 class ModelSpec:
@@ -385,6 +398,7 @@ def _get_client(self) -> anthropic.Anthropic:
             self._client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
         return self._client
 
+    @observe(as_type="generation", name="anthropic-completion")
     def get_completion(self, prompt: str) -> str:
         """Get a completion from the model."""
 

llm_quest_benchmark/llm/tracing.py

@@ -0,0 +1,23 @@
+"""Langfuse tracing integration. Activates when LANGFUSE_SECRET_KEY is set."""
+
+import logging
+import os
+from functools import lru_cache
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache(maxsize=1)
+def is_enabled() -> bool:
+    return bool(os.getenv("LANGFUSE_SECRET_KEY"))
+
+
+def flush() -> None:
+    if not is_enabled():
+        return
+    try:
+        from langfuse import Langfuse
+
+        Langfuse().flush()
+    except Exception:
+        logger.debug("Langfuse flush failed", exc_info=True)

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "openai>=1.12.0",
     "openrouter>=0.5.0",
     "typer>=0.9.0",
+    "langfuse>=3.0.0",
 ]
 
 [project.optional-dependencies]