cleanup + add more responses conversations

Author: Dylan Huang

eval_protocol/__init__.py

@@ -32,15 +32,26 @@
     _FIREWORKS_AVAILABLE = False
 # Import submodules to make them available via eval_protocol.rewards, etc.
 from . import mcp, rewards
-from .models import EvaluateResult, Message, MetricResult
+from .models import EvaluateResult, Message, MetricResult, EvaluationRow
 from .playback_policy import PlaybackPolicyBase
 from .resources import create_llm_resource
 from .reward_function import RewardFunction
 from .typed_interface import reward_function
+from .quickstart import aha_judge, split_multi_turn_rows
+from .pytest import evaluation_test, SingleTurnRolloutProcessor
+from .adapters import OpenAIResponsesAdapter, LangfuseAdapter, BraintrustAdapter, LangSmithAdapter
 
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "aha_judge",
+    "split_multi_turn_rows",
+    "evaluation_test",
+    "SingleTurnRolloutProcessor",
+    "OpenAIResponsesAdapter",
+    "LangfuseAdapter",
+    "BraintrustAdapter",
+    "LangSmithAdapter",
     # Core interfaces
     "Message",
     "MetricResult",

eval_protocol/adapters/__init__.py

@@ -73,3 +73,17 @@
     __all__.extend(["create_trl_adapter"])
 except ImportError:
     pass
+
+try:
+    from .openai_responses import OpenAIResponsesAdapter
+
+    __all__.extend(["OpenAIResponsesAdapter"])
+except ImportError:
+    pass
+
+try:
+    from .langsmith import LangSmithAdapter
+
+    __all__.extend(["LangSmithAdapter"])
+except ImportError:
+    pass

eval_protocol/quickstart/llm_judge_braintrust.py

@@ -16,7 +16,8 @@
 adapter = create_braintrust_adapter()
 
 
-@pytest.mark.asyncio
+@pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")  # pyright: ignore[reportAttributeAccessIssue]
+@pytest.mark.asyncio  # pyright: ignore[reportAttributeAccessIssue]
 @evaluation_test(
     input_rows=[
         adapter.get_evaluation_rows(

eval_protocol/quickstart/llm_judge_langfuse.py

@@ -3,6 +3,7 @@
 """
 
 from datetime import datetime
+import os
 
 import pytest
 
@@ -17,7 +18,8 @@
 adapter = create_langfuse_adapter()
 
 
-@pytest.mark.asyncio
+@pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")  # pyright: ignore[reportAttributeAccessIssue]
+@pytest.mark.asyncio  # pyright: ignore[reportAttributeAccessIssue]
 @evaluation_test(
     input_rows=[
         adapter.get_evaluation_rows(

eval_protocol/quickstart/llm_judge_langsmith.py

@@ -56,8 +56,8 @@ def fetch_langsmith_traces_as_evaluation_rows(
         return []
 
 
-@pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")
-@pytest.mark.asyncio
+@pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")  # pyright: ignore[reportAttributeAccessIssue]
+@pytest.mark.asyncio  # pyright: ignore[reportAttributeAccessIssue]
 @evaluation_test(
     input_rows=[fetch_langsmith_traces_as_evaluation_rows()],
     completion_params=[

eval_protocol/quickstart/llm_judge_openai_responses.py

@@ -20,17 +20,22 @@
 
 import pytest
 
-from eval_protocol.models import EvaluationRow
-from eval_protocol.pytest import evaluation_test
-from eval_protocol.pytest.default_single_turn_rollout_process import SingleTurnRolloutProcessor
-from eval_protocol.quickstart import aha_judge, split_multi_turn_rows
-from eval_protocol.adapters.openai_responses import OpenAIResponsesAdapter
+from eval_protocol import (
+    evaluation_test,
+    aha_judge,
+    split_multi_turn_rows,
+    EvaluationRow,
+    SingleTurnRolloutProcessor,
+    OpenAIResponsesAdapter,
+)
 
 adapter = OpenAIResponsesAdapter()
 input_rows = adapter.get_evaluation_rows(
     response_ids=[
         "resp_0e1b7db5d96e92470068c99506443c819e9305e92915d2405f",
-        "resp_05639dcaca074fbc0068c9946593b481908cac70075926d85c",
+        # "resp_05639dcaca074fbc0068c9946593b481908cac70075926d85c",
+        # "resp_0c96a910416e87aa0068c994d0b34c81a3bda0eddf22445aec",
+        # "resp_0efe023280e986f90068c994b85e088190bc8d8263fa603e02",
     ]
 )