python-sdk/eval_protocol/__init__.py at 519bbab288fdd89ffd89ec09a84e20f70314923a · eval-protocol/python-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
"""
Fireworks Eval Protocol - Simplify reward modeling and evaluation for LLM RL fine-tuning.

A Python library for defining, testing, deploying, and using reward functions
for LLM fine-tuning, including launching full RL jobs on the Fireworks platform.

The library also provides an agent evaluation framework for testing and evaluating
tool-augmented models using self-contained task bundles.
"""

import warnings

from .auth import get_fireworks_account_id, get_fireworks_api_key
from .common_utils import load_jsonl
from .config import RewardKitConfig, get_config, load_config
from .mcp_env import (
    AnthropicPolicy,
    FireworksPolicy,
    LiteLLMPolicy,
    OpenAIPolicy,
    make,
    rollout,
    test_mcp,
)
from .data_loader import DynamicDataLoader, InlineDataLoader
from . import mcp, rewards
from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata, Status
from .playback_policy import PlaybackPolicyBase
from .resources import create_llm_resource
from .reward_function import RewardFunction
from .typed_interface import reward_function
from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
from .pytest.parameterize import DefaultParameterIdGenerator
from .log_utils.elasticsearch_direct_http_handler import ElasticsearchDirectHttpHandler
from .log_utils.rollout_id_filter import RolloutIdFilter
from .log_utils.util import setup_rollout_logging_for_elasticsearch_handler
from .log_utils.fireworks_tracing_http_handler import FireworksTracingHttpHandler
from .log_utils.elasticsearch_client import ElasticsearchConfig


from .types.remote_rollout_processor import (
    InitRequest,
    RolloutMetadata,
    StatusResponse,
    create_langfuse_config_tags,
    DataLoaderConfig,
)

try:
    from .adapters import OpenAIResponsesAdapter
except ImportError:
    OpenAIResponsesAdapter = None

try:
    from .adapters import LangfuseAdapter, create_langfuse_adapter
except ImportError:
    LangfuseAdapter = None

try:
    from .adapters import BraintrustAdapter, create_braintrust_adapter
except ImportError:
    BraintrustAdapter = None

try:
    from .adapters import LangSmithAdapter
except ImportError:
    LangSmithAdapter = None


try:
    from .adapters import WeaveAdapter
except ImportError:
    WeaveAdapter = None

warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")

__all__ = [
    "ElasticsearchConfig",
    "ElasticsearchDirectHttpHandler",
    "RolloutIdFilter",
    "setup_rollout_logging_for_elasticsearch_handler",
    "DataLoaderConfig",
    "Status",
    "RemoteRolloutProcessor",
    "InputMetadata",
    "EvaluationRow",
    "DefaultParameterIdGenerator",
    "DynamicDataLoader",
    "InlineDataLoader",
    "aha_judge",
    "multi_turn_assistant_to_ground_truth",
    "assistant_to_ground_truth",
    "evaluation_test",
    "SingleTurnRolloutProcessor",
    "OpenAIResponsesAdapter",
    "LangfuseAdapter",
    "create_langfuse_adapter",
    "BraintrustAdapter",
    "create_braintrust_adapter",
    "LangSmithAdapter",
    "FireworksTracingHttpHandler",
    # Core interfaces
    "Message",
    "MetricResult",
    "EvaluateResult",
    "reward_function",
    "RewardFunction",
    # Authentication
    "get_fireworks_api_key",
    "get_fireworks_account_id",
    # Configuration
    "load_config",
    "get_config",
    "RewardKitConfig",
    # Utilities
    "load_jsonl",
    # MCP Environment API
    "make",
    "rollout",
    "LiteLLMPolicy",
    "AnthropicPolicy",
    "FireworksPolicy",
    "OpenAIPolicy",
    "test_mcp",
    # Playback functionality
    "PlaybackPolicyBase",
    # Resource management
    "create_llm_resource",
    # Submodules
    "rewards",
    "mcp",
    # Remote server types
    "InitRequest",
    "RolloutMetadata",
    "StatusResponse",
    "create_langfuse_config_tags",
]

from . import _version

__version__ = _version.get_versions()["version"]