python-sdk/eval_protocol/__init__.py at e428ff869799cec0953c0fb0a0468b477395690d · eval-protocol/python-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""
Fireworks Eval Protocol - Simplify reward modeling and evaluation for LLM RL fine-tuning.

A Python library for defining, testing, deploying, and using reward functions
for LLM fine-tuning, including launching full RL jobs on the Fireworks platform.

The library also provides an agent evaluation framework for testing and evaluating
tool-augmented models using self-contained task bundles.
"""

import warnings

from .auth import get_fireworks_account_id, get_fireworks_api_key
from .common_utils import load_jsonl
from .mcp_env import (
    AnthropicPolicy,
    FireworksPolicy,
    LiteLLMPolicy,
    OpenAIPolicy,
    make,
    rollout,
    test_mcp,
)
from .data_loader import DynamicDataLoader, InlineDataLoader
from . import mcp, rewards
from .models import EvaluateResult, Message, MetricResult, EvaluationRow, InputMetadata, Status
from .playback_policy import PlaybackPolicyBase
from .resources import create_llm_resource
from .reward_function import RewardFunction
from .typed_interface import reward_function
from .quickstart import aha_judge, multi_turn_assistant_to_ground_truth, assistant_to_ground_truth
from .pytest import evaluation_test, SingleTurnRolloutProcessor, RemoteRolloutProcessor
from .pytest.parameterize import DefaultParameterIdGenerator

from .types.remote_rollout_processor import (
    InitRequest,
    RolloutMetadata,
    StatusResponse,
    create_langfuse_config_tags,
)

try:
    from .adapters import OpenAIResponsesAdapter
except ImportError:
    OpenAIResponsesAdapter = None

try:
    from .adapters import LangfuseAdapter, create_langfuse_adapter
except ImportError:
    LangfuseAdapter = None

try:
    from .adapters import BraintrustAdapter, create_braintrust_adapter
except ImportError:
    BraintrustAdapter = None

try:
    from .adapters import LangSmithAdapter
except ImportError:
    LangSmithAdapter = None

warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")

__all__ = [
    "Status",
    "RemoteRolloutProcessor",
    "InputMetadata",
    "EvaluationRow",
    "DefaultParameterIdGenerator",
    "DynamicDataLoader",
    "InlineDataLoader",
    "aha_judge",
    "multi_turn_assistant_to_ground_truth",
    "assistant_to_ground_truth",
    "evaluation_test",
    "SingleTurnRolloutProcessor",
    "OpenAIResponsesAdapter",
    "LangfuseAdapter",
    "create_langfuse_adapter",
    "BraintrustAdapter",
    "create_braintrust_adapter",
    "LangSmithAdapter",
    # Core interfaces
    "Message",
    "MetricResult",
    "EvaluateResult",
    "reward_function",
    "RewardFunction",
    # Authentication
    "get_fireworks_api_key",
    "get_fireworks_account_id",
    # Utilities
    "load_jsonl",
    # MCP Environment API
    "make",
    "rollout",
    "LiteLLMPolicy",
    "AnthropicPolicy",
    "FireworksPolicy",
    "OpenAIPolicy",
    "test_mcp",
    # Playback functionality
    "PlaybackPolicyBase",
    # Resource management
    "create_llm_resource",
    # Submodules
    "rewards",
    "mcp",
    # Remote server types
    "InitRequest",
    "RolloutMetadata",
    "StatusResponse",
    "create_langfuse_config_tags",
]

from . import _version

__version__ = _version.get_versions()["version"]