From e7095637d5e988c9748df736fcdc7ad6694a54ec Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 25 Aug 2025 20:26:38 -0700 Subject: [PATCH 1/4] Super simple hello world test for Pydantic AI --- eval_protocol/models.py | 14 +- eval_protocol/pytest/__init__.py | 13 + .../default_pydantic_ai_rollout_processor.py | 114 ++++++ pyproject.toml | 4 +- tests/pytest/test_pydantic_agent.py | 27 ++ uv.lock | 334 +++++++++++++++++- 6 files changed, 492 insertions(+), 14 deletions(-) create mode 100644 eval_protocol/pytest/default_pydantic_ai_rollout_processor.py create mode 100644 tests/pytest/test_pydantic_agent.py diff --git a/eval_protocol/models.py b/eval_protocol/models.py index d210d0cc..64812d66 100644 --- a/eval_protocol/models.py +++ b/eval_protocol/models.py @@ -397,15 +397,11 @@ def __iter__(self): CompletionParams = Dict[str, Any] """ -Common set of completion parameters that most model providers support in their -API. Set total=False to allow extra fields since LiteLLM + providers have their -own set of parameters. The following parameters are common fields that are -populated. - -model: str -temperature: Optional[float] -max_tokens: Optional[int] -top_p: Optional[float] +The completion parameters for the respective LLM SDK or agent framework. +Depending on the rollout processor, this might be the parameters passed to +LiteLLM completion call or parameters for the "run" method of the "Agent" class +in Pydantic AI. You can also customize this dictionary to whatever you need if +you implement your own custom rollout processor. """ diff --git a/eval_protocol/pytest/__init__.py b/eval_protocol/pytest/__init__.py index b64d8ef2..6be3093a 100644 --- a/eval_protocol/pytest/__init__.py +++ b/eval_protocol/pytest/__init__.py @@ -8,6 +8,15 @@ from .rollout_processor import RolloutProcessor from .types import RolloutProcessorConfig +# Conditional import for optional dependency +try: + from .default_pydantic_ai_rollout_processor import PydanticAgentRolloutProcessor + + PYDANTIC_AI_AVAILABLE = True +except ImportError: + PYDANTIC_AI_AVAILABLE = False + PydanticAgentRolloutProcessor = None + __all__ = [ "AgentRolloutProcessor", "MCPGymRolloutProcessor", @@ -21,3 +30,7 @@ "BackoffConfig", "get_default_exception_handler_config", ] + +# Only add to __all__ if available +if PYDANTIC_AI_AVAILABLE: + __all__.append("PydanticAgentRolloutProcessor") diff --git a/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py b/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py new file mode 100644 index 00000000..d434b90d --- /dev/null +++ b/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py @@ -0,0 +1,114 @@ +import asyncio +import logging +from typing import List + +from openai.types.chat.chat_completion_assistant_message_param import ChatCompletionAssistantMessageParam + +from eval_protocol.models import EvaluationRow, Message +from eval_protocol.pytest.rollout_processor import RolloutProcessor +from eval_protocol.pytest.types import RolloutProcessorConfig +from openai.types.chat import ChatCompletion, ChatCompletionMessageParam +from openai.types.chat.chat_completion import Choice as ChatCompletionChoice + +from pydantic_ai.models.openai import OpenAIModel +from pydantic import TypeAdapter +from pydantic_ai.messages import ModelMessage +from pydantic_ai._utils import generate_tool_call_id +from pydantic_ai import Agent +from pydantic_ai.messages import ( + ModelRequest, + SystemPromptPart, + ToolReturnPart, + UserPromptPart, +) + +logger = logging.getLogger(__name__) + + +class PydanticAgentRolloutProcessor(RolloutProcessor): + """Rollout processor for Pydantic AI agents. Mainly converts + EvaluationRow.messages to and from Pydantic AI ModelMessage format.""" + + def __init__(self): + # dummy model used for its helper functions for processing messages + self.util = OpenAIModel("dummy-model") + + def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]: + """Create agent rollout tasks and return them for external handling.""" + + max_concurrent = getattr(config, "max_concurrent_rollouts", 8) or 8 + semaphore = asyncio.Semaphore(max_concurrent) + + # validate that the "agent" field is present with a valid Pydantic AI Agent instance in the completion_params dict + if "agent" not in config.kwargs: + raise ValueError("kwargs must contain an 'agent' field with a valid Pydantic AI Agent instance") + if not isinstance(config.kwargs["agent"], Agent): + raise ValueError("kwargs['agent'] must be a valid Pydantic AI Agent instance") + + agent: Agent = config.kwargs["agent"] + + model = OpenAIModel( + config.completion_params["model"], + provider=config.completion_params["provider"], + ) + + async def process_row(row: EvaluationRow) -> EvaluationRow: + """Process a single row with agent rollout.""" + model_messages = [self.convert_ep_message_to_pyd_message(m, row) for m in row.messages] + response = await agent.run(message_history=model_messages, model=model) + row.messages = await self.convert_pyd_message_to_ep_message(response.all_messages()) + return row + + async def _sem_wrapper(r: EvaluationRow) -> EvaluationRow: + async with semaphore: + result = await process_row(r) + return result + + # Create and return tasks for external handling + tasks = [asyncio.create_task(_sem_wrapper(row)) for row in rows] + return tasks + + async def convert_pyd_message_to_ep_message(self, messages: list[ModelMessage]) -> list[Message]: + oai_messages: list[ChatCompletionMessageParam] = await self.util._map_messages(messages) + return [Message(**m) for m in oai_messages] + + def convert_ep_message_to_pyd_message(self, message: Message, row: EvaluationRow) -> ModelMessage: + if message.role == "assistant": + type_adapter = TypeAdapter(ChatCompletionAssistantMessageParam) + oai_message = type_adapter.validate_python(message) + # Fix: Provide required finish_reason and index, and ensure created is int (timestamp) + return self.util._process_response( + ChatCompletion( + choices=[ChatCompletionChoice(message=oai_message, finish_reason="stop", index=0)], + object="chat.completion", + model="", + id="", + created=( + int(row.created_at.timestamp()) + if hasattr(row.created_at, "timestamp") + else int(row.created_at) + ), + ) + ) + elif message.role == "user": + if isinstance(message.content, str): + return ModelRequest(parts=[UserPromptPart(content=message.content)]) + elif isinstance(message.content, list): + return ModelRequest(parts=[UserPromptPart(content=message.content[0].text)]) + elif message.role == "system": + if isinstance(message.content, str): + return ModelRequest(parts=[SystemPromptPart(content=message.content)]) + elif isinstance(message.content, list): + return ModelRequest(parts=[SystemPromptPart(content=message.content[0].text)]) + elif message.role == "tool": + return ModelRequest( + parts=[ + ToolReturnPart( + content=message.content, + tool_name="", + tool_call_id=message.tool_call_id or generate_tool_call_id(), + ) + ] + ) + else: + raise ValueError(f"Unknown role: {message.role}") diff --git a/pyproject.toml b/pyproject.toml index 739d172f..ec92323b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,9 @@ bigquery = [ svgbench = [ "selenium>=4.0.0", ] +pydantic = [ + "pydantic-ai", +] [tool.pytest.ini_options] addopts = "-q" @@ -170,7 +173,6 @@ dev = [ "haikus==0.3.8", "pytest>=8.4.1", ] - [tool.ruff] line-length = 119 target-version = "py310" diff --git a/tests/pytest/test_pydantic_agent.py b/tests/pytest/test_pydantic_agent.py new file mode 100644 index 00000000..bea37b6e --- /dev/null +++ b/tests/pytest/test_pydantic_agent.py @@ -0,0 +1,27 @@ +import os +import pytest + +from eval_protocol.models import EvaluationRow, Message +from eval_protocol.pytest import evaluation_test +from pydantic_ai import Agent + +from eval_protocol.pytest.default_pydantic_ai_rollout_processor import PydanticAgentRolloutProcessor + +agent = Agent() + + +@pytest.mark.asyncio +@evaluation_test( + input_messages=[Message(role="user", content="Hello, how are you?")], + completion_params=[ + {"model": "accounts/fireworks/models/gpt-oss-120b", "provider": "fireworks"}, + ], + rollout_processor=PydanticAgentRolloutProcessor(), + rollout_processor_kwargs={"agent": agent}, + mode="pointwise", +) +async def test_pydantic_agent(row: EvaluationRow) -> EvaluationRow: + """ + Super simple hello world test for Pydantic AI. + """ + return row diff --git a/uv.lock b/uv.lock index c4cb15b6..6d1b3e4d 100644 --- a/uv.lock +++ b/uv.lock @@ -232,6 +232,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] +[[package]] +name = "argcomplete" +version = "3.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/0f/861e168fc813c56a78b35f3c30d91c6757d1fd185af1110f1aec784b35d0/argcomplete-3.6.2.tar.gz", hash = "sha256:d0519b1bc867f5f4f4713c41ad0aba73a4a5f007449716b16f385f2166dc6adf", size = 73403, upload-time = "2025-04-03T04:57:03.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/da/e42d7a9d8dd33fa775f467e4028a47936da2f01e4b0e561f9ba0d74cb0ca/argcomplete-3.6.2-py3-none-any.whl", hash = "sha256:65b3133a29ad53fb42c48cf5114752c7ab66c1c38544fdf6460f450c09b42591", size = 43708, upload-time = "2025-04-03T04:57:01.591Z" }, +] + [[package]] name = "argon2-cffi" version = "25.1.0" @@ -424,6 +433,34 @@ css = [ { name = "tinycss2" }, ] +[[package]] +name = "boto3" +version = "1.40.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/b1/0f91a07dc87485a791f2d31df1c2dc75dc5d302fbf002e5446ffa04e8e32/boto3-1.40.17.tar.gz", hash = "sha256:e115dc87d5975d32dfa0ebaf19c39e360665317a350004fa94b03200fe853f2e", size = 112054, upload-time = "2025-08-25T19:37:21.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/b5/1973ed8c107beb2fe5a510d5609603537a4e18a920ead021bed5699c5437/boto3-1.40.17-py3-none-any.whl", hash = "sha256:2cacecd689cb51d81fbf54f84b64d0e6e922fbc18ee513c568b9f61caf4221e0", size = 140076, upload-time = "2025-08-25T19:37:20.087Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/87/92b520e7f4ac2a64c6e68bedc1dc3c1335a5aa1efb89a4247b45e34e40ac/botocore-1.40.17.tar.gz", hash = "sha256:769cd04a6a612f2d48b5f456c676fd81733fab682870952f7e2887260ea6a2bc", size = 14375996, upload-time = "2025-08-25T19:37:11.692Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/fd/bcc913de90d6f0549a04ee40caaf7d4e76efb5a622fc6cc47379f686e50f/botocore-1.40.17-py3-none-any.whl", hash = "sha256:603951935c1a741ae70236bf15725c5293074f28503e7029ad0e24ece476a342", size = 14035937, upload-time = "2025-08-25T19:37:06.75Z" }, +] + [[package]] name = "box2d-py" version = "2.3.5" @@ -704,6 +741,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/e8/64c37fadfc2816a7701fa8a6ed8d87327c7d54eacfbfb6edab14a2f2be75/cloudpickle-3.1.1-py3-none-any.whl", hash = "sha256:c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e", size = 20992, upload-time = "2025-01-14T17:02:02.417Z" }, ] +[[package]] +name = "cohere" +version = "5.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastavro" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "requests" }, + { name = "tokenizers" }, + { name = "types-requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/ea/0b4bfb4b7f0f445db97acc979308f80ed5ab31df3786b1951d6e48b30d27/cohere-5.17.0.tar.gz", hash = "sha256:70d2fb7bccf8c9de77b07e1c0b3d93accf6346242e3cdc6ce293b577afa74a63", size = 164665, upload-time = "2025-08-13T06:58:00.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/21/d0eb7c8e5b3bb748190c59819928c38cafcdf8f8aaca9d21074c64cf1cae/cohere-5.17.0-py3-none-any.whl", hash = "sha256:fe7d8228cda5335a7db79a828893765a4d5a40b7f7a43443736f339dc7813fa4", size = 295301, upload-time = "2025-08-13T06:57:59.072Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -1171,6 +1228,9 @@ langfuse = [ openevals = [ { name = "openevals" }, ] +pydantic = [ + { name = "pydantic-ai" }, +] svgbench = [ { name = "selenium" }, ] @@ -1189,6 +1249,9 @@ dev = [ { name = "haikus" }, { name = "pytest" }, ] +pydantic = [ + { name = "pydantic-ai" }, +] [package.metadata] requires-dist = [ @@ -1238,6 +1301,7 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'" }, { name = "psutil", specifier = ">=5.8.0" }, { name = "pydantic", specifier = ">=2.0.0" }, + { name = "pydantic-ai", marker = "extra == 'pydantic'" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.365" }, { name = "pytest", specifier = ">=6.0.0" }, { name = "pytest-asyncio", specifier = ">=0.21.0" }, @@ -1268,7 +1332,7 @@ requires-dist = [ { name = "websockets", specifier = ">=15.0.1" }, { name = "werkzeug", marker = "extra == 'dev'", specifier = ">=2.0.0" }, ] -provides-extras = ["dev", "trl", "openevals", "fireworks", "box2d", "langfuse", "huggingface", "adapters", "bigquery", "svgbench"] +provides-extras = ["dev", "trl", "openevals", "fireworks", "box2d", "langfuse", "huggingface", "adapters", "bigquery", "svgbench", "pydantic"] [package.metadata.requires-dev] dev = [ @@ -1277,6 +1341,16 @@ dev = [ { name = "haikus", specifier = "==0.3.8" }, { name = "pytest", specifier = ">=8.4.1" }, ] +pydantic = [{ name = "pydantic-ai", specifier = ">=0.4.3" }] + +[[package]] +name = "eval-type-backport" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/ea/8b0ac4469d4c347c6a385ff09dc3c048c2d021696664e26c7ee6791631b5/eval_type_backport-0.2.2.tar.gz", hash = "sha256:f0576b4cf01ebb5bd358d02314d31846af5e07678387486e2c798af0e7d849c1", size = 9079, upload-time = "2024-12-21T20:09:46.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/31/55cd413eaccd39125368be33c46de24a1f639f2e12349b0361b4678f3915/eval_type_backport-0.2.2-py3-none-any.whl", hash = "sha256:cb6ad7c393517f476f96d456d0412ea80f0a8cf96f6892834cd9340149111b0a", size = 5830, upload-time = "2024-12-21T20:09:44.175Z" }, +] [[package]] name = "exceptiongroup" @@ -1379,6 +1453,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/a6/5aa862489a2918a096166fd98d9fe86b7fd53c607678b3fa9d8c432d88d5/fastapi_cloud_cli-0.1.5-py3-none-any.whl", hash = "sha256:d80525fb9c0e8af122370891f9fa83cf5d496e4ad47a8dd26c0496a6c85a012a", size = 18992, upload-time = "2025-07-28T13:30:47.427Z" }, ] +[[package]] +name = "fastavro" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/ec/762dcf213e5b97ea1733b27d5a2798599a1fa51565b70a93690246029f84/fastavro-1.12.0.tar.gz", hash = "sha256:a67a87be149825d74006b57e52be068dfa24f3bfc6382543ec92cd72327fe152", size = 1025604, upload-time = "2025-07-31T15:16:42.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/22/60eff8fb290dc6cea71448b97839e8e8f44d3dcae95366f34deed74f9fc3/fastavro-1.12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e38497bd24136aad2c47376ee958be4f5b775d6f03c11893fc636eea8c1c3b40", size = 948880, upload-time = "2025-07-31T15:16:46.014Z" }, + { url = "https://files.pythonhosted.org/packages/30/b1/e0653699d2a085be8b7ddeeff84e9e110ea776555052f99e85a5f9f39bd3/fastavro-1.12.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8d8401b021f4b3dfc05e6f82365f14de8d170a041fbe3345f992c9c13d4f0ff", size = 3226993, upload-time = "2025-07-31T15:16:48.309Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0c/9d27972025a54e424e1c449f015251a65b658b23b0a4715e8cf96bd4005a/fastavro-1.12.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531b89117422db967d4e1547b34089454e942341e50331fa71920e9d5e326330", size = 3240363, upload-time = "2025-07-31T15:16:50.481Z" }, + { url = "https://files.pythonhosted.org/packages/23/c8/41d0bc7dbd5de93a75b277a4cc378cb84740a083b3b33de5ec51e7a69d5e/fastavro-1.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae541edbc6091b890532d3e50d7bcdd324219730598cf9cb4522d1decabde37e", size = 3165740, upload-time = "2025-07-31T15:16:52.79Z" }, + { url = "https://files.pythonhosted.org/packages/52/81/b317b33b838dd4db8753349fd3ac4a92f7a2c4217ce55e6db397fff22481/fastavro-1.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:585a11f612eaadb0dcb1d3d348b90bd0d0d3ee4cf9abafd8b319663e8a0e1dcc", size = 3245059, upload-time = "2025-07-31T15:16:55.151Z" }, + { url = "https://files.pythonhosted.org/packages/62/f3/9df53cc1dad3873279246bb9e3996130d8dd2affbc0537a5554a01a28f84/fastavro-1.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:425fb96fbfbc06a0cc828946dd2ae9d85a5f9ff836af033d8cb963876ecb158e", size = 450639, upload-time = "2025-07-31T15:16:56.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/51/6bd93f2c9f3bb98f84ee0ddb436eb46a308ec53e884d606b70ca9d6b132d/fastavro-1.12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:56f78d1d527bea4833945c3a8c716969ebd133c5762e2e34f64c795bd5a10b3e", size = 962215, upload-time = "2025-07-31T15:16:58.173Z" }, + { url = "https://files.pythonhosted.org/packages/32/37/3e2e429cefe03d1fa98cc4c4edae1d133dc895db64dabe84c17b4dc0921c/fastavro-1.12.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7ce0d117642bb4265ef6e1619ec2d93e942a98f60636e3c0fbf1eb438c49026", size = 3412716, upload-time = "2025-07-31T15:17:00.301Z" }, + { url = "https://files.pythonhosted.org/packages/33/28/eb37d9738ea3649bdcab1b6d4fd0facf9c36261623ea368554734d5d6821/fastavro-1.12.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52e9d9648aad4cca5751bcbe2d3f98e85afb0ec6c6565707f4e2f647ba83ba85", size = 3439283, upload-time = "2025-07-31T15:17:02.505Z" }, + { url = "https://files.pythonhosted.org/packages/57/6f/7aba4efbf73fd80ca20aa1db560936c222dd1b4e5cadbf9304361b9065e3/fastavro-1.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6183875381ec1cf85a1891bf46696fd1ec2ad732980e7bccc1e52e9904e7664d", size = 3354728, upload-time = "2025-07-31T15:17:04.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2d/b0d8539f4622ebf5355b7898ac7930b1ff638de85b6c3acdd0718e05d09e/fastavro-1.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5ad00a2b94d3c8bf9239acf92d56e3e457e1d188687a8d80f31e858ccf91a6d6", size = 3442598, upload-time = "2025-07-31T15:17:06.986Z" }, + { url = "https://files.pythonhosted.org/packages/fe/33/882154b17e0fd468f1a5ae8cc903805531e1fcb699140315366c5f8ec20d/fastavro-1.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:6c4d1c276ff1410f3830648bb43312894ad65709ca0cb54361e28954387a46ac", size = 451836, upload-time = "2025-07-31T15:17:08.219Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f0/df076a541144d2f351820f3d9e20afa0e4250e6e63cb5a26f94688ed508c/fastavro-1.12.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e849c70198e5bdf6f08df54a68db36ff72bd73e8f14b1fd664323df073c496d8", size = 944288, upload-time = "2025-07-31T15:17:09.756Z" }, + { url = "https://files.pythonhosted.org/packages/52/1d/5c1ea0f6e98a441953de822c7455c9ce8c3afdc7b359dd23c5a5e5039249/fastavro-1.12.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b260e1cdc9a77853a2586b32208302c08dddfb5c20720b5179ac5330e06ce698", size = 3404895, upload-time = "2025-07-31T15:17:11.939Z" }, + { url = "https://files.pythonhosted.org/packages/36/8b/115a3ffe67fb48de0de704284fa5e793afa70932b8b2e915cc7545752f05/fastavro-1.12.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:181779688d8b80957953031f0d82ec0761be667a78e03dac642511ff996c771a", size = 3469935, upload-time = "2025-07-31T15:17:14.145Z" }, + { url = "https://files.pythonhosted.org/packages/14/f8/bf3b7370687ab21205e07b37acdd2455ca69f5d25c72d2b315faf357b1cd/fastavro-1.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6881caf914b36a57d1f90810f04a89bd9c837dd4a48e1b66a8b92136e85c415d", size = 3306148, upload-time = "2025-07-31T15:17:16.121Z" }, + { url = "https://files.pythonhosted.org/packages/97/55/fba2726b59a984c7aa2fc19c6e8ef1865eca6a3f66e78810d602ca22af59/fastavro-1.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8bf638248499eb78c422f12fedc08f9b90b5646c3368415e388691db60e7defb", size = 3442851, upload-time = "2025-07-31T15:17:18.738Z" }, + { url = "https://files.pythonhosted.org/packages/a6/3e/25059b8fe0b8084fd858dca77caf0815d73e0ca4731485f34402e8d40c43/fastavro-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ed4f18b7c2f651a5ee2233676f62aac332995086768301aa2c1741859d70b53e", size = 445449, upload-time = "2025-07-31T15:17:20.438Z" }, + { url = "https://files.pythonhosted.org/packages/db/c7/f18b73b39860d54eb724f881b8932882ba10c1d4905e491cd25d159a7e49/fastavro-1.12.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dbe2b690d9caba7d888126cc1dd980a8fcf5ee73de41a104e3f15bb5e08c19c8", size = 936220, upload-time = "2025-07-31T15:17:21.994Z" }, + { url = "https://files.pythonhosted.org/packages/20/22/61ec800fda2a0f051a21b067e4005fd272070132d0a0566c5094e09b666c/fastavro-1.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:07ff9e6c6e8739203ccced3205646fdac6141c2efc83f4dffabf5f7d0176646d", size = 3348450, upload-time = "2025-07-31T15:17:24.186Z" }, + { url = "https://files.pythonhosted.org/packages/ca/79/1f34618fb643b99e08853e8a204441ec11a24d3e1fce050e804e6ff5c5ae/fastavro-1.12.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a172655add31882cab4e1a96b7d49f419906b465b4c2165081db7b1db79852f", size = 3417238, upload-time = "2025-07-31T15:17:26.531Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0b/79611769eb15cc17992dc3699141feb0f75afd37b0cb964b4a08be45214e/fastavro-1.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:be20ce0331b70b35dca1a4c7808afeedf348dc517bd41602ed8fc9a1ac2247a9", size = 3252425, upload-time = "2025-07-31T15:17:28.989Z" }, + { url = "https://files.pythonhosted.org/packages/86/1a/65e0999bcc4bbb38df32706b6ae6ce626d528228667a5e0af059a8b25bb2/fastavro-1.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a52906681384a18b99b47e5f9eab64b4744d6e6bc91056b7e28641c7b3c59d2b", size = 3385322, upload-time = "2025-07-31T15:17:31.232Z" }, + { url = "https://files.pythonhosted.org/packages/e9/49/c06ebc9e5144f7463c2bfcb900ca01f87db934caf131bccbffc5d0aaf7ec/fastavro-1.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:cf153531191bcfc445c21e05dd97232a634463aa717cf99fb2214a51b9886bff", size = 445586, upload-time = "2025-07-31T15:17:32.634Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c8/46ab37076dc0f86bb255791baf9b3c3a20f77603a86a40687edacff8c03d/fastavro-1.12.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:1928e88a760688e490118e1bedf0643b1f3727e5ba59c07ac64638dab81ae2a1", size = 1025933, upload-time = "2025-07-31T15:17:34.321Z" }, + { url = "https://files.pythonhosted.org/packages/a9/7f/cb3e069dcc903034a6fe82182d92c75d981d86aee94bd028200a083696b3/fastavro-1.12.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cd51b706a3ab3fe4af84a0b37f60d1bcd79295df18932494fc9f49db4ba2bab2", size = 3560435, upload-time = "2025-07-31T15:17:36.314Z" }, + { url = "https://files.pythonhosted.org/packages/d0/12/9478c28a2ac4fcc10ad9488dd3dcd5fac1ef550c3022c57840330e7cec4b/fastavro-1.12.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1148263931f6965e1942cf670f146148ca95b021ae7b7e1f98bf179f1c26cc58", size = 3453000, upload-time = "2025-07-31T15:17:38.875Z" }, + { url = "https://files.pythonhosted.org/packages/00/32/a5c8b3af9561c308c8c27da0be998b6237a47dbbdd8d5499f02731bd4073/fastavro-1.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4099e0f6fb8a55f59891c0aed6bfa90c4d20a774737e5282c74181b4703ea0cb", size = 3383233, upload-time = "2025-07-31T15:17:40.833Z" }, + { url = "https://files.pythonhosted.org/packages/42/a0/f6290f3f8059543faf3ef30efbbe9bf3e4389df881891136cd5fb1066b64/fastavro-1.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:10c586e9e3bab34307f8e3227a2988b6e8ac49bff8f7b56635cf4928a153f464", size = 3402032, upload-time = "2025-07-31T15:17:42.958Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.1" @@ -1664,6 +1775,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/3c/2a19a60a473de48717b4efb19398c3f914795b64a96cf3fbe82588044f78/google_crc32c-1.7.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6efb97eb4369d52593ad6f75e7e10d053cf00c48983f7a973105bc70b0ac4d82", size = 28048, upload-time = "2025-03-26T14:41:46.696Z" }, ] +[[package]] +name = "google-genai" +version = "1.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "google-auth" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/1b/da30fa6e2966942d7028a58eb7aa7d04544dcc3aa66194365b2e0adac570/google_genai-1.31.0.tar.gz", hash = "sha256:8572b47aa684357c3e5e10d290ec772c65414114939e3ad2955203e27cd2fcbc", size = 233482, upload-time = "2025-08-18T23:40:21.733Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/27/1525bc9cbec58660f0842ebcbfe910a1dde908c2672373804879666e0bb8/google_genai-1.31.0-py3-none-any.whl", hash = "sha256:5c6959bcf862714e8ed0922db3aaf41885bacf6318751b3421bf1e459f78892f", size = 231876, upload-time = "2025-08-18T23:40:20.385Z" }, +] + [[package]] name = "google-resumable-media" version = "2.7.2" @@ -1739,6 +1869,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/4f/aab73ecaa6b3086a4c89863d94cf26fa84cbff63f52ce9bc4342b3087a06/greenlet-3.2.3-cp314-cp314-win_amd64.whl", hash = "sha256:8c47aae8fbbfcf82cc13327ae802ba13c9c36753b67e760023fd116bc124a62a", size = 301236, upload-time = "2025-06-05T16:15:20.111Z" }, ] +[[package]] +name = "griffe" +version = "1.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/ca/29f36e00c74844ae50d139cf5a8b1751887b2f4d5023af65d460268ad7aa/griffe-1.12.1.tar.gz", hash = "sha256:29f5a6114c0aeda7d9c86a570f736883f8a2c5b38b57323d56b3d1c000565567", size = 411863, upload-time = "2025-08-14T21:08:15.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/f2/4fab6c3e5bcaf38a44cc8a974d2752eaad4c129e45d6533d926a30edd133/griffe-1.12.1-py3-none-any.whl", hash = "sha256:2d7c12334de00089c31905424a00abcfd931b45b8b516967f224133903d302cc", size = 138940, upload-time = "2025-08-14T21:08:13.382Z" }, +] + +[[package]] +name = "groq" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/a2/77fd1460e7d55859219223719aa44ae8902a3a1ad333cd5faf330eb0b894/groq-0.31.0.tar.gz", hash = "sha256:182252e9bf0d696df607c137cbafa851d2c84aaf94bcfe9165c0bc231043490c", size = 136237, upload-time = "2025-08-05T23:14:01.183Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/f8/14672d69a91495f43462c5490067eeafc30346e81bda1a62848e897f9bc3/groq-0.31.0-py3-none-any.whl", hash = "sha256:5e3c7ec9728b7cccf913da982a9b5ebb46dc18a070b35e12a3d6a1e12d6b0f7f", size = 131365, upload-time = "2025-08-05T23:13:59.768Z" }, +] + [[package]] name = "grpcio" version = "1.74.0" @@ -1961,11 +2120,11 @@ wheels = [ [[package]] name = "httpx-sse" -version = "0.4.1" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/fa/66bd985dd0b7c109a3bcb89272ee0bfb7e2b4d06309ad7b38ff866734b2a/httpx_sse-0.4.1.tar.gz", hash = "sha256:8f44d34414bc7b21bf3602713005c5df4917884f76072479b21f68befa4ea26e", size = 12998, upload-time = "2025-06-24T13:21:05.71Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/60/8f4281fa9bbf3c8034fd54c0e7412e66edbab6bc74c4996bd616f8d0406e/httpx-sse-0.4.0.tar.gz", hash = "sha256:1e81a3a3070ce322add1d3529ed42eb5f70817f45ed6ec915ab753f961139721", size = 12624, upload-time = "2023-12-22T08:01:21.083Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/25/0a/6269e3473b09aed2dab8aa1a600c70f31f00ae1349bee30658f7e358a159/httpx_sse-0.4.1-py3-none-any.whl", hash = "sha256:cba42174344c3a5b06f255ce65b350880f962d99ead85e776f23c6618a377a37", size = 8054, upload-time = "2025-06-24T13:21:04.772Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9b/a181f281f65d776426002f330c31849b86b31fc9d848db62e16f03ff739f/httpx_sse-0.4.0-py3-none-any.whl", hash = "sha256:f329af6eae57eaa2bdfd962b42524764af68075ea87370a2de920af5341e318f", size = 7819, upload-time = "2023-12-22T08:01:19.89Z" }, ] [[package]] @@ -2002,6 +2161,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/d5/d9e9b75d8dc9cf125fff16fb0cd51d864a29e8b46b6880d8808940989405/huggingface_hub-0.33.5-py3-none-any.whl", hash = "sha256:29b4e64982c2064006021af297e1b17d44c85a8aaf90a0d7efeff7e7d2426296", size = 515705, upload-time = "2025-07-24T12:30:29.55Z" }, ] +[package.optional-dependencies] +inference = [ + { name = "aiohttp" }, +] + [[package]] name = "hydra-core" version = "1.3.2" @@ -2076,6 +2240,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "invoke" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/42/127e6d792884ab860defc3f4d80a8f9812e48ace584ffc5a346de58cdc6c/invoke-2.2.0.tar.gz", hash = "sha256:ee6cbb101af1a859c7fe84f2a264c059020b0cb7fe3535f9424300ab568f6bd5", size = 299835, upload-time = "2023-07-12T18:05:17.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/66/7f8c48009c72d73bc6bbe6eb87ac838d6a526146f7dab14af671121eb379/invoke-2.2.0-py3-none-any.whl", hash = "sha256:6ea924cc53d4f78e3d98bc436b08069a03077e6f85ad1ddaa8a116d7dad15820", size = 160274, upload-time = "2023-07-12T18:05:16.294Z" }, +] + [[package]] name = "ipykernel" version = "6.30.0" @@ -2341,6 +2514,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" }, ] +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + [[package]] name = "json5" version = "0.12.0" @@ -2763,6 +2945,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/e4/f1546746049c99c6b8b247e2f34485b9eae36faa9322b84e2a17262e6712/litellm-1.74.9-py3-none-any.whl", hash = "sha256:ab8f8a6e4d8689d3c7c4f9c3bbc7e46212cc3ebc74ddd0f3c0c921bb459c9874", size = 8740449, upload-time = "2025-07-28T16:42:36.8Z" }, ] +[[package]] +name = "logfire-api" +version = "4.3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/9a/cf448c5e1a437771a1cde61da39a8eba82ac2022689923f8430cf5bb72f2/logfire_api-4.3.5.tar.gz", hash = "sha256:f4c1aae454ba248000d65bfe7d60439094056813bd97a48725706417330e222e", size = 52881, upload-time = "2025-08-22T16:37:17.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/16/42bfeeb9126a5c452c890b18b7ee4887894494c770ad60ae0718e62bc07b/logfire_api-4.3.5-py3-none-any.whl", hash = "sha256:3b6beb18505730c343b35a5132dfbf842f52094308d23bf7edd75b7f511bb4c8", size = 88449, upload-time = "2025-08-22T16:37:13.636Z" }, +] + [[package]] name = "loguru" version = "0.7.3" @@ -2901,6 +3092,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mistralai" +version = "1.9.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "eval-type-backport" }, + { name = "httpx" }, + { name = "invoke" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/6f/12296d29c480a4101e2bc092347895ce1a8047b6bbc52f97f124177df0b4/mistralai-1.9.8.tar.gz", hash = "sha256:74eac8b3aee410dffbd8ef0878adb7f593940fa9592d9eb4428da9b067209b22", size = 204432, upload-time = "2025-08-25T16:30:31.354Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/b3/3c1d449eea89153a77e7093e90e0282d1a718865ae6787c379256b1db288/mistralai-1.9.8-py3-none-any.whl", hash = "sha256:f4874d62932245c438c4fce04aaf740a9d6da651dc598bf660978a21fb73f017", size = 439113, upload-time = "2025-08-25T16:30:29.855Z" }, +] + [[package]] name = "mistune" version = "3.1.3" @@ -4438,6 +4647,78 @@ email = [ { name = "email-validator" }, ] +[[package]] +name = "pydantic-ai" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic-ai-slim", extra = ["anthropic", "bedrock", "cli", "cohere", "evals", "google", "groq", "huggingface", "mcp", "mistral", "openai", "vertexai"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/45/7a4b717a9c77a1d7d1c406c5acac4c7f0400def99c2c57d5dffa7bcafbe1/pydantic_ai-0.4.3.tar.gz", hash = "sha256:c469145af1cd4cf0ad031bfe703b86dd38beee7c2d399482f8b12f738a8cd2b0", size = 41367498, upload-time = "2025-07-16T16:37:25.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/77/624351aa51527271cb522bf9f0908c8b10f79f7d2cb1da94b6ca15c26216/pydantic_ai-0.4.3-py3-none-any.whl", hash = "sha256:b2700a730bd42a6f295570789b0a33f456b229a119f612b10da23171a4fff8c2", size = 10144, upload-time = "2025-07-16T16:37:16.884Z" }, +] + +[[package]] +name = "pydantic-ai-slim" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "eval-type-backport" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "griffe" }, + { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pydantic-graph" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/3d/912eee9931ffdf32a3f8999b8499e3df2281c53cab7bf91c60494900a988/pydantic_ai_slim-0.4.3.tar.gz", hash = "sha256:a09cd99acb6246552bfefd59c3413f9741e7b4c0a08f52c0822f3f46be6303bd", size = 169558, upload-time = "2025-07-16T16:37:29.964Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/ea/d9eb75d0acc23ae97a243c4172293173effa3afe55c949690d1a3dd46624/pydantic_ai_slim-0.4.3-py3-none-any.whl", hash = "sha256:1a8930473afc8259a93ae68bb8c34fb528c8c27266b771d93bbbc31ab2fd3404", size = 226233, upload-time = "2025-07-16T16:37:19.554Z" }, +] + +[package.optional-dependencies] +anthropic = [ + { name = "anthropic" }, +] +bedrock = [ + { name = "boto3" }, +] +cli = [ + { name = "argcomplete" }, + { name = "prompt-toolkit" }, + { name = "rich" }, +] +cohere = [ + { name = "cohere", marker = "sys_platform != 'emscripten'" }, +] +evals = [ + { name = "pydantic-evals" }, +] +google = [ + { name = "google-genai" }, +] +groq = [ + { name = "groq" }, +] +huggingface = [ + { name = "huggingface-hub", extra = ["inference"] }, +] +mcp = [ + { name = "mcp" }, +] +mistral = [ + { name = "mistralai" }, +] +openai = [ + { name = "openai" }, +] +vertexai = [ + { name = "google-auth" }, + { name = "requests" }, +] + [[package]] name = "pydantic-core" version = "2.33.2" @@ -4525,6 +4806,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/56/8a7ca5d2cd2cda1d245d34b1c9a942920a718082ae8e54e5f3e5a58b7add/pydantic_core-2.33.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:329467cecfb529c925cf2bbd4d60d2c509bc2fb52a20c1045bf09bb70971a9c1", size = 2066757, upload-time = "2025-04-23T18:33:30.645Z" }, ] +[[package]] +name = "pydantic-evals" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "eval-type-backport", marker = "python_full_version < '3.11'" }, + { name = "logfire-api" }, + { name = "pydantic" }, + { name = "pydantic-ai-slim" }, + { name = "pyyaml" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/d8/007003d9eb01791c3b6daa728c4b3c4d0dbfd1c5f8ce72aac455fae4521f/pydantic_evals-0.4.3.tar.gz", hash = "sha256:e1a4ed01e56da564a2db3635336210f4f5df4f930076662ecbb29472fd0c8306", size = 43280, upload-time = "2025-07-16T16:37:31.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/b7/de93497e2812f7124ec342d283af2f37ab86d43e96fae7b6a7a69b65ec1d/pydantic_evals-0.4.3-py3-none-any.whl", hash = "sha256:c9b375ded3f4467d4f758a360a28b8945801c69f61283226571d7416ff37d023", size = 52025, upload-time = "2025-07-16T16:37:21.253Z" }, +] + +[[package]] +name = "pydantic-graph" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "logfire-api" }, + { name = "pydantic" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/07/a84a51e1e1af4fa2883e20f7a6527431a5c32561b576da9b641caabb9dc8/pydantic_graph-0.4.3.tar.gz", hash = "sha256:45d32db5fe2c3d78f8db36181fd10564a24a6d165f2138873d716c967ebf05d1", size = 21867, upload-time = "2025-07-16T16:37:32.063Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/f9/50fa967164728141447430f39bcf21d4adce29b5c057f1aa8f27374969be/pydantic_graph-0.4.3-py3-none-any.whl", hash = "sha256:e8cc3af1b65e07aa29c69c7c2e1c9bc09e024f642b70fef115c473438e57c2ea", size = 27494, upload-time = "2025-07-16T16:37:22.451Z" }, +] + [[package]] name = "pydantic-settings" version = "2.10.1" @@ -5393,6 +5707,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/85/338e603dc68e7d9994d5d84f24adbf69bae760ba5efd3e20f5ff2cec18da/ruff-0.9.10-py3-none-win_arm64.whl", hash = "sha256:5fd804c0327a5e5ea26615550e706942f348b197d5475ff34c19733aee4b2e69", size = 10436892, upload-time = "2025-03-07T15:27:41.687Z" }, ] +[[package]] +name = "s3transfer" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, +] + [[package]] name = "safetensors" version = "0.5.3" From 180d73f370797e158d49b7708f5fb4ec1c04e62a Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 25 Aug 2025 20:28:02 -0700 Subject: [PATCH 2/4] update uv.lock --- uv.lock | 4 ---- 1 file changed, 4 deletions(-) diff --git a/uv.lock b/uv.lock index 6d1b3e4d..3a5e1207 100644 --- a/uv.lock +++ b/uv.lock @@ -1249,9 +1249,6 @@ dev = [ { name = "haikus" }, { name = "pytest" }, ] -pydantic = [ - { name = "pydantic-ai" }, -] [package.metadata] requires-dist = [ @@ -1341,7 +1338,6 @@ dev = [ { name = "haikus", specifier = "==0.3.8" }, { name = "pytest", specifier = ">=8.4.1" }, ] -pydantic = [{ name = "pydantic-ai", specifier = ">=0.4.3" }] [[package]] name = "eval-type-backport" From 6cbe4b816a7e43df21d92c0831915c713692528b Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 25 Aug 2025 20:39:24 -0700 Subject: [PATCH 3/4] support FIREWORKS_API_KEY in env --- .../pytest/default_pydantic_ai_rollout_processor.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py b/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py index d434b90d..cfa0257f 100644 --- a/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py +++ b/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py @@ -1,3 +1,4 @@ +import os import asyncio import logging from typing import List @@ -21,6 +22,7 @@ ToolReturnPart, UserPromptPart, ) +from pydantic_ai.providers.fireworks import FireworksProvider logger = logging.getLogger(__name__) @@ -47,9 +49,17 @@ def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> agent: Agent = config.kwargs["agent"] + if config.completion_params["provider"] == "fireworks": + api_key = os.getenv("FIREWORKS_API_KEY") + if not api_key: + raise ValueError("FIREWORKS_API_KEY is not set") + provider = FireworksProvider(api_key=api_key) + else: + provider = config.completion_params["provider"] + model = OpenAIModel( config.completion_params["model"], - provider=config.completion_params["provider"], + provider=provider, ) async def process_row(row: EvaluationRow) -> EvaluationRow: From 6718c4b158a776c24e680909febea774325c855f Mon Sep 17 00:00:00 2001 From: Dylan Huang Date: Mon, 25 Aug 2025 20:51:15 -0700 Subject: [PATCH 4/4] fix issue --- .../pytest/default_pydantic_ai_rollout_processor.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py b/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py index cfa0257f..7b295f1a 100644 --- a/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py +++ b/eval_protocol/pytest/default_pydantic_ai_rollout_processor.py @@ -22,6 +22,7 @@ ToolReturnPart, UserPromptPart, ) +from pydantic_ai.providers.openai import OpenAIProvider from pydantic_ai.providers.fireworks import FireworksProvider logger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ class PydanticAgentRolloutProcessor(RolloutProcessor): def __init__(self): # dummy model used for its helper functions for processing messages - self.util = OpenAIModel("dummy-model") + self.util = OpenAIModel("dummy-model", provider=OpenAIProvider(api_key="dummy")) def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> List[asyncio.Task[EvaluationRow]]: """Create agent rollout tasks and return them for external handling.""" @@ -49,17 +50,9 @@ def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) -> agent: Agent = config.kwargs["agent"] - if config.completion_params["provider"] == "fireworks": - api_key = os.getenv("FIREWORKS_API_KEY") - if not api_key: - raise ValueError("FIREWORKS_API_KEY is not set") - provider = FireworksProvider(api_key=api_key) - else: - provider = config.completion_params["provider"] - model = OpenAIModel( config.completion_params["model"], - provider=provider, + provider=config.completion_params["provider"], ) async def process_row(row: EvaluationRow) -> EvaluationRow: