fix test

Author: xzrderek

tests/data_loader/test_dynamic_data_loader.py

@@ -1,5 +1,5 @@
 from eval_protocol.data_loader import DynamicDataLoader
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import evaluation_test
 
 
@@ -27,6 +27,7 @@ def test_dynamic_data_loader(row: EvaluationRow) -> EvaluationRow:
         == "Factory function that generates evaluation rows dynamically."
     )
     assert row.input_metadata.dataset_info.get("data_loader_preprocessed") is False
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row
 
 
@@ -45,6 +46,7 @@ def test_dynamic_data_loader_lambda(row: EvaluationRow) -> EvaluationRow:
     assert row.input_metadata.dataset_info.get("data_loader_num_rows_after_preprocessing") == 1
     assert row.input_metadata.dataset_info.get("data_loader_type") == "DynamicDataLoader"
     assert row.input_metadata.dataset_info.get("data_loader_preprocessed") is False
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row
 
 
@@ -72,4 +74,5 @@ def test_dynamic_data_loader_max_dataset_rows(row: EvaluationRow) -> EvaluationR
     assert row.input_metadata.dataset_info.get("data_loader_type") == "DynamicDataLoader"
     assert row.input_metadata.dataset_info.get("data_loader_preprocessed") is False
 
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row

tests/data_loader/test_inline_data_loader.py

@@ -1,5 +1,5 @@
 from eval_protocol.data_loader.inline_data_loader import InlineDataLoader
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import evaluation_test
 from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 
@@ -20,6 +20,7 @@ def test_inline_data_loader(row: EvaluationRow) -> EvaluationRow:
     assert row.input_metadata.dataset_info.get("data_loader_type") == "InlineDataLoader"
     assert row.input_metadata.dataset_info.get("data_loader_variant_description") is None
     assert row.input_metadata.dataset_info.get("data_loader_preprocessed") is False
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row
 
 
@@ -41,4 +42,5 @@ def test_inline_data_loader_max_dataset_rows(row: EvaluationRow) -> EvaluationRo
     assert row.input_metadata.dataset_info.get("data_loader_type") == "InlineDataLoader"
     assert row.input_metadata.dataset_info.get("data_loader_preprocessed") is False
 
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row

tests/pytest/test_get_metadata.py

@@ -1,7 +1,7 @@
 import asyncio
 
 from eval_protocol.pytest import evaluation_test
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 
 
 @evaluation_test(
@@ -22,6 +22,8 @@
 )
 def test_pytest_async(rows: list[EvaluationRow]) -> list[EvaluationRow]:
     """Run math evaluation on sample dataset using pytest interface."""
+    for row in rows:
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return rows
 
 

tests/pytest/test_pytest_async.py

@@ -1,6 +1,6 @@
 import pytest
 
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import evaluation_test
 
 
@@ -20,6 +20,8 @@
 )
 async def test_pytest_async(rows: list[EvaluationRow]) -> list[EvaluationRow]:
     """Run math evaluation on sample dataset using pytest interface."""
+    for row in rows:
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return rows
 
 
@@ -36,6 +38,7 @@ async def test_pytest_async(rows: list[EvaluationRow]) -> list[EvaluationRow]:
 )
 async def test_pytest_async_pointwise(row: EvaluationRow) -> EvaluationRow:
     """Run pointwise evaluation on sample dataset using pytest interface."""
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row
 
 

tests/pytest/test_pytest_env_overwrite.py

@@ -1,7 +1,7 @@
 import atexit
 import shutil
 import tempfile
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import evaluation_test
 from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 from eval_protocol.pytest.default_single_turn_rollout_process import SingleTurnRolloutProcessor
@@ -23,6 +23,7 @@ def test_input_messages_in_decorator(row: EvaluationRow) -> EvaluationRow:
         """Run math evaluation on sample dataset using pytest interface."""
         assert row.messages[0].content == "What is the capital of France?"
         assert row.execution_metadata.invocation_id == "test-invocation-123"
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
 
@@ -38,6 +39,7 @@ def test_input_messages_in_env(row: EvaluationRow) -> EvaluationRow:
         """Run math evaluation on sample dataset using pytest interface."""
         assert row.messages[0].content == "What is 5 * 6?"
         assert row.input_metadata.completion_params["model"] == "gpt-40"
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
 
@@ -60,6 +62,7 @@ def test_input_messages_in_env(row: EvaluationRow) -> EvaluationRow:
     )
     def test_input_override(row: EvaluationRow) -> EvaluationRow:
         assert row.messages[0].content == "What is 10 / 2?"
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
 
@@ -79,6 +82,7 @@ def test_no_op_rollout_processor_override_from_none(row: EvaluationRow) -> Evalu
         # Verify that no actual model call was made (NoOpRolloutProcessor doesn't modify messages)
         assert len(row.messages) == 1
         assert row.messages[0].role == "user"
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
     @evaluation_test(
@@ -96,6 +100,7 @@ def test_no_op_rollout_processor_override_from_other(row: EvaluationRow) -> Eval
         assert row.messages[0].role == "user"
         # Verify the original message content is preserved (no assistant response added)
         assert row.messages[0].content == "Test override"
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
     @evaluation_test(
@@ -115,6 +120,7 @@ def test_no_op_rollout_processor_override_multiple_rows(row: EvaluationRow) -> E
         # Verify rows pass through unchanged
         assert len(row.messages) == 1
         assert row.messages[0].role == "user"
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
 

tests/pytest/test_pytest_ids.py

@@ -2,7 +2,7 @@
 
 import eval_protocol.dataset_logger as dataset_logger
 from eval_protocol.dataset_logger.dataset_logger import DatasetLogger
-from eval_protocol.models import EvaluationRow
+from eval_protocol.models import EvaluationRow, EvaluateResult
 from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 from tests.pytest.test_markdown_highlighting import markdown_dataset_to_evaluation_row
 
@@ -37,6 +37,7 @@ async def test_evaluation_test_decorator(monkeypatch):
         logger=logger,
     )
     def eval_fn(row: EvaluationRow) -> EvaluationRow:
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
     dataset_paths = [
@@ -83,6 +84,7 @@ def eval_fn(row: EvaluationRow) -> EvaluationRow:
         unique_rollout_ids.add(row.execution_metadata.rollout_id)
         unique_invocation_ids.add(row.execution_metadata.invocation_id)
         unique_row_ids.add(row.input_metadata.row_id)
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
     dataset_paths = [

tests/pytest/test_pytest_input_messages.py

@@ -1,7 +1,7 @@
 from typing import List
 
 import pytest
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import SingleTurnRolloutProcessor, evaluation_test
 
 
@@ -19,4 +19,6 @@
 )
 def test_input_messages_in_decorator(rows: List[EvaluationRow]) -> List[EvaluationRow]:
     """Run math evaluation on sample dataset using pytest interface."""
+    for row in rows:
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return rows

tests/pytest/test_pytest_input_rows.py

@@ -1,4 +1,4 @@
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest import evaluation_test
 from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 
@@ -12,4 +12,5 @@
 def test_input_messages_in_decorator(row: EvaluationRow) -> EvaluationRow:
     """Run math evaluation on sample dataset using pytest interface."""
     assert row.messages[0].content == "What is the capital of France?"
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
     return row

tests/pytest/test_pytest_stable_row_id.py

@@ -1,6 +1,6 @@
 from typing import List
 
-from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.models import EvaluationRow, Message, EvaluateResult
 from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
 from tests.pytest.test_markdown_highlighting import markdown_dataset_to_evaluation_row
 
@@ -30,6 +30,7 @@ async def test_evaluation_test_decorator_ids_single():
     )
     def eval_fn(row: EvaluationRow) -> EvaluationRow:
         row_ids.add(row.input_metadata.row_id)
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
     # Manually invoke all parameter combinations within a single test
@@ -81,6 +82,7 @@ def eval_fn(row: EvaluationRow) -> EvaluationRow:
         assert row.input_metadata is not None
         assert row.input_metadata.row_id is not None and isinstance(row.input_metadata.row_id, str)
         row_ids.add(row.input_metadata.row_id)
+        row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
         return row
 
     # Single invocation (one dataset, one param set) with multiple runs

tests/remote_server/test_remote_fireworks.py

@@ -119,7 +119,7 @@ async def test_remote_rollout_and_fetch_fireworks(row: EvaluationRow) -> Evaluat
     - trigger remote rollout via RemoteRolloutProcessor (calls init/status)
     - fetch traces from Langfuse via Fireworks tracing proxy filtered by metadata via output_data_loader; FAIL if none found
     """
-    row.evaluation_result = EvaluateResult(score=0.0, reason="Test reason")
+    row.evaluation_result = EvaluateResult(score=0.0, reason="Dummy evaluation result")
 
     assert row.messages[0].content == "What is the capital of France?", "Row should have correct message content"
     assert len(row.messages) > 1, "Row should have a response. If this fails, we fellback to the original row."