assert error if evaluation_result not set

Author: xzrderek

eval_protocol/pytest/evaluation_test.py

@@ -620,7 +620,13 @@ async def _collect_result(config, lst):
 
                     experiment_duration_seconds = time.perf_counter() - experiment_start_time
 
-                    # for groupwise mode, the result contains eval otuput from multiple completion_params, we need to differentiate them
+                    if not all(r.evaluation_result is not None for run_results in all_results for r in run_results):
+                        raise AssertionError(
+                            "Some EvaluationRow instances are missing evaluation_result. "
+                            "Your @evaluation_test function (or its rollout processor) must set `row.evaluation_result`"
+                        )
+
+                    # for groupwise mode, the result contains eval output from multiple completion_params, we need to differentiate them
                     # rollout_id is used to differentiate the result from different completion_params
                     if mode == "groupwise":
                         results_by_group = [

tests/pytest/test_pytest_missing_evaluation_result.py

@@ -0,0 +1,32 @@
+import pytest
+
+from eval_protocol.models import EvaluationRow, Message
+from eval_protocol.pytest.default_no_op_rollout_processor import NoOpRolloutProcessor
+from eval_protocol.pytest.evaluation_test import evaluation_test
+
+
+@pytest.mark.asyncio
+async def test_missing_evaluation_result_raises_assertion_error() -> None:
+    """evaluation_test should raise if any EvaluationRow is missing evaluation_result."""
+
+    input_messages = [
+        [Message(role="user", content="Test message")],
+    ]
+
+    @evaluation_test(
+        input_messages=[input_messages],
+        rollout_processor=NoOpRolloutProcessor(),
+        mode="pointwise",
+        num_runs=1,
+    )
+    def eval_fn(row: EvaluationRow) -> EvaluationRow:
+        # Intentionally forget to set row.evaluation_result
+        return row
+
+    with pytest.raises(AssertionError) as excinfo:
+        # Trigger the evaluation; this should hit the assertion added in evaluation_test.py
+        await eval_fn(input_messages=input_messages)  # pyright: ignore[reportCallIssue]
+
+    msg = str(excinfo.value)
+    assert "Some EvaluationRow instances are missing evaluation_result" in msg
+    assert "must set `row.evaluation_result`" in msg