allow for manual parametrization using pytest

Author: Dylan Huang

eval_protocol/__init__.py

@@ -39,6 +39,7 @@
 from .typed_interface import reward_function
 from .quickstart import aha_judge, split_multi_turn_rows
 from .pytest import evaluation_test, SingleTurnRolloutProcessor
+from .pytest.parameterize import DefaultParameterIdGenerator
 
 from .adapters import OpenAIResponsesAdapter
 
@@ -61,6 +62,7 @@
 warnings.filterwarnings("default", category=DeprecationWarning, module="eval_protocol")
 
 __all__ = [
+    "DefaultParameterIdGenerator",
     "aha_judge",
     "split_multi_turn_rows",
     "evaluation_test",

eval_protocol/pytest/evaluation_test.py

@@ -158,8 +158,14 @@ def evaluation_test(
         exception_handler_config: Configuration for exception handling and backoff retry logic.
             If not provided, a default configuration will be used with common retryable exceptions.
     """
+    # Default to [None] when completion_params is not provided
+    # This allows evaluation-only tests (e.g., using NoOpRolloutProcessor)
+    # to work without requiring model generation parameters
     if completion_params is None:
+        completion_params_provided = False
         completion_params = [None]
+    else:
+        completion_params_provided = True
     if rollout_processor is None:
         rollout_processor = NoOpRolloutProcessor()
 
@@ -201,6 +207,7 @@ def decorator(
             combinations,
             input_dataset,
             completion_params,
+            completion_params_provided,
             input_messages,
             input_rows,
             evaluation_test_kwargs,
@@ -565,12 +572,14 @@ async def execute_run_with_progress(run_idx: int, config):
             return create_dynamically_parameterized_wrapper(
                 test_func,
                 wrapper_body,
-                pytest_parametrize_args["argnames"],
+                pytest_parametrize_args["sig_parameters"],
             )
 
         # Create the pytest wrapper
         pytest_wrapper = create_wrapper_with_signature()
-        pytest_wrapper = pytest.mark.parametrize(**pytest_parametrize_args)(pytest_wrapper)
+        pytest_wrapper = pytest.mark.parametrize(**pytest_parametrize_args["pytest_parametrize_kwargs"])(
+            pytest_wrapper
+        )
         pytest_wrapper = pytest.mark.asyncio(pytest_wrapper)
 
         # Create the dual mode wrapper

eval_protocol/pytest/parameterize.py

@@ -9,12 +9,28 @@
 from eval_protocol.pytest.types import DatasetPathParam, EvaluationInputParam, InputMessagesParam, TestFunction
 
 
-class PytestParametrizeArgs(TypedDict):
+class PytestMarkParametrizeKwargs(TypedDict):
     argnames: Sequence[str]
     argvalues: Iterable[ParameterSet | Sequence[object] | object]
     ids: Iterable[str] | None
 
 
+class ParametrizeArgs(TypedDict):
+    """
+    This contains all the necessary information to properly hijack the test
+    function's signature and dynamically inject usage of
+    pytest.mark.parametrize. The two will differ when a user manually provides
+    the pytest.mark.parametrize decorator instead of passing completion_params
+    on their own.
+    """
+
+    # for create_dynamically_parameterized_wrapper
+    sig_parameters: Sequence[str]
+
+    # for pytest.mark.parametrize
+    pytest_parametrize_kwargs: PytestMarkParametrizeKwargs
+
+
 class ParameterIdGenerator(Protocol):
     """Protocol for generating pytest parameter IDs from parameter combinations."""
 
@@ -30,7 +46,7 @@ def generate_id(self, combo: CombinationTuple) -> str | None:
         ...
 
 
-class DefaultParameterIdGenerator:
+class DefaultParameterIdGenerator(ParameterIdGenerator):
     """Default ID generator that creates meaningful IDs from parameter combinations."""
 
     def __init__(self, max_length: int = 200):
@@ -46,34 +62,48 @@ def generate_id(self, combo: CombinationTuple) -> str | None:
         dataset, completion_params, messages, rows, evaluation_test_kwargs = combo
 
         if completion_params:
-            # Get all string, numeric, and boolean values from completion_params, sorted by key
-            str_values = []
-            for key in sorted(completion_params.keys()):
-                value = completion_params[key]
-                if isinstance(value, (str, int, float, bool)):
-                    str_values.append(str(value))
+            id = self.generate_id_from_dict(completion_params, self.max_length)
+            if id:
+                return id
+        else:
+            if rows:
+                return f"rows(len={len(rows)})"
+            elif messages:
+                return f"messages(len={len(messages)})"
+            elif dataset:
+                return f"dataset(len={len(dataset)})"
+        return None
 
-            if str_values:
-                id_str = ":".join(str_values)
+    @staticmethod
+    def generate_id_from_dict(d: dict[str, object], max_length: int = 200) -> str | None:
+        # Get all string, numeric, and boolean values from completion_params, sorted by key
+        str_values = []
+        for key in sorted(d.keys()):
+            value = d[key]
+            if isinstance(value, (str, int, float, bool)):
+                str_values.append(str(value))
 
-                # Truncate if too long
-                if len(id_str) > self.max_length:
-                    id_str = id_str[: self.max_length - 3] + "..."
+        if str_values:
+            id_str = ":".join(str_values)
 
-                return id_str
+            # Truncate if too long
+            if len(id_str) > max_length:
+                id_str = id_str[: max_length - 3] + "..."
 
+            return id_str
         return None
 
 
 def pytest_parametrize(
     combinations: list[CombinationTuple],
     input_dataset: Sequence[DatasetPathParam] | None,
     completion_params: Sequence[CompletionParams | None] | None,
+    completion_params_provided: bool,
     input_messages: Sequence[list[InputMessagesParam] | None] | None,
     input_rows: Sequence[list[EvaluationRow]] | None,
     evaluation_test_kwargs: Sequence[EvaluationInputParam | None] | None,
     id_generator: ParameterIdGenerator | None = None,
-) -> PytestParametrizeArgs:
+) -> ParametrizeArgs:
     """
     This function dynamically generates pytest.mark.parametrize arguments for a given
     set of combinations. This is the magic that allows developers to pass in their
@@ -84,16 +114,23 @@ def pytest_parametrize(
 
     # Create parameter tuples for pytest.mark.parametrize
     argnames: list[str] = []
+    sig_parameters: list[str] = []
     if input_dataset is not None:
         argnames.append("dataset_path")
+        sig_parameters.append("dataset_path")
     if completion_params is not None:
-        argnames.append("completion_params")
+        if completion_params_provided:
+            argnames.append("completion_params")
+        sig_parameters.append("completion_params")
     if input_messages is not None:
         argnames.append("input_messages")
+        sig_parameters.append("input_messages")
     if input_rows is not None:
         argnames.append("input_rows")
+        sig_parameters.append("input_rows")
     if evaluation_test_kwargs is not None:
         argnames.append("evaluation_test_kwargs")
+        sig_parameters.append("evaluation_test_kwargs")
 
     # Use default ID generator if none provided
     if id_generator is None:
@@ -109,7 +146,7 @@ def pytest_parametrize(
         # Build parameter tuple based on what's provided
         if input_dataset is not None:
             param_tuple.append(dataset)
-        if completion_params is not None:
+        if completion_params_provided:
             param_tuple.append(cp)
         if input_messages is not None:
             param_tuple.append(messages)
@@ -132,7 +169,12 @@ def pytest_parametrize(
             ids.append(combo_id)
 
     # Return None for ids if no IDs were generated (let pytest use defaults)
-    return PytestParametrizeArgs(argnames=argnames, argvalues=argvalues, ids=ids if ids else None)
+    return ParametrizeArgs(
+        pytest_parametrize_kwargs=PytestMarkParametrizeKwargs(
+            argnames=argnames, argvalues=argvalues, ids=ids if ids else None
+        ),
+        sig_parameters=sig_parameters,
+    )
 
 
 def create_dynamically_parameterized_wrapper(

eval_protocol/quickstart/llm_judge_openai_responses.py

@@ -27,6 +27,7 @@
     EvaluationRow,
     SingleTurnRolloutProcessor,
     OpenAIResponsesAdapter,
+    DefaultParameterIdGenerator,
 )
 
 adapter = OpenAIResponsesAdapter()
@@ -41,17 +42,20 @@
 
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")
-@pytest.mark.asyncio
-@evaluation_test(
-    input_rows=[input_rows],
-    completion_params=[
+@pytest.mark.parametrize(
+    "completion_params",
+    [
         {
             "model": "fireworks_ai/accounts/fireworks/models/deepseek-v3p1",
         },
         {
             "model": "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct-0905",
         },
     ],
+    ids=DefaultParameterIdGenerator.generate_id_from_dict,
+)
+@evaluation_test(
+    input_rows=[input_rows],
     rollout_processor=SingleTurnRolloutProcessor(),
     preprocess_fn=split_multi_turn_rows,
     mode="all",

tests/pytest/test_parameterized_ids.py

@@ -1,12 +1,47 @@
+from collections.abc import Awaitable, Callable
+
+import pytest
 from eval_protocol.models import EvaluationRow, Message
 from eval_protocol.pytest import evaluation_test
 from eval_protocol.pytest.parameterize import DefaultParameterIdGenerator, pytest_parametrize
 from eval_protocol.pytest.generate_parameter_combinations import generate_parameter_combinations
+from eval_protocol.pytest.types import TestFunction
+
+
+def verify_parametrize_mark(test_function: TestFunction, expected_ids_set: list[object]):
+    # The function should exist and be callable
+    assert test_function is not None
+    assert callable(test_function)
+
+    # Test that the decorator was applied (function should have pytest marks)
+    import pytest
+
+    marks = getattr(test_function, "pytestmark", [])
+    assert len(marks) > 0, "Function should have pytest marks from evaluation_test decorator"
+
+    # Verify it's a parametrize mark
+    parametrize_marks = [mark for mark in marks if hasattr(mark, "name") and mark.name == "parametrize"]
+    assert len(parametrize_marks) > 0, "Should have parametrize mark"
+
+    assert len(parametrize_marks) == len(expected_ids_set), (
+        f"Expected {len(expected_ids_set)} parametrize marks, got {len(parametrize_marks)}"
+    )
+
+    # Check that the parametrize mark has IDs
+    for parametrize_mark, expected_ids in zip(parametrize_marks, expected_ids_set):
+        assert hasattr(parametrize_mark, "kwargs"), "Parametrize mark should have kwargs"
+        assert "ids" in parametrize_mark.kwargs, "Should have ids in kwargs"
+
+        # Extract the IDs from the parametrize mark
+        ids = parametrize_mark.kwargs.get("ids")
+        if not ids:
+            raise ValueError("No IDs found in parametrize mark")
+        # Should have IDs for all parameters that have string/numeric values
+        assert ids == expected_ids, f"Expected {expected_ids}, got {ids}"
 
 
 def test_parameterized_ids():
     """Test that evaluation_test generates proper parameter IDs."""
-    collected_ids = []
 
     @evaluation_test(
         input_messages=[[[Message(role="user", content="Hello, how are you?")]]],
@@ -17,35 +52,38 @@ def test_parameterized_ids():
         ],
     )
     def test_parameterized_ids(row: EvaluationRow) -> EvaluationRow:
-        # Collect the row to verify it was processed
-        collected_ids.append(row.input_metadata.row_id)
         return row
 
-    # The function should exist and be callable
-    assert test_parameterized_ids is not None
-    assert callable(test_parameterized_ids)
-
-    # Test that the decorator was applied (function should have pytest marks)
-    import pytest
+    verify_parametrize_mark(
+        test_parameterized_ids, [["fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "gpt-4", "0.5"]]
+    )
 
-    marks = getattr(test_parameterized_ids, "pytestmark", [])
-    assert len(marks) > 0, "Function should have pytest marks from evaluation_test decorator"
 
-    # Verify it's a parametrize mark
-    parametrize_marks = [mark for mark in marks if hasattr(mark, "name") and mark.name == "parametrize"]
-    assert len(parametrize_marks) > 0, "Should have parametrize mark"
+def test_parametrized_ids_with_manual_decorator_and_input_rows():
+    """Test that evaluation_test generates proper parameter IDs."""
 
-    # Check that the parametrize mark has IDs
-    parametrize_mark = parametrize_marks[0]
-    assert hasattr(parametrize_mark, "kwargs"), "Parametrize mark should have kwargs"
-    assert "ids" in parametrize_mark.kwargs, "Should have ids in kwargs"
+    @pytest.mark.parametrize(
+        "completion_params",
+        [
+            {"model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b"},
+            {"model": "gpt-4"},
+            {"temperature": 0.5},
+        ],
+        ids=DefaultParameterIdGenerator.generate_id_from_dict,
+    )
+    @evaluation_test(
+        input_rows=[[EvaluationRow(messages=[Message(role="user", content="Hello, how are you?")])]],
+    )
+    def test_parameterized_ids(row: EvaluationRow) -> EvaluationRow:
+        return row
 
-    # Extract the IDs from the parametrize mark
-    ids = parametrize_mark.kwargs.get("ids")
-    if ids is not None:
-        # Should have IDs for all parameters that have string/numeric values
-        expected_ids = ["fireworks_ai/accounts/fireworks/models/gpt-oss-120b", "gpt-4", "0.5"]
-        assert list(ids) == expected_ids, f"Expected {expected_ids}, got {list(ids)}"
+    verify_parametrize_mark(
+        test_parameterized_ids,
+        [
+            ["rows(len=1)"],
+            DefaultParameterIdGenerator.generate_id_from_dict,
+        ],
+    )
 
 
 def test_default_id_generator():
@@ -113,14 +151,15 @@ def test_pytest_parametrize_with_custom_id_generator():
         combinations=combinations,
         input_dataset=None,
         completion_params=[{"model": "gpt-4"}, {"model": "claude-3"}, {"temperature": 0.5}],
+        completion_params_provided=True,
         input_messages=None,
         input_rows=None,
         evaluation_test_kwargs=None,
     )
 
-    assert result["argnames"] == ["completion_params"]
-    assert len(list(result["argvalues"])) == 3
-    assert result["ids"] == ["gpt-4", "claude-3", "0.5"]  # All have string/numeric values
+    assert result["pytest_parametrize_kwargs"]["argnames"] == ["completion_params"]
+    assert len(list(result["pytest_parametrize_kwargs"]["argvalues"])) == 3
+    assert result["pytest_parametrize_kwargs"]["ids"] == ["gpt-4", "claude-3", "0.5"]  # All have string/numeric values
 
 
 def test_id_generator_max_length():