Add rollout result post processor

Author: morgendave

eval_protocol/pytest/evaluation_test_utils.py

@@ -374,9 +374,6 @@ async def execute_row_with_backoff_retry(row: EvaluationRow) -> EvaluationRow:
                 except ResponseQualityError as quality_error:
                     # Re-raise ResponseQualityError to trigger retry logic
                     raise quality_error
-                except Exception as post_process_error:
-                    # Wrap unexpected post-processor errors in ResponseQualityError
-                    raise ResponseQualityError(f"Post-processor failed: {post_process_error}") from post_process_error
 
             return result
 
@@ -393,8 +390,6 @@ async def execute_row_with_backoff(task: asyncio.Task[EvaluationRow], row: Evalu
                         config.post_processor.process(result)
                     except ResponseQualityError as quality_error:
                         raise quality_error
-                    except Exception as post_process_error:
-                        raise ResponseQualityError(f"Post-processor failed: {post_process_error}") from post_process_error
 
                 _set_rollout_status_to_finished(result)
 

eval_protocol/pytest/exception_config.py

@@ -79,13 +79,11 @@ class BackoffConfig:
     # Optional custom giveup function - if provided, overrides the default exception handling logic
     giveup_func: Callable[[Exception], bool] = lambda e: False
 
-    def get_backoff_decorator(self, exceptions: Set[Type[Exception]], exception_backoff_overrides: Dict[Type[Exception], "BackoffConfig"] | None = None):
+    def get_backoff_decorator(self, exceptions: Set[Type[Exception]]):
         """Get the appropriate backoff decorator based on configuration.
         
         Args:
             exceptions: Set of exception types to retry
-            exception_backoff_overrides: Optional mapping of exception types to custom backoff configs.
-                If an exception type has an override, that config will be used instead of this one.
         """
         if not exceptions:
             # If no exceptions specified, return a no-op decorator
@@ -94,67 +92,7 @@ def no_op_decorator(func):
 
             return no_op_decorator
 
-        # If no overrides, use simple decorator for all exceptions
-        if not exception_backoff_overrides:
-            return self._create_single_decorator(exceptions, self)
-        
-        # Group exceptions by their backoff config to avoid double backoff
-        # Each exception type gets exactly one decorator based on its config
-        # Use a tuple of config attributes as the key since BackoffConfig is not hashable
-        config_to_exceptions: Dict[tuple, tuple[Set[Type[Exception]], "BackoffConfig"]] = {}
-        
-        for exc_type in exceptions:
-            if exc_type in exception_backoff_overrides:
-                override_config = exception_backoff_overrides[exc_type]
-            else:
-                override_config = self
-            
-            # Create a hashable key from config attributes
-            # Note: jitter and giveup_func are callable, which are hashable in Python
-            config_key = (
-                override_config.strategy,
-                override_config.base_delay,
-                override_config.max_delay,
-                override_config.max_tries,
-                override_config.factor,
-                id(override_config.jitter) if override_config.jitter is not None else None,
-                id(override_config.giveup_func) if override_config.giveup_func is not None else None,
-                override_config.raise_on_giveup,
-            )
-            
-            if config_key not in config_to_exceptions:
-                config_to_exceptions[config_key] = (set(), override_config)
-            exc_set, _ = config_to_exceptions[config_key]
-            exc_set.add(exc_type)
-        
-        # If all exceptions use the same config, use a single decorator
-        if len(config_to_exceptions) == 1:
-            exc_set, config = next(iter(config_to_exceptions.values()))
-            return self._create_single_decorator(exc_set, config)
-        
-        # Create separate decorators for each config group
-        # Each exception type gets exactly one decorator, preventing double backoff
-        decorators_by_config: list[tuple[Set[Type[Exception]], Callable]] = []
-        
-        for exc_set, config in config_to_exceptions.values():
-            decorator = self._create_single_decorator(exc_set, config)
-            if decorator:
-                decorators_by_config.append((exc_set, decorator))
-        
-        # Create a combined decorator that applies all decorators
-        # Each decorator only catches exceptions in its exception set, so no double backoff
-        def combined_decorator(func):
-            decorated_func = func
-            
-            # Apply each decorator in order (inner to outer)
-            # Each decorator only catches exceptions in its specific exception set
-            # Since exception sets are disjoint (grouped by config), no double backoff
-            for exc_set, decorator in decorators_by_config:
-                decorated_func = decorator(decorated_func)
-            
-            return decorated_func
-        
-        return combined_decorator
+        return self._create_single_decorator(exceptions, self)
 
     def _create_single_decorator(self, exc_set: Set[Type[Exception]], config: "BackoffConfig"):
         """Create a single backoff decorator for a set of exceptions."""
@@ -197,10 +135,6 @@ class ExceptionHandlerConfig:
     # Backoff configuration
     backoff_config: BackoffConfig = field(default_factory=BackoffConfig)
 
-    # Per-exception backoff overrides - allows custom backoff config for specific exception types
-    # For example, ResponseQualityError can use no backoff (base_delay=0, max_delay=0)
-    exception_backoff_overrides: Dict[Type[Exception], BackoffConfig] = field(default_factory=dict)
-
     def __post_init__(self):
         """Automatically apply environment variable overrides after initialization."""
         # Override backoff settings from environment variables
@@ -211,22 +145,11 @@ def __post_init__(self):
         if "EP_FAIL_ON_MAX_RETRY" in os.environ:
             fail_on_max_retry = os.environ["EP_FAIL_ON_MAX_RETRY"].lower()
             self.backoff_config.raise_on_giveup = fail_on_max_retry != "false"
-        
-        # Set default no-backoff config for ResponseQualityError if not already set
-        if eval_protocol.exceptions.ResponseQualityError not in self.exception_backoff_overrides:
-            # Default: no backoff for ResponseQualityError (immediate retry)
-            self.exception_backoff_overrides[eval_protocol.exceptions.ResponseQualityError] = BackoffConfig(
-                strategy="constant",
-                base_delay=0.0,
-                max_delay=0.0,
-                max_tries=self.backoff_config.max_tries,
-            )
 
     def get_backoff_decorator(self):
         """Get the backoff decorator configured for this exception handler."""
         return self.backoff_config.get_backoff_decorator(
-            self.retryable_exceptions,
-            self.exception_backoff_overrides if self.exception_backoff_overrides else None
+            self.retryable_exceptions
         )
 
 

eval_protocol/pytest/rollout_result_post_processor.py

@@ -0,0 +1,57 @@
+"""
+Rollout result post-processing plugin for quality checks.
+
+This module provides an abstract base class for post-processing rollout results
+to guard response quality. Post-processors can validate results and raise
+ResponseQualityError if quality checks fail.
+"""
+
+from abc import ABC, abstractmethod
+
+from eval_protocol.models import EvaluationRow
+
+
+class RolloutResultPostProcessor(ABC):
+    """
+    Abstract base class for rollout result post-processing plugins.
+
+    Post-processors validate rollout results and can raise ResponseQualityError
+    if quality checks fail. This allows for customizable quality guards that
+    can be overridden by users.
+    """
+
+    @abstractmethod
+    def process(self, result: EvaluationRow) -> None:
+        """
+        Process and validate a rollout result.
+
+        This method should perform quality checks on the result. If quality
+        checks fail, it should raise ResponseQualityError with an appropriate
+        message.
+
+        Args:
+            result: The EvaluationRow result from the rollout
+
+        Raises:
+            ResponseQualityError: If quality checks fail
+        """
+        pass
+
+
+class NoOpRolloutResultPostProcessor(RolloutResultPostProcessor):
+    """
+    Default no-op implementation of RolloutResultPostProcessor.
+
+    This implementation does not perform any quality checks and always passes.
+    Use this as a default when no post-processing is needed.
+    """
+
+    def process(self, result: EvaluationRow) -> None:
+        """
+        No-op implementation that does not perform any quality checks.
+
+        Args:
+            result: The EvaluationRow result from the rollout
+        """
+        pass
+

tests/test_exception_config.py

@@ -0,0 +1,114 @@
+"""
+Unit tests for exception_config module.
+
+Tests the BackoffConfig and ExceptionHandlerConfig classes, including:
+1. Backoff decorator creation
+2. Per-exception backoff overrides
+3. ResponseQualityError default no-backoff configuration
+4. Exception grouping to avoid double backoff
+"""
+
+import pytest
+from eval_protocol.pytest.exception_config import BackoffConfig, ExceptionHandlerConfig, DEFAULT_RETRYABLE_EXCEPTIONS
+from eval_protocol.exceptions import ResponseQualityError
+
+
+def test_backoff_config_no_exceptions():
+    """Test that BackoffConfig returns no-op decorator when no exceptions specified."""
+    config = BackoffConfig()
+    decorator = config.get_backoff_decorator(set())
+    
+    # Should be a no-op decorator
+    def test_func():
+        return "test"
+    
+    decorated = decorator(test_func)
+    assert decorated() == "test"
+    assert decorated is test_func  # Should be the same function
+
+
+def test_backoff_config_no_overrides():
+    """Test that BackoffConfig creates a single decorator."""
+    config = BackoffConfig(strategy="constant", base_delay=0.1, max_tries=2)
+    exceptions = {ConnectionError, TimeoutError}
+    
+    decorator = config.get_backoff_decorator(exceptions)
+    assert decorator is not None
+    
+    # Decorator should be callable
+    def test_func():
+        raise ConnectionError("test")
+    
+    decorated = decorator(test_func)
+    assert callable(decorated)
+
+
+def test_exception_handler_config_default_response_quality_error():
+    """Test that ExceptionHandlerConfig includes ResponseQualityError by default."""
+    config = ExceptionHandlerConfig()
+    
+    # ResponseQualityError should be in retryable_exceptions
+    assert ResponseQualityError in config.retryable_exceptions
+
+
+def test_exception_handler_config_get_backoff_decorator():
+    """Test that ExceptionHandlerConfig.get_backoff_decorator() works correctly."""
+    config = ExceptionHandlerConfig()
+    decorator = config.get_backoff_decorator()
+    
+    assert decorator is not None
+    assert callable(decorator)
+    
+    # Should be able to decorate a function
+    def test_func():
+        raise ConnectionError("test")
+    
+    decorated = decorator(test_func)
+    assert callable(decorated)
+
+
+def test_backoff_config_expo_strategy():
+
+    """Test that BackoffConfig creates expo decorator correctly."""
+    config = BackoffConfig(strategy="expo", base_delay=1.0, max_tries=2)
+    exceptions = {ConnectionError}
+    
+    decorator = config.get_backoff_decorator(exceptions)
+    assert decorator is not None
+    
+    def test_func():
+        raise ConnectionError("test")
+    
+    decorated = decorator(test_func)
+    assert callable(decorated)
+
+
+def test_backoff_config_constant_strategy():
+    """Test that BackoffConfig creates constant decorator correctly."""
+    config = BackoffConfig(strategy="constant", base_delay=0.1, max_tries=2)
+    exceptions = {ConnectionError}
+    
+    decorator = config.get_backoff_decorator(exceptions)
+    assert decorator is not None
+    
+    def test_func():
+        raise ConnectionError("test")
+    
+    decorated = decorator(test_func)
+    assert callable(decorated)
+
+
+def test_backoff_config_invalid_strategy():
+    """Test that BackoffConfig raises ValueError for invalid strategy."""
+    config = BackoffConfig(strategy="invalid", base_delay=1.0, max_tries=2)
+    exceptions = {ConnectionError}
+    
+    with pytest.raises(ValueError, match="Unknown backoff strategy"):
+        config.get_backoff_decorator(exceptions)
+
+
+def test_exception_handler_config_response_quality_error_in_defaults():
+    """Test that ResponseQualityError is in DEFAULT_RETRYABLE_EXCEPTIONS."""
+    assert ResponseQualityError in DEFAULT_RETRYABLE_EXCEPTIONS
+
+