format

Author: mayinghan

eval_protocol/pytest/evaluation_test.py

@@ -584,9 +584,10 @@ def _log_eval_error(
                         # log the fresh_dataset
                         for row in fresh_dataset:
                             active_logger.log(row)
-                            
+
                         # prepare parallel eval helper function
                         semaphore = asyncio.Semaphore(max_concurrent_evaluations)
+
                         async def _execute_eval_with_semaphore(**kwargs):
                             async with semaphore:
                                 # NOTE: we will still evaluate errored rows (give users control over this)
@@ -802,23 +803,23 @@ async def dual_mode_wrapper(*args, **kwargs):
 
                 # If not a direct call, use the pytest wrapper
                 return await pytest_wrapper(*args, **kwargs)
-            
-            dual_mode_wrapper._origin_func = test_func 
+
+            dual_mode_wrapper._origin_func = test_func
             dual_mode_wrapper._evaluator_id = test_func.__name__
             # Generate (stable) evaluator ID from function source code hash
             try:
                 func_source = inspect.getsource(test_func)
                 parsed = ast.parse(func_source)
                 normalized_source = ast.unparse(parsed)
-                clean_source = ''.join(normalized_source.split()) + test_func.__name__
-                func_hash = hashlib.sha256(clean_source.encode('utf-8')).hexdigest()[:12]
+                clean_source = "".join(normalized_source.split()) + test_func.__name__
+                func_hash = hashlib.sha256(clean_source.encode("utf-8")).hexdigest()[:12]
                 dual_mode_wrapper._version = f"{test_func.__name__}_{func_hash}"
             except (OSError, TypeError, SyntaxError):
                 pass
             dual_mode_wrapper._metainfo = {
-               "mode": mode,
-               "max_rollout_concurrency": max_concurrent_rollouts,
-               "max_evaluation_concurrency": max_concurrent_evaluations,
+                "mode": mode,
+                "max_rollout_concurrency": max_concurrent_rollouts,
+                "max_evaluation_concurrency": max_concurrent_evaluations,
             }
 
             # Copy all attributes from the pytest wrapper to our dual mode wrapper

tests/pytest/test_get_metadata.py

@@ -4,6 +4,7 @@
 from eval_protocol.pytest import evaluation_test
 from eval_protocol.models import EvaluationRow, Message
 
+
 @evaluation_test(
     input_messages=[
         [
@@ -23,24 +24,18 @@ def test_pytest_async(rows: List[EvaluationRow]) -> List[EvaluationRow]:
     return rows
 
 
-
 def test_pytest_func_metainfo():
-    assert hasattr(test_pytest_async, "_origin_func") 
+    assert hasattr(test_pytest_async, "_origin_func")
     origin_func = test_pytest_async._origin_func
     assert not asyncio.iscoroutinefunction(origin_func)
     assert asyncio.iscoroutinefunction(test_pytest_async)
     assert test_pytest_async._metainfo["mode"] == "groupwise"
     assert test_pytest_async._metainfo["max_rollout_concurrency"] == 5
     assert test_pytest_async._metainfo["max_evaluation_concurrency"] == 10
-    
+
     # Test evaluator ID generation
     assert hasattr(test_pytest_async, "_evaluator_id")
     evaluator_id = test_pytest_async._evaluator_id
     assert evaluator_id.startswith("eval_")
     assert len(evaluator_id) == 17  # "eval_" + 12 character hash
     print(f"Generated evaluator ID: {evaluator_id}")
-    
-
-    
-    
-