Update quickstart

Author: xzrderek

eval_protocol/pytest/evaluation_test.py

@@ -379,7 +379,38 @@ async def _execute_groupwise_eval_with_semaphore(
                                 pointwise_tasks.append(
                                     asyncio.create_task(_execute_pointwise_eval_with_semaphore(row=row))
                                 )
-                            results = await asyncio.gather(*pointwise_tasks)
+
+                            # Add tqdm progress bar for evaluations with proper cleanup
+                            eval_position = run_idx + 2  # Position after rollout progress bar
+                            with tqdm(
+                                total=len(pointwise_tasks),
+                                desc=f"  Eval {run_idx + 1}",
+                                unit="eval",
+                                file=sys.__stderr__,
+                                leave=False,
+                                position=eval_position,
+                                dynamic_ncols=True,
+                                miniters=1,
+                                mininterval=0.1,
+                                bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]",
+                            ) as eval_pbar:
+
+                                async def task_with_progress(task):
+                                    try:
+                                        result = await task
+                                        return result
+                                    finally:
+                                        eval_pbar.update(1)
+
+                                wrapped_tasks = [task_with_progress(task) for task in pointwise_tasks]
+                                try:
+                                    results = await asyncio.gather(*wrapped_tasks)
+                                except Exception:
+                                    # Propagate cancellation to the real tasks and await them to quiesce
+                                    for task in pointwise_tasks:
+                                        task.cancel()
+                                    await asyncio.gather(*pointwise_tasks, return_exceptions=True)
+                                    raise
 
                             all_results[run_idx] = results
                         elif mode == "groupwise":
@@ -510,14 +541,23 @@ async def _collect_result(config, lst):  # pyright: ignore[reportUnknownParamete
                         ) as run_pbar:
 
                             async def execute_run_with_progress(run_idx: int, config):
-                                result = await execute_run(run_idx, config)
-                                run_pbar.update(1)
-                                return result
+                                try:
+                                    result = await execute_run(run_idx, config)
+                                    return result
+                                finally:
+                                    run_pbar.update(1)
 
                             tasks = []
                             for run_idx in range(num_runs):
                                 tasks.append(asyncio.create_task(execute_run_with_progress(run_idx, config)))
-                            await asyncio.gather(*tasks)  # pyright: ignore[reportUnknownArgumentType]
+                            try:
+                                await asyncio.gather(*tasks)
+                            except Exception:
+                                # Propagate cancellation to tasks and await them to quiesce
+                                for task in tasks:
+                                    task.cancel()
+                                await asyncio.gather(*tasks, return_exceptions=True)
+                                raise
 
                     experiment_duration_seconds = time.perf_counter() - experiment_start_time
 

eval_protocol/pytest/utils.py

@@ -277,7 +277,7 @@ async def execute_row_with_backoff_and_log(task: asyncio.Task, row: EvaluationRo
         position = run_idx + 1  # Position 0 is reserved for main run bar, so shift up by 1
         with tqdm(
             total=len(retry_tasks),
-            desc=f"  Run {position}",
+            desc=f"  Run {run_idx + 1}",
             unit="rollout",
             file=sys.__stderr__,
             leave=False,

eval_protocol/quickstart/llm_judge_braintrust.py

@@ -13,25 +13,26 @@
     EvaluationRow,
     SingleTurnRolloutProcessor,
     create_braintrust_adapter,
+    DefaultParameterIdGenerator,
 )
+
 # adapter = create_braintrust_adapter()
+input_rows = [
+    #         adapter.get_evaluation_rows(
+    #             btql_query=f"""
+    # select: *
+    # from: project_logs('{os.getenv("BRAINTRUST_PROJECT_ID")}') traces
+    # filter: is_root = true
+    # limit: 10
+    # """
+    #         )
+]
 
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")
-@pytest.mark.asyncio
-@evaluation_test(
-    input_rows=[
-        #         adapter.get_evaluation_rows(
-        #             btql_query=f"""
-        # select: *
-        # from: project_logs('{os.getenv("BRAINTRUST_PROJECT_ID")}') traces
-        # filter: is_root = true
-        # limit: 10
-        # """
-        #         )
-        []
-    ],
-    completion_params=[
+@pytest.mark.parametrize(
+    "completion_params",
+    [
         {"model": "gpt-4.1"},
         {
             "max_tokens": 131000,
@@ -44,10 +45,13 @@
             "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-20b",
         },
     ],
+    ids=DefaultParameterIdGenerator.generate_id_from_dict,
+)
+@evaluation_test(
+    input_rows=input_rows,
     rollout_processor=SingleTurnRolloutProcessor(),
     preprocess_fn=multi_turn_assistant_to_ground_truth,
     max_concurrent_rollouts=64,
-    aggregation_method="bootstrap",
 )
 async def test_llm_judge(row: EvaluationRow) -> EvaluationRow:
     return await aha_judge(row)

eval_protocol/quickstart/llm_judge_langfuse.py

@@ -14,25 +14,25 @@
     EvaluationRow,
     SingleTurnRolloutProcessor,
     create_langfuse_adapter,
+    DefaultParameterIdGenerator,
 )
 
 from eval_protocol.quickstart import aha_judge
 
 adapter = create_langfuse_adapter()
+input_rows = adapter.get_evaluation_rows(
+    to_timestamp=datetime(2025, 9, 12, 0, 11, 18),
+    limit=1,
+    sample_size=1,
+    sleep_between_gets=3.0,
+    max_retries=5,
+)
 
 
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")
-@evaluation_test(
-    input_rows=[
-        adapter.get_evaluation_rows(
-            to_timestamp=datetime(2025, 9, 12, 0, 11, 18),
-            limit=711,
-            sample_size=50,
-            sleep_between_gets=3.0,
-            max_retries=5,
-        )
-    ],
-    completion_params=[
+@pytest.mark.parametrize(
+    "completion_params",
+    [
         {"model": "gpt-4.1"},
         {
             "max_tokens": 131000,
@@ -45,10 +45,14 @@
             "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-20b",
         },
     ],
+    ids=DefaultParameterIdGenerator.generate_id_from_dict,
+)
+@evaluation_test(
+    input_rows=[input_rows],
     rollout_processor=SingleTurnRolloutProcessor(),
     preprocess_fn=multi_turn_assistant_to_ground_truth,
-    max_concurrent_rollouts=2,
-    aggregation_method="bootstrap",
+    max_concurrent_rollouts=64,
+    max_concurrent_evaluations=2,
 )
 async def test_llm_judge(row: EvaluationRow) -> EvaluationRow:
     return await aha_judge(row)

eval_protocol/quickstart/llm_judge_langsmith.py

@@ -31,6 +31,7 @@
     EvaluationRow,
     SingleTurnRolloutProcessor,
     LangSmithAdapter,
+    DefaultParameterIdGenerator,
 )
 
 
@@ -53,11 +54,13 @@ def fetch_langsmith_traces_as_evaluation_rows(
         return []
 
 
+input_rows = fetch_langsmith_traces_as_evaluation_rows()
+
+
 @pytest.mark.skipif(os.environ.get("CI") == "true", reason="Skip in CI")
-@pytest.mark.asyncio
-@evaluation_test(
-    input_rows=[fetch_langsmith_traces_as_evaluation_rows()],
-    completion_params=[
+@pytest.mark.parametrize(
+    "completion_params",
+    [
         {
             "model": "fireworks_ai/accounts/fireworks/models/qwen3-235b-a22b-instruct-2507",
         },
@@ -67,9 +70,12 @@ def fetch_langsmith_traces_as_evaluation_rows(
             "model": "fireworks_ai/accounts/fireworks/models/gpt-oss-120b",
         },
     ],
+    ids=DefaultParameterIdGenerator.generate_id_from_dict,
+)
+@evaluation_test(
+    input_rows=[input_rows],
     rollout_processor=SingleTurnRolloutProcessor(),
     preprocess_fn=multi_turn_assistant_to_ground_truth,
-    aggregation_method="bootstrap",
 )
 async def test_llm_judge_langsmith(row: EvaluationRow) -> EvaluationRow:
     """LLM Judge evaluation over LangSmith-sourced rows, persisted locally by Eval Protocol.

eval_protocol/quickstart/llm_judge_openai_responses.py

@@ -57,7 +57,6 @@
     input_rows=[input_rows],
     rollout_processor=SingleTurnRolloutProcessor(),
     preprocess_fn=multi_turn_assistant_to_ground_truth,
-    aggregation_method="bootstrap",
 )
 async def test_llm_judge_openai_responses(row: EvaluationRow) -> EvaluationRow:
     return await aha_judge(row)