|
33 | 33 | AggregationMethod = Literal["mean", "max", "min", "bootstrap"] |
34 | 34 |
|
35 | 35 |
|
| 36 | +async def run_tasks_with_eval_progress(pointwise_tasks: list, run_idx: int): |
| 37 | + """ |
| 38 | + Run evaluation tasks with a progress bar and proper cancellation handling. |
| 39 | +
|
| 40 | + Args: |
| 41 | + pointwise_tasks: List of asyncio tasks to execute |
| 42 | + run_idx: Run index for progress bar positioning and naming |
| 43 | +
|
| 44 | + Returns: |
| 45 | + Results from all tasks |
| 46 | + """ |
| 47 | + eval_position = run_idx + 2 # Position after rollout progress bar |
| 48 | + with tqdm( |
| 49 | + total=len(pointwise_tasks), |
| 50 | + desc=f" Eval {run_idx + 1}", |
| 51 | + unit="eval", |
| 52 | + file=sys.__stderr__, |
| 53 | + leave=False, |
| 54 | + position=eval_position, |
| 55 | + dynamic_ncols=True, |
| 56 | + miniters=1, |
| 57 | + mininterval=0.1, |
| 58 | + bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]", |
| 59 | + ) as eval_pbar: |
| 60 | + |
| 61 | + async def task_with_progress(task): |
| 62 | + try: |
| 63 | + result = await task |
| 64 | + return result |
| 65 | + finally: |
| 66 | + eval_pbar.update(1) |
| 67 | + |
| 68 | + wrapped_tasks = [task_with_progress(task) for task in pointwise_tasks] |
| 69 | + try: |
| 70 | + results = await asyncio.gather(*wrapped_tasks) |
| 71 | + return results |
| 72 | + except Exception: |
| 73 | + # Propagate cancellation to the real tasks and await them to quiesce |
| 74 | + for task in pointwise_tasks: |
| 75 | + task.cancel() |
| 76 | + await asyncio.gather(*pointwise_tasks, return_exceptions=True) |
| 77 | + raise |
| 78 | + |
| 79 | + |
| 80 | +async def run_tasks_with_run_progress(execute_run_func, num_runs, config): |
| 81 | + """ |
| 82 | + Run tasks with a parallel runs progress bar, preserving original logic. |
| 83 | +
|
| 84 | + Args: |
| 85 | + execute_run_func: The execute_run function to call |
| 86 | + num_runs: Number of runs to execute |
| 87 | + config: Configuration to pass to execute_run_func |
| 88 | + """ |
| 89 | + with tqdm( |
| 90 | + total=num_runs, |
| 91 | + desc="Runs (Parallel)", |
| 92 | + unit="run", |
| 93 | + file=sys.__stderr__, |
| 94 | + position=0, |
| 95 | + leave=True, |
| 96 | + dynamic_ncols=True, |
| 97 | + miniters=1, |
| 98 | + bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]", |
| 99 | + ) as run_pbar: |
| 100 | + |
| 101 | + async def execute_run_with_progress(run_idx: int, config): |
| 102 | + result = await execute_run_func(run_idx, config) |
| 103 | + run_pbar.update(1) |
| 104 | + return result |
| 105 | + |
| 106 | + tasks = [] |
| 107 | + for run_idx in range(num_runs): |
| 108 | + tasks.append(asyncio.create_task(execute_run_with_progress(run_idx, config))) |
| 109 | + try: |
| 110 | + await asyncio.gather(*tasks) |
| 111 | + except Exception: |
| 112 | + # Propagate cancellation to tasks and await them to quiesce |
| 113 | + for task in tasks: |
| 114 | + task.cancel() |
| 115 | + await asyncio.gather(*tasks, return_exceptions=True) |
| 116 | + raise |
| 117 | + |
| 118 | + |
36 | 119 | def calculate_bootstrap_scores(all_scores: list[float]) -> float: |
37 | 120 | """ |
38 | 121 | Calculate bootstrap confidence intervals for individual scores. |
@@ -277,7 +360,7 @@ async def execute_row_with_backoff_and_log(task: asyncio.Task, row: EvaluationRo |
277 | 360 | position = run_idx + 1 # Position 0 is reserved for main run bar, so shift up by 1 |
278 | 361 | with tqdm( |
279 | 362 | total=len(retry_tasks), |
280 | | - desc=f" Run {position}", |
| 363 | + desc=f" Run {run_idx + 1}", |
281 | 364 | unit="rollout", |
282 | 365 | file=sys.__stderr__, |
283 | 366 | leave=False, |
|
0 commit comments