diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml index 79e8986b4d..73aa2b51a5 100644 --- a/distributed/distributed-schema.yaml +++ b/distributed/distributed-schema.yaml @@ -116,7 +116,7 @@ properties: worker-saturation: oneOf: - type: number - exclusiveMinimum: 0 + minimum: 0 # String "inf", not to be confused with .inf which in YAML means float # infinity. This is necessary because there's no way to parse a float # infinity from a DASK_* environment variable. @@ -125,7 +125,13 @@ properties: Controls how many root tasks are sent to workers (like a `readahead`). Up to worker-saturation * nthreads root tasks are sent to a - worker at a time. If `.inf`, all runnable tasks are immediately sent to workers. + worker at a time. + + Special values: + - 0: Only send tasks to completely idle workers (no queuing). Useful for + long-running tasks to avoid head-of-line blocking. + - .inf: All runnable tasks are immediately sent to workers. + The target number is rounded up, so any `worker-saturation` value > 1.0 guarantees at least one extra task will be sent to workers. diff --git a/distributed/scheduler.py b/distributed/scheduler.py index d73da7c71f..b811a8b874 100644 --- a/distributed/scheduler.py +++ b/distributed/scheduler.py @@ -1838,10 +1838,10 @@ def __init__( self.WORKER_SATURATION = math.inf if ( not isinstance(self.WORKER_SATURATION, (int, float)) - or self.WORKER_SATURATION <= 0 + or self.WORKER_SATURATION < 0 ): raise ValueError( # pragma: nocover - "`distributed.scheduler.worker-saturation` must be a float > 0; got " + "`distributed.scheduler.worker-saturation` must be a float >= 0; got " + repr(self.WORKER_SATURATION) ) @@ -9278,8 +9278,19 @@ def heartbeat_interval(n: int) -> float: def _task_slots_available(ws: WorkerState, saturation_factor: float) -> int: - """Number of tasks that can be sent to this worker without oversaturating it""" + """Number of tasks that can be sent to this worker without oversaturating it + + When saturation_factor is 0, tasks are only sent to completely idle workers + (no queuing). This is useful for long-running tasks where you want to avoid + head-of-line blocking. + """ assert not math.isinf(saturation_factor) + + # Special case: saturation_factor == 0 means no queuing + # Only send tasks to fill idle threads (no tasks beyond thread count) + if saturation_factor == 0: + return ws.nthreads - (len(ws.processing) - len(ws.long_running)) + return max(math.ceil(saturation_factor * ws.nthreads), 1) - ( len(ws.processing) - len(ws.long_running) ) diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py index 4b9d917df9..d7cf074bac 100644 --- a/distributed/tests/test_scheduler.py +++ b/distributed/tests/test_scheduler.py @@ -626,6 +626,7 @@ def func(first, second): (1.1, (3, 2)), (1.0, (2, 1)), (0.1, (1, 1)), + (0.0, (2, 1)), # No queuing: only executing tasks, no queued tasks # This is necessary because there's no way to parse a float infinite from # a DASK_* environment variable ("inf", (6, 4)), @@ -674,6 +675,12 @@ async def test_bad_saturation_factor(): async with Scheduler(dashboard_address=":0"): pass + # Negative values should be rejected + with pytest.raises(ValueError, match=">= 0"): + with dask.config.set({"distributed.scheduler.worker-saturation": -1.0}): + async with Scheduler(dashboard_address=":0"): + pass + @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3) async def test_move_data_over_break_restrictions(client, s, a, b, c):