feat(supervisor): add opt-in dequeue backpressure (#3836)

The supervisor can now pause dequeuing - and freeze consumer-pool
scale-up - when a backpressure signal says the cluster can't place more
work, then ramp dequeuing back up gradually once it clears. The signal
is a verdict published to a Redis key by a cluster-side component; the
supervisor reads it on a short refresh and gates `preDequeue` on it.

Off by default (`TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED`). Everything
fails open: a missing, stale, or unreadable verdict never pins the
brake, and the hot-path read is a synchronous cached lookup with no I/O.
The scale-up freeze leaves scale-down untouched, and on release the
resume is ramped so a deep queue isn't hammered all at once.

Dry-run is on by default (`TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN`): even
once enabled it only logs what it would have done, and surfaces the
computed state through metrics, until explicitly set to act. Prometheus:
`supervisor_backpressure_engaged`, `_dry_run`,
`_skipped_dequeues_total`.

Refs TRI-5354

Author: nicktrn

.changeset/backpressure-scale-up-freeze.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/core": patch
+---
+
+Add optional `shouldPauseScaling` to the supervisor consumer pool scaling options to freeze scale-up while it returns true (scale-down stays allowed).

.server-changes/supervisor-dequeue-backpressure.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: feature
+---
+
+Add opt-in dequeue backpressure to the supervisor. When enabled, the supervisor reads a verdict from Redis and pauses dequeuing while the worker cluster is saturated, then resumes once capacity is available. Disabled by default - no behavior change for existing deployments.

apps/supervisor/package.json

@@ -18,13 +18,15 @@
     "@kubernetes/client-node": "^1.0.0",
     "@trigger.dev/core": "workspace:*",
     "dockerode": "^4.0.6",
+    "ioredis": "^5.3.2",
     "p-limit": "^6.2.0",
     "prom-client": "^15.1.0",
     "socket.io": "4.7.4",
     "std-env": "^3.8.0",
     "zod": "3.25.76"
   },
   "devDependencies": {
+    "@internal/testcontainers": "workspace:*",
     "@types/dockerode": "^3.3.33"
   }
 }

apps/supervisor/src/backpressure/backpressureMetrics.ts

@@ -0,0 +1,34 @@
+import { Counter, Gauge, type Registry } from "prom-client";
+
+/** Prometheus metrics for dequeue backpressure. */
+export class BackpressureMetrics {
+  /** 1 while backpressure is engaged (computed signal, set even in dry-run). */
+  readonly engaged: Gauge<string>;
+  /** 1 when running in dry-run (gates inert). */
+  readonly dryRun: Gauge<string>;
+  /** Dequeue attempts the gate skipped - or would have, in dry-run (labelled). */
+  readonly skipsTotal: Counter<string>;
+
+  constructor(opts: { register: Registry; prefix?: string }) {
+    const prefix = opts.prefix ?? "supervisor_backpressure";
+
+    this.engaged = new Gauge({
+      name: `${prefix}_engaged`,
+      help: "1 while dequeue backpressure is engaged (computed signal, regardless of dry-run)",
+      registers: [opts.register],
+    });
+
+    this.dryRun = new Gauge({
+      name: `${prefix}_dry_run`,
+      help: "1 when dequeue backpressure is in dry-run mode (gates inert)",
+      registers: [opts.register],
+    });
+
+    this.skipsTotal = new Counter({
+      name: `${prefix}_skipped_dequeues_total`,
+      help: "Dequeue attempts skipped by backpressure (or would be, in dry-run)",
+      labelNames: ["dry_run"],
+      registers: [opts.register],
+    });
+  }
+}