From 0132a30fa63837f24c6f11e580b6d068047b03cb Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 14:16:49 +0200 Subject: [PATCH] feat(ci): provisioning + executor knobs for weather-backfill dispatches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two speed levers measured on the 2026-07-05 KNYC station run: - provisioning (spot|standard, default spot): Spot capacity capped the 10-shard run at 4 concurrent VMs all day and preempted the 2020 shard 104 minutes in (exit 50001) before any month had flushed — the retry restarted from zero. standard = every task starts immediately, no preemption, ~4x VM price (a full 10-shard station run costs ~+$8) — the right trade for measurement runs. Applies to all modes; full mode keeps its spot default unless overridden. - executor (process|thread, default process; mode=station only): the thread executor decodes ONE file at a time regardless of vCPUs — h5netcdf serializes behind HDF5's global mutex (observed live: 2018/ 2019 year-shards past 3h with idle cores). The spawn-context process pool (safe since PR #100) breaks the mutex AND multiplies the per-process rate limiter. thread kept as a dispatch-time fallback; the first station dispatch after a CLI change should smoke process on a cheap 1-year window. Both inputs are validated defensively in the build step; the Batch JSON takes provisioningModel via jq --arg (injection-safe pattern unchanged). Verified: station honors both knobs; pilot/full command lines unchanged. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/run-weather-backfill.yml | 33 ++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index 7bb02fb..85a6e53 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -63,6 +63,22 @@ on: required: true default: "goes16,goes17,goes18,goes19" type: string + provisioning: + description: "VM provisioning model. spot = cheap, but capacity-capped and preemptible (a 2020 year-shard lost 104 min of work to preemption on 2026-07-05). standard = on-demand: all tasks start immediately, no preemption — ~4x VM price (n2-standard-8 ≈ $0.39/hr; a full 10-shard station run ≈ +$8). Use standard for measurement runs." + required: true + default: "spot" + type: choice + options: + - spot + - standard + executor: + description: "Slice executor for mode=station. process = spawn-context worker processes: breaks the HDF5 global-mutex decode serialization (threads decode ONE file at a time regardless of vCPUs) AND multiplies the per-process rate limiter. thread = the older serial-decode path, kept as fallback. First station dispatch after a CLI change should smoke-test process on a cheap 1-year window." + required: true + default: "process" + type: choice + options: + - process + - thread year_start: description: "First year (inclusive) of the backfill window (mode=pilot and mode=station)." required: true @@ -126,6 +142,8 @@ jobs: PILOT_STATION: ${{ inputs.pilot_station }} PRODUCTS: ${{ inputs.products }} SATELLITES: ${{ inputs.satellites }} + PROVISIONING: ${{ inputs.provisioning }} + EXECUTOR: ${{ inputs.executor }} YEAR_START: ${{ inputs.year_start }} YEAR_END: ${{ inputs.year_end }} R2_BUCKET: ${{ vars.R2_BUCKET }} @@ -240,9 +258,16 @@ jobs: # image would cartesian-product all four = wrong-hemisphere # downloads). available_since clamps make the off-era platform # nearly free. + # --executor from the workflow input: process (spawn) breaks the + # HDF5 decode mutex (PR #100 made spawn safe); thread is the + # fallback. Validate the choice defensively. + case "$EXECUTOR" in process|thread) ;; *) + echo "::error::executor must be process|thread, got '$EXECUTOR'"; exit 1 ;; + esac COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ - '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') + --arg ex "$EXECUTOR" \ + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8","--executor",$ex]') else TASK_COUNT=1 # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single @@ -264,6 +289,9 @@ jobs: --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') fi + case "$PROVISIONING" in spot) PROV_MODEL="SPOT" ;; standard) PROV_MODEL="STANDARD" ;; *) + echo "::error::provisioning must be spot|standard, got '$PROVISIONING'"; exit 1 ;; + esac jq -n \ --arg img "${IMAGE}" \ --argjson tc "${TASK_COUNT}" \ @@ -276,6 +304,7 @@ jobs: --arg rb "${R2_BUCKET}" \ --arg pb "${PROGRESS_BUCKET}" \ --arg sa "${RUNTIME_SA}" \ + --arg pm "${PROV_MODEL}" \ '{ taskGroups: [{ taskCount: $tc, @@ -299,7 +328,7 @@ jobs: }], allocationPolicy: { serviceAccount: { email: $sa }, - instances: [{ policy: { machineType: $mt, provisioningModel: "SPOT" } }] + instances: [{ policy: { machineType: $mt, provisioningModel: $pm } }] }, logsPolicy: { destination: "CLOUD_LOGGING" } }' > batch-job.json