diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index 7bb02fb..85a6e53 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -63,6 +63,22 @@ on: required: true default: "goes16,goes17,goes18,goes19" type: string + provisioning: + description: "VM provisioning model. spot = cheap, but capacity-capped and preemptible (a 2020 year-shard lost 104 min of work to preemption on 2026-07-05). standard = on-demand: all tasks start immediately, no preemption — ~4x VM price (n2-standard-8 ≈ $0.39/hr; a full 10-shard station run ≈ +$8). Use standard for measurement runs." + required: true + default: "spot" + type: choice + options: + - spot + - standard + executor: + description: "Slice executor for mode=station. process = spawn-context worker processes: breaks the HDF5 global-mutex decode serialization (threads decode ONE file at a time regardless of vCPUs) AND multiplies the per-process rate limiter. thread = the older serial-decode path, kept as fallback. First station dispatch after a CLI change should smoke-test process on a cheap 1-year window." + required: true + default: "process" + type: choice + options: + - process + - thread year_start: description: "First year (inclusive) of the backfill window (mode=pilot and mode=station)." required: true @@ -126,6 +142,8 @@ jobs: PILOT_STATION: ${{ inputs.pilot_station }} PRODUCTS: ${{ inputs.products }} SATELLITES: ${{ inputs.satellites }} + PROVISIONING: ${{ inputs.provisioning }} + EXECUTOR: ${{ inputs.executor }} YEAR_START: ${{ inputs.year_start }} YEAR_END: ${{ inputs.year_end }} R2_BUCKET: ${{ vars.R2_BUCKET }} @@ -240,9 +258,16 @@ jobs: # image would cartesian-product all four = wrong-hemisphere # downloads). available_since clamps make the off-era platform # nearly free. + # --executor from the workflow input: process (spawn) breaks the + # HDF5 decode mutex (PR #100 made spawn safe); thread is the + # fallback. Validate the choice defensively. + case "$EXECUTOR" in process|thread) ;; *) + echo "::error::executor must be process|thread, got '$EXECUTOR'"; exit 1 ;; + esac COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ - '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') + --arg ex "$EXECUTOR" \ + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8","--executor",$ex]') else TASK_COUNT=1 # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single @@ -264,6 +289,9 @@ jobs: --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') fi + case "$PROVISIONING" in spot) PROV_MODEL="SPOT" ;; standard) PROV_MODEL="STANDARD" ;; *) + echo "::error::provisioning must be spot|standard, got '$PROVISIONING'"; exit 1 ;; + esac jq -n \ --arg img "${IMAGE}" \ --argjson tc "${TASK_COUNT}" \ @@ -276,6 +304,7 @@ jobs: --arg rb "${R2_BUCKET}" \ --arg pb "${PROGRESS_BUCKET}" \ --arg sa "${RUNTIME_SA}" \ + --arg pm "${PROV_MODEL}" \ '{ taskGroups: [{ taskCount: $tc, @@ -299,7 +328,7 @@ jobs: }], allocationPolicy: { serviceAccount: { email: $sa }, - instances: [{ policy: { machineType: $mt, provisioningModel: "SPOT" } }] + instances: [{ policy: { machineType: $mt, provisioningModel: $pm } }] }, logsPolicy: { destination: "CLOUD_LOGGING" } }' > batch-job.json