feat(ci): weather-backfill speed knobs — process executor + provisioning

.github/workflows/run-weather-backfill.yml

@@ -63,6 +63,22 @@ on:
         required: true
         default: "goes16,goes17,goes18,goes19"
         type: string
+      provisioning:
+        description: "VM provisioning model. spot = cheap, but capacity-capped and preemptible (a 2020 year-shard lost 104 min of work to preemption on 2026-07-05). standard = on-demand: all tasks start immediately, no preemption — ~4x VM price (n2-standard-8 ≈ $0.39/hr; a full 10-shard station run ≈ +$8). Use standard for measurement runs."
+        required: true
+        default: "spot"
+        type: choice
+        options:
+          - spot
+          - standard
+      executor:
+        description: "Slice executor for mode=station. process = spawn-context worker processes: breaks the HDF5 global-mutex decode serialization (threads decode ONE file at a time regardless of vCPUs) AND multiplies the per-process rate limiter. thread = the older serial-decode path, kept as fallback. First station dispatch after a CLI change should smoke-test process on a cheap 1-year window."
+        required: true
+        default: "process"
+        type: choice
+        options:
+          - process
+          - thread
       year_start:
         description: "First year (inclusive) of the backfill window (mode=pilot and mode=station)."
         required: true
@@ -126,6 +142,8 @@ jobs:
           PILOT_STATION: ${{ inputs.pilot_station }}
           PRODUCTS: ${{ inputs.products }}
           SATELLITES: ${{ inputs.satellites }}
+          PROVISIONING: ${{ inputs.provisioning }}
+          EXECUTOR: ${{ inputs.executor }}
           YEAR_START: ${{ inputs.year_start }}
           YEAR_END: ${{ inputs.year_end }}
           R2_BUCKET: ${{ vars.R2_BUCKET }}
@@ -240,9 +258,16 @@ jobs:
             # image would cartesian-product all four = wrong-hemisphere
             # downloads). available_since clamps make the off-era platform
             # nearly free.
+            # --executor from the workflow input: process (spawn) breaks the
+            # HDF5 decode mutex (PR #100 made spawn safe); thread is the
+            # fallback. Validate the choice defensively.
+            case "$EXECUTOR" in process|thread) ;; *)
+              echo "::error::executor must be process|thread, got '$EXECUTOR'"; exit 1 ;;
+            esac
             COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \
               --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \
-              '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]')
+              --arg ex "$EXECUTOR" \
+              '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8","--executor",$ex]')
           else
             TASK_COUNT=1
             # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single
@@ -264,6 +289,9 @@ jobs:
               --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \
               '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]')
           fi
+          case "$PROVISIONING" in spot) PROV_MODEL="SPOT" ;; standard) PROV_MODEL="STANDARD" ;; *)
+            echo "::error::provisioning must be spot|standard, got '$PROVISIONING'"; exit 1 ;;
+          esac
           jq -n \
             --arg img "${IMAGE}" \
             --argjson tc "${TASK_COUNT}" \
@@ -276,6 +304,7 @@ jobs:
             --arg rb "${R2_BUCKET}" \
             --arg pb "${PROGRESS_BUCKET}" \
             --arg sa "${RUNTIME_SA}" \
+            --arg pm "${PROV_MODEL}" \
             '{
               taskGroups: [{
                 taskCount: $tc,
@@ -299,7 +328,7 @@ jobs:
               }],
               allocationPolicy: {
                 serviceAccount: { email: $sa },
-                instances: [{ policy: { machineType: $mt, provisioningModel: "SPOT" } }]
+                instances: [{ policy: { machineType: $mt, provisioningModel: $pm } }]
               },
               logsPolicy: { destination: "CLOUD_LOGGING" }
             }' > batch-job.json