Phase 28: harden hosted-GCE deploy (SDK dep fix + infra IAM/monitoring/budgets)

.gcloudignore

@@ -0,0 +1,11 @@
+# Cloud Build upload whitelist — the Dockerfiles only need packages/ + services/
+# + deploy/ as build context. Everything else (.git, node_modules, infra/,
+# packages-ts/, tests/, dist/) is irrelevant to the image builds and would bloat
+# the source upload. Whitelist pattern: ignore all, then re-include the context.
+*
+!packages/
+!packages/**
+!services/
+!services/**
+!deploy/
+!deploy/**

infra/batch.tf

@@ -145,6 +145,9 @@ resource "google_cloud_run_v2_job" "weather_incremental" {
   name     = "weather-incremental"
   location = var.weather_region
 
+  # 28-full deploy: allow TF to replace a tainted/failed initial revision.
+  deletion_protection = false
+
   template {
     template {
       service_account = google_service_account.weather_incremental.email

infra/budgets.tf

@@ -42,16 +42,23 @@ resource "google_monitoring_notification_channel" "budget_pubsub" {
   }
 }
 
-# Allow the Cloud Billing budget service agent to publish threshold events to the
-# alerts topic (required for the Pub/Sub notification channel to receive budget
-# events). billing-budget-alerts@system.gserviceaccount.com is the fixed Cloud
-# Billing budget-notifications service account.
-resource "google_pubsub_topic_iam_member" "billing_budget_publisher" {
-  project = var.secrets_project
-  topic   = google_pubsub_topic.budget_alerts.name
-  role    = "roles/pubsub.publisher"
-  member  = "serviceAccount:billing-budget-alerts@system.gserviceaccount.com"
-}
+# DEFERRED — Pub/Sub budget notifications require the Cloud Billing budget
+# service agent `billing-budget-alerts@system.gserviceaccount.com`, which Google
+# only auto-provisions after the FIRST budget with a Pub/Sub notification is
+# created for the billing account via the Cloud Console (there is no API/CLI to
+# force it). Until then this binding fails with "service account does not exist",
+# and a budget that references the Pub/Sub channel fails with INVALID_ARGUMENT.
+# So budgets below notify by EMAIL only (the essential cost tripwire); the
+# Pub/Sub fan-out is re-enabled by: (1) an operator creating one budget with a
+# Pub/Sub notification in the Console to provision the system SA, then (2)
+# restoring this binding + the budget_pubsub channel in all_updates_rule.
+#
+# resource "google_pubsub_topic_iam_member" "billing_budget_publisher" {
+#   project = var.secrets_project
+#   topic   = google_pubsub_topic.budget_alerts.name
+#   role    = "roles/pubsub.publisher"
+#   member  = "serviceAccount:billing-budget-alerts@system.gserviceaccount.com"
+# }
 
 locals {
   # Per-project budget definitions: (display, project id, USD cap). The satellite
@@ -117,9 +124,12 @@ resource "google_billing_budget" "per_project" {
   }
 
   all_updates_rule {
+    # EMAIL only until the Cloud Billing budget service agent is provisioned (see
+    # the deferred billing_budget_publisher note above). Re-add
+    # google_monitoring_notification_channel.budget_pubsub.id here once the system
+    # SA exists so budget events also fan out to the alerts topic.
     monitoring_notification_channels = [
       google_monitoring_notification_channel.budget_email.id,
-      google_monitoring_notification_channel.budget_pubsub.id,
     ]
     # Also surface every threshold crossing (not just default schedule) so the
     # 50% early tripwire always fires a notification.

infra/cloud_run.tf

@@ -59,6 +59,9 @@ resource "google_cloud_run_v2_service" "earnings_serving" {
   name     = "earnings-serving"
   location = var.serving_region
 
+  # 28-full deploy: allow TF to replace a tainted/failed initial revision.
+  deletion_protection = false
+
   # Service-level min floor. scheduler.tf PATCHes THIS field
   # (updateMask=scaling.minInstanceCount) to flip the live-window warm/cool
   # 0<->1, so min MUST stay at the service level: a revision-level
@@ -182,6 +185,9 @@ resource "google_cloud_run_v2_service" "weather_serving" {
   name     = "weather-serving"
   location = var.serving_region
 
+  # 28-full deploy: allow TF to replace a tainted/failed initial revision.
+  deletion_protection = false
+
   template {
     # max_instance_count lives in template.scaling (the top-level scaling block
     # does not accept it under the pinned provider); min stays 0 for idle-cheap.
@@ -288,6 +294,9 @@ resource "google_cloud_run_v2_service" "stt" {
   name     = "earnings-stt"
   location = var.stt_region
 
+  # 28-full deploy: allow TF to replace a tainted/failed initial revision.
+  deletion_protection = false
+
   # google-beta + launch_stage BETA: Cloud Run GPU accelerator fields are a beta
   # surface. Bounded concurrency ≤ L4 quota (H8).
   provider     = google-beta
@@ -478,6 +487,9 @@ resource "google_cloud_run_v2_job" "capture" {
   name     = "earnings-capture"
   location = var.serving_region
 
+  # 28-full deploy: allow TF to replace a tainted/failed initial revision.
+  deletion_protection = false
+
   template {
     template {
       service_account = google_service_account.earnings_capture.email
@@ -532,6 +544,9 @@ resource "google_cloud_run_v2_job" "rolefact" {
   name     = "earnings-rolefact"
   location = var.serving_region
 
+  # 28-full deploy: allow TF to replace a tainted/failed initial revision.
+  deletion_protection = false
+
   template {
     template {
       service_account = google_service_account.earnings_rolefact.email

infra/deploy_iam.tf

@@ -29,7 +29,12 @@
 # =====================================================================
 # 1. Public invoker on the two serving services (GATE #2, fail-closed auth)
 # =====================================================================
+# Gated behind var.enable_public_invoker (default false): the org's Domain
+# Restricted Sharing policy REJECTS allUsers until an org-admin adds a
+# project-scoped allowAll exception on mr-serving. Flip the var true + re-apply
+# once that exception is in place to make the services publicly reachable.
 resource "google_cloud_run_v2_service_iam_member" "earnings_serving_public" {
+  count    = var.enable_public_invoker ? 1 : 0
   project  = google_cloud_run_v2_service.earnings_serving.project
   location = google_cloud_run_v2_service.earnings_serving.location
   name     = google_cloud_run_v2_service.earnings_serving.name
@@ -38,6 +43,7 @@ resource "google_cloud_run_v2_service_iam_member" "earnings_serving_public" {
 }
 
 resource "google_cloud_run_v2_service_iam_member" "weather_serving_public" {
+  count    = var.enable_public_invoker ? 1 : 0
   project  = google_cloud_run_v2_service.weather_serving.project
   location = google_cloud_run_v2_service.weather_serving.location
   name     = google_cloud_run_v2_service.weather_serving.name

infra/monitoring.tf

@@ -7,6 +7,32 @@
 # mostlyright-satellite project (where the incremental job + its logs are); the
 # uptime check + channels reference the shared ops channels in budgets.tf.
 
+# --- Per-project notification channels (H6) ---
+# A monitoring alert policy can ONLY reference notification channels that live in
+# the SAME project. The shared ops channels (budget_email/budget_pubsub) live in
+# mostlyright-backend (budgets.tf), so the satellite + ingest alert policies below
+# need their OWN same-project email channels (else creation fails 400
+# "channel ... does not match the request's project"). Email is the essential
+# paging path; the backend Pub/Sub fan-out is not duplicated per-project (the
+# per-project alerts email vu directly).
+resource "google_monitoring_notification_channel" "ops_email_satellite" {
+  project      = var.satellite_project_id
+  display_name = "Phase 28 ops alerts (email) — satellite"
+  type         = "email"
+  labels = {
+    email_address = var.budget_notification_email
+  }
+}
+
+resource "google_monitoring_notification_channel" "ops_email_ingest" {
+  project      = google_project.ingest.project_id
+  display_name = "Phase 28 ops alerts (email) — ingest"
+  type         = "email"
+  labels = {
+    email_address = var.budget_notification_email
+  }
+}
+
 # --- (a) Incremental job FAILED-EXECUTION alert (H6) ---
 # Fires when the weather-incremental Cloud Run Job records a failed execution.
 resource "google_monitoring_alert_policy" "incremental_failed" {
@@ -36,8 +62,7 @@ resource "google_monitoring_alert_policy" "incremental_failed" {
   }
 
   notification_channels = [
-    google_monitoring_notification_channel.budget_email.id,
-    google_monitoring_notification_channel.budget_pubsub.id,
+    google_monitoring_notification_channel.ops_email_satellite.id,
   ]
 
   alert_strategy {
@@ -57,13 +82,25 @@ resource "google_logging_metric" "r2_partition_age" {
   name    = "weather_r2_newest_partition_age_seconds"
   filter  = "resource.type=\"cloud_run_job\" AND jsonPayload.metric=\"weather.r2.newest_partition_age_seconds\""
 
+  # A log-based metric that EXTRACTS a numeric value (value_extractor) MUST be a
+  # DISTRIBUTION (the API rejects value_extractor on GAUGE/scalar types). Each run
+  # logs one age value; the distribution buckets them and the data-freshness alert
+  # reads a percentile (below). metric_kind is DELTA for distribution log metrics.
   metric_descriptor {
-    metric_kind = "GAUGE"
-    value_type  = "DOUBLE"
+    metric_kind = "DELTA"
+    value_type  = "DISTRIBUTION"
     unit        = "s"
   }
 
   value_extractor = "EXTRACT(jsonPayload.value)"
+
+  bucket_options {
+    exponential_buckets {
+      num_finite_buckets = 64
+      growth_factor      = 2
+      scale              = 1
+    }
+  }
 }
 
 resource "google_monitoring_alert_policy" "data_freshness" {
@@ -80,16 +117,18 @@ resource "google_monitoring_alert_policy" "data_freshness" {
       threshold_value = var.data_freshness_max_age_days * 86400
       duration        = "0s"
 
+      # DISTRIBUTION metric → use a percentile aligner (ALIGN_MAX is invalid for
+      # distributions). P99 of the logged partition-age approximates the worst
+      # (stalest) age in the window.
       aggregations {
         alignment_period   = "3600s"
-        per_series_aligner = "ALIGN_MAX"
+        per_series_aligner = "ALIGN_PERCENTILE_99"
       }
     }
   }
 
   notification_channels = [
-    google_monitoring_notification_channel.budget_email.id,
-    google_monitoring_notification_channel.budget_pubsub.id,
+    google_monitoring_notification_channel.ops_email_satellite.id,
   ]
 
   alert_strategy {
@@ -195,8 +234,7 @@ resource "google_monitoring_alert_policy" "capture_dlq_depth" {
   }
 
   notification_channels = [
-    google_monitoring_notification_channel.budget_email.id,
-    google_monitoring_notification_channel.budget_pubsub.id,
+    google_monitoring_notification_channel.ops_email_ingest.id,
   ]
 
   alert_strategy {

infra/runtime_agent_iam.tf

@@ -0,0 +1,63 @@
+# Phase 28 (28-full deploy) — IAM that the FIRST live apply proved is required
+# but the original root omitted. These were applied by hand during the deploy;
+# they are folded in here so a clean `tofu apply` from scratch reproduces a
+# working platform (no post-apply manual patching).
+
+# =====================================================================
+# 1. Cloud Run SERVICE AGENTS need cross-project AR reader (image pull)
+# =====================================================================
+# Cloud Run pulls a service/job's container image as the CONSUMING project's
+# Google-managed Cloud Run Service Agent (service-<projnum>@serverless-robot-
+# prod), NOT as the deploy or runtime SA. Because the image lives in the
+# mostlyright-backend Artifact Registry (cross-project), each consuming project's
+# service agent must have artifactregistry.reader on the repo — otherwise service
+# creation fails: "Service Agent ... must have permission to read the image".
+# (Jobs don't validate at create time, but they still need this to RUN.)
+locals {
+  cloud_run_service_agents = {
+    serving   = "service-${google_project.serving.number}@serverless-robot-prod.iam.gserviceaccount.com"
+    ingest    = "service-${google_project.ingest.number}@serverless-robot-prod.iam.gserviceaccount.com"
+    satellite = "service-${var.satellite_project_number}@serverless-robot-prod.iam.gserviceaccount.com"
+  }
+}
+
+resource "google_artifact_registry_repository_iam_member" "cloud_run_agent_reader" {
+  for_each = local.cloud_run_service_agents
+
+  project    = local.ar_project
+  location   = local.ar_location
+  repository = local.ar_repository
+  role       = "roles/artifactregistry.reader"
+  member     = "serviceAccount:${each.value}"
+}
+
+# =====================================================================
+# 2. weather-backfill RUNTIME SA — Cloud Batch VM roles
+# =====================================================================
+# The backfill Batch VMs run as the weather-backfill runtime SA. That SA needs:
+#   - batch.agentReporter : the on-VM Batch agent reports task state back to the
+#     Batch service. Without it the VM provisions but "no VM has agent reporting
+#     correctly" → the job FAILS during scheduling (observed on the first pilot).
+#   - logging.logWriter   : the taskSpec logsPolicy=CLOUD_LOGGING writes stdout.
+#   - artifactregistry.reader : the VM pulls the weather-backfill image from the
+#     cross-project backend repo (the VM SA is the puller for Batch, unlike Cloud
+#     Run which uses the service agent above).
+resource "google_project_iam_member" "backfill_agent_reporter" {
+  project = var.satellite_project_id
+  role    = "roles/batch.agentReporter"
+  member  = "serviceAccount:${google_service_account.weather_backfill.email}"
+}
+
+resource "google_project_iam_member" "backfill_log_writer" {
+  project = var.satellite_project_id
+  role    = "roles/logging.logWriter"
+  member  = "serviceAccount:${google_service_account.weather_backfill.email}"
+}
+
+resource "google_artifact_registry_repository_iam_member" "backfill_ar_reader" {
+  project    = local.ar_project
+  location   = local.ar_location
+  repository = local.ar_repository
+  role       = "roles/artifactregistry.reader"
+  member     = "serviceAccount:${google_service_account.weather_backfill.email}"
+}

infra/variables.tf

@@ -304,6 +304,22 @@ variable "api_domain" {
   default     = "api.mostlyright.md"
 }
 
+# ---------------------------------------------------------------------------
+# Public (allUsers) invoker gate — DEFAULT OFF until the org allows it.
+# ---------------------------------------------------------------------------
+# The two internet-facing serving services (weather-serving, earnings-serving)
+# are meant to be public (GATE #2) behind the fail-closed app API key. Binding
+# `allUsers` as run.invoker is REJECTED by the org's Domain Restricted Sharing
+# policy (constraints/iam.allowedPolicyMemberDomains) until an org-admin adds a
+# project-scoped exception on mr-serving (allowAll). Gate the allUsers bindings
+# behind this var so a clean apply succeeds meanwhile; flip to true AFTER the
+# org policy exception is in place, then re-apply to make the services public.
+variable "enable_public_invoker" {
+  description = "Bind allUsers run.invoker on the two serving services. DEFAULT false: blocked by the org Domain-Restricted-Sharing policy until an org-admin adds a project-scoped allowAll exception on mr-serving; flip true + re-apply afterward."
+  type        = bool
+  default     = false
+}
+
 # ---------------------------------------------------------------------------
 # Container image tags. Every image is pushed to the REUSED Artifact Registry
 # (var.artifact_registry). The deploy workflows build+push the tag; the Cloud

packages/weather/pyproject.toml

@@ -120,8 +120,17 @@ polars = [
 # build + unit tests are keyless via mocked transport + synthetic fixtures).
 satellite = [
     "boto3>=1.34,<2.0",
-    "s3fs>=2024.0",
-    "gcsfs>=2024.0",
+    # s3fs and gcsfs each hard-pin `fsspec==<their-own-release>` and release in
+    # lockstep with fsspec, but on their own cadence — so an open `>=` range makes
+    # pip explode the s3fs×gcsfs×fsspec exact-pin combinatorics and give up with
+    # ResolutionImpossible (it never reaches the aligned triple within its
+    # backtrack budget). Pin the triple EXACTLY to the latest co-released,
+    # py3.12-installable version (2026.6.0 for all three, verified on pypi.org
+    # 2026-07-04) so there is nothing to backtrack. Bump all three together when
+    # refreshing.
+    "fsspec==2026.6.0",
+    "s3fs==2026.6.0",
+    "gcsfs==2026.6.0",
     "h5netcdf>=1.3",
     "h5py>=3.0",
     "xarray>=2024.0",