diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 00000000..c6b72639 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,11 @@ +# Cloud Build upload whitelist — the Dockerfiles only need packages/ + services/ +# + deploy/ as build context. Everything else (.git, node_modules, infra/, +# packages-ts/, tests/, dist/) is irrelevant to the image builds and would bloat +# the source upload. Whitelist pattern: ignore all, then re-include the context. +* +!packages/ +!packages/** +!services/ +!services/** +!deploy/ +!deploy/** diff --git a/infra/batch.tf b/infra/batch.tf index 0c67647d..47dee15d 100644 --- a/infra/batch.tf +++ b/infra/batch.tf @@ -145,6 +145,9 @@ resource "google_cloud_run_v2_job" "weather_incremental" { name = "weather-incremental" location = var.weather_region + # 28-full deploy: allow TF to replace a tainted/failed initial revision. + deletion_protection = false + template { template { service_account = google_service_account.weather_incremental.email diff --git a/infra/budgets.tf b/infra/budgets.tf index 763d861e..2c87859b 100644 --- a/infra/budgets.tf +++ b/infra/budgets.tf @@ -42,16 +42,23 @@ resource "google_monitoring_notification_channel" "budget_pubsub" { } } -# Allow the Cloud Billing budget service agent to publish threshold events to the -# alerts topic (required for the Pub/Sub notification channel to receive budget -# events). billing-budget-alerts@system.gserviceaccount.com is the fixed Cloud -# Billing budget-notifications service account. -resource "google_pubsub_topic_iam_member" "billing_budget_publisher" { - project = var.secrets_project - topic = google_pubsub_topic.budget_alerts.name - role = "roles/pubsub.publisher" - member = "serviceAccount:billing-budget-alerts@system.gserviceaccount.com" -} +# DEFERRED — Pub/Sub budget notifications require the Cloud Billing budget +# service agent `billing-budget-alerts@system.gserviceaccount.com`, which Google +# only auto-provisions after the FIRST budget with a Pub/Sub notification is +# created for the billing account via the Cloud Console (there is no API/CLI to +# force it). Until then this binding fails with "service account does not exist", +# and a budget that references the Pub/Sub channel fails with INVALID_ARGUMENT. +# So budgets below notify by EMAIL only (the essential cost tripwire); the +# Pub/Sub fan-out is re-enabled by: (1) an operator creating one budget with a +# Pub/Sub notification in the Console to provision the system SA, then (2) +# restoring this binding + the budget_pubsub channel in all_updates_rule. +# +# resource "google_pubsub_topic_iam_member" "billing_budget_publisher" { +# project = var.secrets_project +# topic = google_pubsub_topic.budget_alerts.name +# role = "roles/pubsub.publisher" +# member = "serviceAccount:billing-budget-alerts@system.gserviceaccount.com" +# } locals { # Per-project budget definitions: (display, project id, USD cap). The satellite @@ -117,9 +124,12 @@ resource "google_billing_budget" "per_project" { } all_updates_rule { + # EMAIL only until the Cloud Billing budget service agent is provisioned (see + # the deferred billing_budget_publisher note above). Re-add + # google_monitoring_notification_channel.budget_pubsub.id here once the system + # SA exists so budget events also fan out to the alerts topic. monitoring_notification_channels = [ google_monitoring_notification_channel.budget_email.id, - google_monitoring_notification_channel.budget_pubsub.id, ] # Also surface every threshold crossing (not just default schedule) so the # 50% early tripwire always fires a notification. diff --git a/infra/cloud_run.tf b/infra/cloud_run.tf index 75941c27..cb942a63 100644 --- a/infra/cloud_run.tf +++ b/infra/cloud_run.tf @@ -59,6 +59,9 @@ resource "google_cloud_run_v2_service" "earnings_serving" { name = "earnings-serving" location = var.serving_region + # 28-full deploy: allow TF to replace a tainted/failed initial revision. + deletion_protection = false + # Service-level min floor. scheduler.tf PATCHes THIS field # (updateMask=scaling.minInstanceCount) to flip the live-window warm/cool # 0<->1, so min MUST stay at the service level: a revision-level @@ -182,6 +185,9 @@ resource "google_cloud_run_v2_service" "weather_serving" { name = "weather-serving" location = var.serving_region + # 28-full deploy: allow TF to replace a tainted/failed initial revision. + deletion_protection = false + template { # max_instance_count lives in template.scaling (the top-level scaling block # does not accept it under the pinned provider); min stays 0 for idle-cheap. @@ -288,6 +294,9 @@ resource "google_cloud_run_v2_service" "stt" { name = "earnings-stt" location = var.stt_region + # 28-full deploy: allow TF to replace a tainted/failed initial revision. + deletion_protection = false + # google-beta + launch_stage BETA: Cloud Run GPU accelerator fields are a beta # surface. Bounded concurrency ≤ L4 quota (H8). provider = google-beta @@ -478,6 +487,9 @@ resource "google_cloud_run_v2_job" "capture" { name = "earnings-capture" location = var.serving_region + # 28-full deploy: allow TF to replace a tainted/failed initial revision. + deletion_protection = false + template { template { service_account = google_service_account.earnings_capture.email @@ -532,6 +544,9 @@ resource "google_cloud_run_v2_job" "rolefact" { name = "earnings-rolefact" location = var.serving_region + # 28-full deploy: allow TF to replace a tainted/failed initial revision. + deletion_protection = false + template { template { service_account = google_service_account.earnings_rolefact.email diff --git a/infra/deploy_iam.tf b/infra/deploy_iam.tf index 523e1cb1..af1fa30c 100644 --- a/infra/deploy_iam.tf +++ b/infra/deploy_iam.tf @@ -29,7 +29,12 @@ # ===================================================================== # 1. Public invoker on the two serving services (GATE #2, fail-closed auth) # ===================================================================== +# Gated behind var.enable_public_invoker (default false): the org's Domain +# Restricted Sharing policy REJECTS allUsers until an org-admin adds a +# project-scoped allowAll exception on mr-serving. Flip the var true + re-apply +# once that exception is in place to make the services publicly reachable. resource "google_cloud_run_v2_service_iam_member" "earnings_serving_public" { + count = var.enable_public_invoker ? 1 : 0 project = google_cloud_run_v2_service.earnings_serving.project location = google_cloud_run_v2_service.earnings_serving.location name = google_cloud_run_v2_service.earnings_serving.name @@ -38,6 +43,7 @@ resource "google_cloud_run_v2_service_iam_member" "earnings_serving_public" { } resource "google_cloud_run_v2_service_iam_member" "weather_serving_public" { + count = var.enable_public_invoker ? 1 : 0 project = google_cloud_run_v2_service.weather_serving.project location = google_cloud_run_v2_service.weather_serving.location name = google_cloud_run_v2_service.weather_serving.name diff --git a/infra/monitoring.tf b/infra/monitoring.tf index c6921510..601f7466 100644 --- a/infra/monitoring.tf +++ b/infra/monitoring.tf @@ -7,6 +7,32 @@ # mostlyright-satellite project (where the incremental job + its logs are); the # uptime check + channels reference the shared ops channels in budgets.tf. +# --- Per-project notification channels (H6) --- +# A monitoring alert policy can ONLY reference notification channels that live in +# the SAME project. The shared ops channels (budget_email/budget_pubsub) live in +# mostlyright-backend (budgets.tf), so the satellite + ingest alert policies below +# need their OWN same-project email channels (else creation fails 400 +# "channel ... does not match the request's project"). Email is the essential +# paging path; the backend Pub/Sub fan-out is not duplicated per-project (the +# per-project alerts email vu directly). +resource "google_monitoring_notification_channel" "ops_email_satellite" { + project = var.satellite_project_id + display_name = "Phase 28 ops alerts (email) — satellite" + type = "email" + labels = { + email_address = var.budget_notification_email + } +} + +resource "google_monitoring_notification_channel" "ops_email_ingest" { + project = google_project.ingest.project_id + display_name = "Phase 28 ops alerts (email) — ingest" + type = "email" + labels = { + email_address = var.budget_notification_email + } +} + # --- (a) Incremental job FAILED-EXECUTION alert (H6) --- # Fires when the weather-incremental Cloud Run Job records a failed execution. resource "google_monitoring_alert_policy" "incremental_failed" { @@ -36,8 +62,7 @@ resource "google_monitoring_alert_policy" "incremental_failed" { } notification_channels = [ - google_monitoring_notification_channel.budget_email.id, - google_monitoring_notification_channel.budget_pubsub.id, + google_monitoring_notification_channel.ops_email_satellite.id, ] alert_strategy { @@ -57,13 +82,25 @@ resource "google_logging_metric" "r2_partition_age" { name = "weather_r2_newest_partition_age_seconds" filter = "resource.type=\"cloud_run_job\" AND jsonPayload.metric=\"weather.r2.newest_partition_age_seconds\"" + # A log-based metric that EXTRACTS a numeric value (value_extractor) MUST be a + # DISTRIBUTION (the API rejects value_extractor on GAUGE/scalar types). Each run + # logs one age value; the distribution buckets them and the data-freshness alert + # reads a percentile (below). metric_kind is DELTA for distribution log metrics. metric_descriptor { - metric_kind = "GAUGE" - value_type = "DOUBLE" + metric_kind = "DELTA" + value_type = "DISTRIBUTION" unit = "s" } value_extractor = "EXTRACT(jsonPayload.value)" + + bucket_options { + exponential_buckets { + num_finite_buckets = 64 + growth_factor = 2 + scale = 1 + } + } } resource "google_monitoring_alert_policy" "data_freshness" { @@ -80,16 +117,18 @@ resource "google_monitoring_alert_policy" "data_freshness" { threshold_value = var.data_freshness_max_age_days * 86400 duration = "0s" + # DISTRIBUTION metric → use a percentile aligner (ALIGN_MAX is invalid for + # distributions). P99 of the logged partition-age approximates the worst + # (stalest) age in the window. aggregations { alignment_period = "3600s" - per_series_aligner = "ALIGN_MAX" + per_series_aligner = "ALIGN_PERCENTILE_99" } } } notification_channels = [ - google_monitoring_notification_channel.budget_email.id, - google_monitoring_notification_channel.budget_pubsub.id, + google_monitoring_notification_channel.ops_email_satellite.id, ] alert_strategy { @@ -195,8 +234,7 @@ resource "google_monitoring_alert_policy" "capture_dlq_depth" { } notification_channels = [ - google_monitoring_notification_channel.budget_email.id, - google_monitoring_notification_channel.budget_pubsub.id, + google_monitoring_notification_channel.ops_email_ingest.id, ] alert_strategy { diff --git a/infra/runtime_agent_iam.tf b/infra/runtime_agent_iam.tf new file mode 100644 index 00000000..9871d41b --- /dev/null +++ b/infra/runtime_agent_iam.tf @@ -0,0 +1,63 @@ +# Phase 28 (28-full deploy) — IAM that the FIRST live apply proved is required +# but the original root omitted. These were applied by hand during the deploy; +# they are folded in here so a clean `tofu apply` from scratch reproduces a +# working platform (no post-apply manual patching). + +# ===================================================================== +# 1. Cloud Run SERVICE AGENTS need cross-project AR reader (image pull) +# ===================================================================== +# Cloud Run pulls a service/job's container image as the CONSUMING project's +# Google-managed Cloud Run Service Agent (service-@serverless-robot- +# prod), NOT as the deploy or runtime SA. Because the image lives in the +# mostlyright-backend Artifact Registry (cross-project), each consuming project's +# service agent must have artifactregistry.reader on the repo — otherwise service +# creation fails: "Service Agent ... must have permission to read the image". +# (Jobs don't validate at create time, but they still need this to RUN.) +locals { + cloud_run_service_agents = { + serving = "service-${google_project.serving.number}@serverless-robot-prod.iam.gserviceaccount.com" + ingest = "service-${google_project.ingest.number}@serverless-robot-prod.iam.gserviceaccount.com" + satellite = "service-${var.satellite_project_number}@serverless-robot-prod.iam.gserviceaccount.com" + } +} + +resource "google_artifact_registry_repository_iam_member" "cloud_run_agent_reader" { + for_each = local.cloud_run_service_agents + + project = local.ar_project + location = local.ar_location + repository = local.ar_repository + role = "roles/artifactregistry.reader" + member = "serviceAccount:${each.value}" +} + +# ===================================================================== +# 2. weather-backfill RUNTIME SA — Cloud Batch VM roles +# ===================================================================== +# The backfill Batch VMs run as the weather-backfill runtime SA. That SA needs: +# - batch.agentReporter : the on-VM Batch agent reports task state back to the +# Batch service. Without it the VM provisions but "no VM has agent reporting +# correctly" → the job FAILS during scheduling (observed on the first pilot). +# - logging.logWriter : the taskSpec logsPolicy=CLOUD_LOGGING writes stdout. +# - artifactregistry.reader : the VM pulls the weather-backfill image from the +# cross-project backend repo (the VM SA is the puller for Batch, unlike Cloud +# Run which uses the service agent above). +resource "google_project_iam_member" "backfill_agent_reporter" { + project = var.satellite_project_id + role = "roles/batch.agentReporter" + member = "serviceAccount:${google_service_account.weather_backfill.email}" +} + +resource "google_project_iam_member" "backfill_log_writer" { + project = var.satellite_project_id + role = "roles/logging.logWriter" + member = "serviceAccount:${google_service_account.weather_backfill.email}" +} + +resource "google_artifact_registry_repository_iam_member" "backfill_ar_reader" { + project = local.ar_project + location = local.ar_location + repository = local.ar_repository + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.weather_backfill.email}" +} diff --git a/infra/variables.tf b/infra/variables.tf index 2f985794..b0b12a5f 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -304,6 +304,22 @@ variable "api_domain" { default = "api.mostlyright.md" } +# --------------------------------------------------------------------------- +# Public (allUsers) invoker gate — DEFAULT OFF until the org allows it. +# --------------------------------------------------------------------------- +# The two internet-facing serving services (weather-serving, earnings-serving) +# are meant to be public (GATE #2) behind the fail-closed app API key. Binding +# `allUsers` as run.invoker is REJECTED by the org's Domain Restricted Sharing +# policy (constraints/iam.allowedPolicyMemberDomains) until an org-admin adds a +# project-scoped exception on mr-serving (allowAll). Gate the allUsers bindings +# behind this var so a clean apply succeeds meanwhile; flip to true AFTER the +# org policy exception is in place, then re-apply to make the services public. +variable "enable_public_invoker" { + description = "Bind allUsers run.invoker on the two serving services. DEFAULT false: blocked by the org Domain-Restricted-Sharing policy until an org-admin adds a project-scoped allowAll exception on mr-serving; flip true + re-apply afterward." + type = bool + default = false +} + # --------------------------------------------------------------------------- # Container image tags. Every image is pushed to the REUSED Artifact Registry # (var.artifact_registry). The deploy workflows build+push the tag; the Cloud diff --git a/packages/weather/pyproject.toml b/packages/weather/pyproject.toml index ac5c185b..52581f3e 100644 --- a/packages/weather/pyproject.toml +++ b/packages/weather/pyproject.toml @@ -120,8 +120,17 @@ polars = [ # build + unit tests are keyless via mocked transport + synthetic fixtures). satellite = [ "boto3>=1.34,<2.0", - "s3fs>=2024.0", - "gcsfs>=2024.0", + # s3fs and gcsfs each hard-pin `fsspec==` and release in + # lockstep with fsspec, but on their own cadence — so an open `>=` range makes + # pip explode the s3fs×gcsfs×fsspec exact-pin combinatorics and give up with + # ResolutionImpossible (it never reaches the aligned triple within its + # backtrack budget). Pin the triple EXACTLY to the latest co-released, + # py3.12-installable version (2026.6.0 for all three, verified on pypi.org + # 2026-07-04) so there is nothing to backtrack. Bump all three together when + # refreshing. + "fsspec==2026.6.0", + "s3fs==2026.6.0", + "gcsfs==2026.6.0", "h5netcdf>=1.3", "h5py>=3.0", "xarray>=2024.0",