From f4591933a2caf4763f8d84e24d0ca69879e24c38 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 15:13:35 +0200 Subject: [PATCH 1/5] =?UTF-8?q?feat(weather):=20register=20ABI-L2-DSRC=20?= =?UTF-8?q?=E2=80=94=20cheap=20sector=20DSR=20(live-verified)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers the GOES Downward Shortwave Radiation SECTOR product, absent from PRODUCTS despite living on every GOES bucket. Live-verified 2026-07-05 against real samples on noaa-goes16/17/18: - goes16: CONUS grid (166, 394) — the registry baseline - goes17/goes18: PACUS grid (157, 380) — per-satellite split via _DSRC_SECTOR_SHAPES in the shape gate (same accept-only-your-own-variant discipline as the DSRF split; swapped sectors fail loudly; goes19 publishes NO DSRC and falls back to the registry baseline so an unexpected file still fails loud) - all eras on goes_lat_lon_projection (the existing coarse lat/lon extractor branch — no extractor change needed) - hourly cadence, ~70-150 KB files (vs multi-MB full-disk DSRF decode) - goes16 archive: 2019-12 -> 2024 day-108 — the SAME NOAA product-overhaul cutover as the DSRF coarse->full-disk transition; empty listings outside the window are negligible at hourly cadence Why: for 2019-2024 US-station DSR features this is ~30x cheaper to fetch+decode than full-disk DSRF. As a sector product it correctly gets sector semantics from the coverage filter (cross-side excluded; _is_full_disk_goes_product False — pinned by test). Tests: per-satellite sector acceptance, swapped-sector + garbage rejection, registry presence, sector semantics. Co-Authored-By: Claude Opus 4.8 --- .../weather/_fetchers/_goes_extract.py | 27 +++++++++++++ .../mostlyright/weather/_fetchers/_goes_s3.py | 16 ++++++++ .../weather/tests/test_satellite_backfill.py | 12 ++++++ packages/weather/tests/test_satellite_s3.py | 40 +++++++++++++++++++ 4 files changed, 95 insertions(+) diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py index ff99e273..fd537cd0 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py @@ -283,6 +283,33 @@ class ProductVariable: description=("Downward Shortwave Radiation at the surface (full-disk product only)"), tier="v1", ), + ("ABI-L2-DSRC", "DSR"): ProductVariable( + product_code="ABI-L2-DSRC", + variable="DSR", + units_expected="W m-2", + # goes16 CONUS sector grid, verified on a LIVE sample 2026-07-05 + # (OR_ABI-L2-DSRC-M6_G16_s20231671801177). goes17/goes18 publish the + # PACUS sector at (157, 380) — the per-satellite split lives in + # _goes_s3._DSRC_SECTOR_SHAPES (same pattern as the DSRF grid split). + # goes19 publishes NO DSRC (full-disk DSRF only). All eras ship on + # goes_lat_lon_projection (the coarse lat/lon extractor branch). + # goes16 archive spans 2019-12 -> 2024 day-108 (the same NOAA + # product-overhaul cutover as the DSRF coarse->full-disk transition); + # outside that window listings are simply empty (hourly cadence makes + # the empty-listing cost negligible). + grid_shape_expected=(166, 394), + cadence_s=3600, + bounds=(0.0, 1400.0), + dqf_good=frozenset({0}), + dqf_variable="DQF", + is_3d_profile=False, + is_categorical=False, + description=( + "Downward Shortwave Radiation at the surface (CONUS/PACUS sector, " + "hourly ~150 KB files — the cheap DSR complement to full-disk DSRF)" + ), + tier="v1", + ), # ==================== v2 ==================== ("ABI-L2-ACHAC", "HT"): ProductVariable( product_code="ABI-L2-ACHAC", diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index 338c8a3b..d6e524c0 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -203,6 +203,17 @@ # too. _DSRF_GOES16_COARSE_SHAPE: tuple[int, int] = (326, 326) +# ABI-L2-DSRC per-satellite SECTOR grids (live-verified 2026-07-05): goes16 +# publishes the CONUS sector (166, 394) — the registry baseline — while +# goes17/goes18 publish the PACUS sector (157, 380). goes19 publishes no DSRC. +# Same accept-only-your-own-variant discipline as the DSRF split: a swapped +# sector still fails loudly. +_DSRC_SECTOR_SHAPES: dict[str, tuple[int, int]] = { + "goes16": (166, 394), + "goes17": (157, 380), + "goes18": (157, 380), +} + # --------------------------------------------------------------------------- # Rate limiter (identical to iem_mos._RateLimiter / 2i 475-492) @@ -468,6 +479,11 @@ def _validate_dataset_shape(ds: Any, product: str, satellite: str) -> None: continue accepted = {expected} + if product == "ABI-L2-DSRC": + # Per-satellite sector split (goes16 CONUS vs goes17/18 PACUS). + # Unknown satellites (goes19 publishes no DSRC) keep the registry + # baseline so an unexpected file still fails loudly. + accepted = {_DSRC_SECTOR_SHAPES.get(satellite, expected)} if product == "ABI-L2-DSRF": if satellite == "goes16": # goes16 DSRF is BIMODAL: it shipped on the coarse 326x326 lat/lon diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 56d3994d..91bd58b8 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1647,6 +1647,18 @@ def test_partial_year_clamps_only_post_until_months(self) -> None: # unless NO native platform was requested at all (explicit oblique study), # in which case the request is honored fully. # --------------------------------------------------------------------------- +class TestDsrcSectorSemantics: + def test_dsrc_is_not_full_disk(self) -> None: + """DSRC is a SECTOR product (CONUS/PACUS lat/lon grids), so the + cross-side West-for-East exclusion applies to it — unlike DSRF.""" + assert not _backfill._is_full_disk_goes_product("ABI-L2-DSRC") + + def test_dsrc_registered(self) -> None: + from mostlyright.weather._fetchers._goes_extract import PRODUCTS + + assert ("ABI-L2-DSRC", "DSR") in PRODUCTS + + class TestGoesFullDiskCrossSide: """Codex P2 (final stamp): the West-for-East exclusion must be PRODUCT-aware. PACUS/CONUS-sector products truly cannot see East stations, diff --git a/packages/weather/tests/test_satellite_s3.py b/packages/weather/tests/test_satellite_s3.py index 16900de0..d2dd6dc3 100644 --- a/packages/weather/tests/test_satellite_s3.py +++ b/packages/weather/tests/test_satellite_s3.py @@ -400,6 +400,46 @@ def test_shape_validation_dsrf_swapped_grids_rejected() -> None: _goes_s3._validate_dataset_shape(ds_coarse, "ABI-L2-DSRF", "goes19") +# --------------------------------------------------------------------------- +# ABI-L2-DSRC (DSR, CONUS/PACUS sector, hourly ~150 KB) — registered from LIVE +# samples 2026-07-05: goes16 CONUS grid (166, 394); goes17/goes18 PACUS grid +# (157, 380); goes19 publishes NO DSRC (DSRF full-disk only). All eras on +# goes_lat_lon_projection. goes16 archive: 2019-12 -> 2024 day-108 (the same +# NOAA product-overhaul cutover as the DSRF grid transition). +# --------------------------------------------------------------------------- +def _latlon_ds(var_name: str, shape) -> mock.MagicMock: + ds = mock.MagicMock() + var = mock.MagicMock() + var.dims = ("lat", "lon") + var.shape = shape + ds.variables = {var_name: var, "goes_lat_lon_projection": mock.MagicMock()} + ds.__contains__ = lambda self, k: k in ds.variables + ds.__getitem__ = lambda self, k: ds.variables[k] + return ds + + +def test_shape_validation_dsrc_goes16_conus_accepted() -> None: + _goes_s3._validate_dataset_shape(_latlon_ds("DSR", (166, 394)), "ABI-L2-DSRC", "goes16") + + +def test_shape_validation_dsrc_goes_west_pacus_accepted() -> None: + for sat in ("goes17", "goes18"): + _goes_s3._validate_dataset_shape(_latlon_ds("DSR", (157, 380)), "ABI-L2-DSRC", sat) + + +def test_shape_validation_dsrc_swapped_sector_rejected() -> None: + """A PACUS-shaped file labeled goes16 (and vice versa) fails loudly.""" + with pytest.raises(GoesDataCorruptError): + _goes_s3._validate_dataset_shape(_latlon_ds("DSR", (157, 380)), "ABI-L2-DSRC", "goes16") + with pytest.raises(GoesDataCorruptError): + _goes_s3._validate_dataset_shape(_latlon_ds("DSR", (166, 394)), "ABI-L2-DSRC", "goes18") + + +def test_shape_validation_dsrc_garbage_rejected() -> None: + with pytest.raises(GoesDataCorruptError): + _goes_s3._validate_dataset_shape(_latlon_ds("DSR", (999, 999)), "ABI-L2-DSRC", "goes16") + + def test_shape_validation_dsrf_goes16_fulldisk_accepted() -> None: """goes16 DSRF switched coarse-latlon -> full-disk ABI grid (~2024 day-108). From 2196edd543a572ba87abf83c7b22df19b1950600 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 15:19:51 +0200 Subject: [PATCH 2/5] fix(weather): sync no-extra GOES product fallback + drift guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2 on the DSRC registration: _sources._goes_products() carries a hard-coded fallback product set for base installs (no [satellite] extra), and DSRC was only added to the heavy registry — so a base-install/hosted request for a registered product was rejected as unknown before reaching the hosted path. Adds ABI-L2-DSRC to the fallback AND a drift-guard test that parses the fallback literal out of the function source (ast) and pins it equal to the heavy registry's product set — a future registration that forgets the fallback now fails CI loudly instead of shipping a silent base-install rejection. Co-Authored-By: Claude Opus 4.8 --- .../mostlyright/weather/satellite/_sources.py | 7 +++++ .../weather/tests/test_satellite_sources.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/packages/weather/src/mostlyright/weather/satellite/_sources.py b/packages/weather/src/mostlyright/weather/satellite/_sources.py index 985a50d2..144e474f 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_sources.py +++ b/packages/weather/src/mostlyright/weather/satellite/_sources.py @@ -72,6 +72,13 @@ def _goes_products() -> frozenset[str]: "ABI-L2-AODC", "ABI-L2-CTPC", "ABI-L2-DSIC", + # Keep in sync with _goes_extract.PRODUCTS — the no-extra + # drift-guard test (test_satellite_sources.py) asserts this + # fallback equals the heavy registry's product set, so a new + # registration that forgets this list fails CI loudly (Codex + # P2 on the DSRC addition: a base-install/hosted request for a + # registered product was rejected as unknown). + "ABI-L2-DSRC", "ABI-L2-DSRF", "ABI-L2-LSTC", "ABI-L2-LVMPC", diff --git a/packages/weather/tests/test_satellite_sources.py b/packages/weather/tests/test_satellite_sources.py index 2789cce6..61885ee2 100644 --- a/packages/weather/tests/test_satellite_sources.py +++ b/packages/weather/tests/test_satellite_sources.py @@ -156,3 +156,32 @@ def test_goes_satellites_for_longitude_split_boundary_matches_router() -> None: split = _sources._GOES_EAST_WEST_SPLIT_LON_DEG assert _sources.goes_satellites_for_longitude(split) == frozenset({"goes16", "goes19"}) assert _sources.goes_satellites_for_longitude(split - 0.01) == frozenset({"goes17", "goes18"}) + + +def test_goes_fallback_product_set_matches_heavy_registry() -> None: + """DRIFT GUARD (Codex P2, DSRC registration): the no-extra fallback set + inside _goes_products() must equal the heavy registry's product codes — + otherwise a base-install/hosted request for a newly registered product is + rejected as unknown before it can reach the hosted path. This test runs in + the [satellite] lane where BOTH sources are importable and pins them equal. + """ + import ast + import inspect + import textwrap + + from mostlyright.weather._fetchers._goes_extract import _KNOWN_PRODUCTS + + # Parse the literal fallback set out of the function source so the + # ImportError branch is checked WITHOUT having to simulate a missing extra. + src = textwrap.dedent(inspect.getsource(_sources._goes_products)) + literals: set[str] = set() + for node in ast.walk(ast.parse(src)): + if isinstance(node, ast.Set): + literals = { + el.value + for el in node.elts + if isinstance(el, ast.Constant) and isinstance(el.value, str) + } + break + assert literals, "could not locate the fallback set literal" + assert literals == set(_KNOWN_PRODUCTS) From 25494d69ceb47dcf6444cd0e85edb1326b60ecc3 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 18:16:46 +0200 Subject: [PATCH 3/5] fix(weather): parity-audit P1/P2 fixes + probe-measured speed constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Constellation-wide parity audit (8 live-audit agents, every registry cell sampled against real bucket data, 2026-07-05) found 3 P1s that would have silently zeroed data in the full backfill, plus availability-floor waste: P1 — DSRF variants (parity audit): goes17 DSRF is COARSE (326,326) for its ENTIRE 2019-2023 life (retired before the 2024 day-108 overhaul); goes18 DSRF is BIMODAL exactly like goes16 (cutover confirmed at the same day: last coarse 2024/107, first full-disk 2024/108). The gate's bimodal branch was goes16-only — every goes17 DSRF file and all pre-cutover goes18 DSRF raised "rejected before pixel read". New _DSRF_VARIANTS map: goes16/goes18 bimodal (projection-keyed), goes17 coarse-only (a "full-disk goes17" file is mislabeled -> loud), goes19+ full-disk only. P1 — profile size caps: pre-Feb-2024 LVM/LVT files are ~29.4 MiB (live-measured 30,830,368 B; NOAA compressed them to ~7 MiB at 2024 day-038). The 16 MiB default cap zeroed the entire pre-2024 3D-profile era — same failure family as the old ACMC cap. LVMPC/LVTPC join DSRF in the 64 MiB cap set. P2 — per-(satellite, product) availability floors: goes16/17 TPWC/ACHAC/DSRC/LVM/LVT first objects are Dec 2019 (CTPC/DSIC 2019) — the satellite-level 2017 floor alone enumerates years of guaranteed- empty hourly listings. _PRODUCT_AVAILABLE_SINCE + _available_since_for() (max of both floors) wired into the backfill clamp. Floors set conservatively EARLY (too-early only wastes cheap listings). P2 — VIIRS phantom windows: JRR-CloudMask objects begin 2023-06-20 (npp) / 2023-02-19 (n20) / 2023-11-13 (n21), live-listed; the platform-launch floors (2012/2018/2023-01) enumerated up to ~11.5 years of nothing. Speed (probe-measured, in-region Batch job probe-goes-ondemand-121450, n2-standard-8, gcp mirror, sweep 1..64, 20 s wall): knee at 32 workers, derive_rate_cap 118.0 Hz. SOURCE-LIMITS seed records the measurement; shipped constants carry margin: _GOES_S3_RATE_HZ 20->100, _DEFAULT_MAX_WORKERS 8->32 (provenance-lock: shipped <= recorded). Station workflow: n2-standard-16 / 16 workers (compute budget approved). Also: ACMC/AODC era-variable-set comments corrected (2-var ACMC era runs through ~Dec 2021 on goes16, mid-2022 on goes17; AODC gained AE1/AE2 mid-2020). Co-Authored-By: Claude Opus 4.8 --- .github/workflows/run-weather-backfill.yml | 11 ++- .../weather/_fetchers/_goes_extract.py | 6 +- .../mostlyright/weather/_fetchers/_goes_s3.py | 95 +++++++++++++++---- .../weather/_fetchers/_viirs_s3.py | 10 +- .../satellite/SOURCE-LIMITS-satellite.md | 10 +- .../weather/satellite/_backfill.py | 16 +++- .../weather/tests/test_satellite_probe.py | 9 +- packages/weather/tests/test_satellite_s3.py | 69 ++++++++++++++ 8 files changed, 189 insertions(+), 37 deletions(-) diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index 85a6e532..687963fc 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -232,10 +232,13 @@ jobs: # pilot's n2-standard-4, but one station is far cheaper than the # 65-station fleet. Run all year-shards concurrently (parallelism # capped at 48 to match the fleet's Spot-availability ceiling). - MACHINE_TYPE="n2-standard-8" + # n2-standard-16 (compute budget approved 2026-07-05): with the + # spawn process pool + the probe-raised rate cap, decode scales + # with cores — 16 workers on 16 vCPUs. + MACHINE_TYPE="n2-standard-16" PARALLELISM=$(( TASK_COUNT < 48 ? TASK_COUNT : 48 )) - CPU_MILLI=8000 - MEM_MIB=32768 + CPU_MILLI=16000 + MEM_MIB=65536 # Explicit mode (satellites/products/year-window/out all passed). The # CLI decomposes BATCH_TASK_INDEX/COUNT into this task's (station, # year). --max-workers 8 matches the 8-vCPU machine. GOES-East @@ -267,7 +270,7 @@ jobs: COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ --arg ex "$EXECUTOR" \ - '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8","--executor",$ex]') + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","16","--executor",$ex]') else TASK_COUNT=1 # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py index fd537cd0..a51db4d0 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_extract.py @@ -892,8 +892,10 @@ def _extract_from_dataset( if var_name not in ds.variables: # Skip missing variables instead of aborting the entire file. # Real-world GOES-16 files from different time periods have - # different variable sets (e.g. ACMC files from 2017 have only - # BCM+DQF, while 2024 files have BCM+ACM+Cloud_Probabilities+DQF). + # different variable sets. Parity audit (2026-07-05): the 2-variable + # BCM+DQF ACMC era runs 2017 through ~Dec 2021 on goes16 (and + # through ~mid-2022 on goes17) — 3-var files only after that; AODC + # gained AE1/AE2 mid-2020. goes18/goes19 are full-var from birth. # Raising GoesDataCorruptError here would abort extraction for # ALL registered variables in the product, losing even the # variables that DO exist. Verified 2026-04-12 during sprint2i diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index d6e524c0..d6d803f0 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -164,9 +164,50 @@ "goes17": date(2023, 1, 10), } +# Per-PRODUCT availability floors (parity audit P2, live-listed 2026-07-05). +# Several enterprise L2 products rolled out YEARS after a platform's first +# light — e.g. goes16 TPWC/ACHAC/DSRC/LVM/LVT first objects are Dec 2019 — +# so the satellite-level floor alone enumerates ~2.5 years of guaranteed-empty +# hourly listings per product. Floors are set conservatively EARLY (a too-early +# floor only wastes cheap listings; a too-late one silently loses data): +# Dec-2019 products floor at 2019-12-01; CTPC/DSIC (first objects in 2019, +# exact day unverified) floor at 2019-01-01. The Dec-2019 rollout was a +# product-wide NOAA enterprise update, so goes17 carries the same floors. +# Consumed via _available_since_for() (max of satellite + product floors). +_PRODUCT_AVAILABLE_SINCE: dict[tuple[str, str], date] = { + ("goes16", "ABI-L2-TPWC"): date(2019, 12, 1), + ("goes16", "ABI-L2-ACHAC"): date(2019, 12, 1), + ("goes16", "ABI-L2-DSRC"): date(2019, 12, 1), + ("goes16", "ABI-L2-LVMPC"): date(2019, 12, 1), + ("goes16", "ABI-L2-LVTPC"): date(2019, 12, 1), + ("goes16", "ABI-L2-CTPC"): date(2019, 1, 1), + ("goes16", "ABI-L2-DSIC"): date(2019, 1, 1), + ("goes17", "ABI-L2-TPWC"): date(2019, 12, 1), + ("goes17", "ABI-L2-ACHAC"): date(2019, 12, 1), + ("goes17", "ABI-L2-DSRC"): date(2019, 12, 1), + ("goes17", "ABI-L2-LVMPC"): date(2019, 12, 1), + ("goes17", "ABI-L2-LVTPC"): date(2019, 12, 1), + ("goes17", "ABI-L2-CTPC"): date(2019, 1, 1), + ("goes17", "ABI-L2-DSIC"): date(2019, 1, 1), +} + + +def _available_since_for(satellite: str, product: str) -> date | None: + """The effective availability floor for a (satellite, product) pair. + + max() of the satellite first-light floor and the per-product rollout floor + — the backfill clamp skips any month wholly before this with NO I/O. + """ + sat = _AVAILABLE_SINCE.get(satellite) + prod = _PRODUCT_AVAILABLE_SINCE.get((satellite, product)) + if sat is None: + return prod + return max(sat, prod) if prod is not None else sat + + # Conservative top-level S3/GCS call-rate cap (2i ``_GOES_S3_RATE_HZ``). The # empirical concurrency probe (25-05 D10) may later derive a tuned value. -_GOES_S3_RATE_HZ = 20.0 +_GOES_S3_RATE_HZ = 100.0 _MAX_S3_RETRIES = 3 _S3_BACKOFF_BASE_S = 1.0 @@ -190,8 +231,16 @@ # fleet-wide. 16 MiB = >3x headroom over the observed modern sizes while still # bounding the in-memory buffer well below the DSRF cap. _DEFAULT_SIZE_CAP_BYTES = 16 * 1024 * 1024 # 16 MiB +# Products needing a cap above the CONUS-class default: DSRF full-disk +# (~50 MB) and the 3D profile pair — pre-Feb-2024 LVM/LVT files are ~29.4 MiB +# (live-measured 30,830,368 B; NOAA switched them to a compressed ~7 MiB layout +# at 2024 day-038). The 16 MiB default silently zeroed the entire pre-2024 +# profile era "before read" (parity audit P1 — the same failure family as the +# old 2 MiB ACMC cap). +_LARGE_CAP_PRODUCTS: frozenset[str] = frozenset({"ABI-L2-DSRF", "ABI-L2-LVMPC", "ABI-L2-LVTPC"}) _SIZE_CAP_BYTES: dict[str, int] = { - p: (64 * 1024 * 1024 if p == "ABI-L2-DSRF" else _DEFAULT_SIZE_CAP_BYTES) for (p, _v) in PRODUCTS + p: (64 * 1024 * 1024 if p in _LARGE_CAP_PRODUCTS else _DEFAULT_SIZE_CAP_BYTES) + for (p, _v) in PRODUCTS } # The coarse DSRF grid shape (D6 §A.6). The registry carries the full-disk ABI @@ -214,6 +263,20 @@ "goes18": (157, 380), } +# DSRF grid VARIANT per satellite (parity audit, live-verified 2026-07-05): +# - "bimodal": coarse (326,326) lat/lon before the 2024 day-108 NOAA product +# overhaul, full-disk (5424,5424) ABI fixed grid after — keyed per-file on +# the projection variable. goes16 AND goes18 (goes18 cutover confirmed at +# the SAME day: last coarse 2024/107, first full-disk 2024/108). +# - "coarse": coarse for the platform's ENTIRE life. goes17 (2019-2023) died +# before the overhaul — its every DSRF file is (326,326) lat/lon. +# - absent (goes19+): registry full-disk only (born after the overhaul). +_DSRF_VARIANTS: dict[str, str] = { + "goes16": "bimodal", + "goes17": "coarse", + "goes18": "bimodal", +} + # --------------------------------------------------------------------------- # Rate limiter (identical to iem_mos._RateLimiter / 2i 475-492) @@ -485,26 +548,24 @@ def _validate_dataset_shape(ds: Any, product: str, satellite: str) -> None: # baseline so an unexpected file still fails loudly. accepted = {_DSRC_SECTOR_SHAPES.get(satellite, expected)} if product == "ABI-L2-DSRF": - if satellite == "goes16": - # goes16 DSRF is BIMODAL: it shipped on the coarse 326x326 lat/lon - # grid (goes_lat_lon_projection) through early 2024, then switched - # to the full-disk ABI fixed grid (goes_imager_projection, - # 5424x5424) from ~2024 day-108. Key the accepted shape on the - # file's OWN projection variable — the SAME per-file branch - # _extract_from_dataset uses — so each grid must match its own - # expected shape and a mislabeled/swapped grid still fails loudly - # (no silent accept). Before this, goes16 DSRF was hard-coded to - # the coarse shape and every post-cutover full-disk file was - # rejected "before pixel read", stalling the DSRF >= mid-2024 - # backfill. + # Variant per satellite (_DSRF_VARIANTS, parity audit 2026-07-05): + # bimodal platforms (goes16/goes18) key the accepted shape on the + # file's OWN projection variable — the SAME per-file branch + # _extract_from_dataset uses — so each grid era must match its own + # expected shape and a mislabeled/swapped grid still fails loudly. + # goes17 was coarse for its ENTIRE life (died before the 2024 + # day-108 overhaul): coarse is its ONLY accepted shape, and a + # "full-disk goes17" file is mislabeled data. goes19+ (absent from + # the map) is full-disk only. + variant = _DSRF_VARIANTS.get(satellite) + if variant == "bimodal": if "goes_lat_lon_projection" in ds.variables: accepted = {_DSRF_GOES16_COARSE_SHAPE} else: accepted = {expected} + elif variant == "coarse": + accepted = {_DSRF_GOES16_COARSE_SHAPE} else: - # Other satellites (goes19) ship DSRF ONLY on the full-disk ABI - # grid — a coarse/lat-lon file here is swapped/corrupt, so keep the - # registry full-disk shape as the sole accepted variant. accepted = {expected} if actual not in accepted: raise GoesDataCorruptError( diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_viirs_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_viirs_s3.py index ca31023b..39434530 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_viirs_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_viirs_s3.py @@ -67,10 +67,14 @@ #: (the operational launch dates of each JPSS platform: Suomi-NPP 2011-10, #: NOAA-20 2017-11, NOAA-21 2022-11; clamped to the start of the launch year so #: a pre-launch day returns [] without any I/O). +# JRR-CloudMask floors on the NODD buckets, LIVE-LISTED 2026-07-05 (parity +# audit): the platforms flew years earlier, but the JRR product objects only +# begin on these dates — the old platform-launch floors created phantom windows +# (viirs-npp: ~11.5 years of guaranteed-empty listings). _AVAILABLE_SINCE: dict[str, date] = { - "viirs-npp": date(2012, 1, 1), - "viirs-n20": date(2018, 1, 1), - "viirs-n21": date(2023, 1, 1), + "viirs-npp": date(2023, 6, 20), + "viirs-n20": date(2023, 2, 19), + "viirs-n21": date(2023, 11, 13), } # Conservative top-level S3 call-rate cap (shared discipline with GOES/Himawari). diff --git a/packages/weather/src/mostlyright/weather/satellite/SOURCE-LIMITS-satellite.md b/packages/weather/src/mostlyright/weather/satellite/SOURCE-LIMITS-satellite.md index 3faa8aeb..113ea59b 100644 --- a/packages/weather/src/mostlyright/weather/satellite/SOURCE-LIMITS-satellite.md +++ b/packages/weather/src/mostlyright/weather/satellite/SOURCE-LIMITS-satellite.md @@ -1,6 +1,10 @@ # SOURCE-LIMITS.md — satellite (GOES ABI L2, NOAA NODD) -**Status:** CONSERVATIVE-PENDING — no live probe has been run yet. +**Status:** MEASURED — live in-region probe run 2026-07-05 (Cloud Batch job +`probe-goes-ondemand-121450`, us-central1 n2-standard-8 STANDARD, gcp mirror, +sweep 1/4/8/16/32/64, wall 20 s). Knee at 32 workers; derived aggregate rate +cap 118.0 Hz. Shipped constants carry margin below the record +(`_GOES_S3_RATE_HZ = 100.0`, `_DEFAULT_MAX_WORKERS = 32`). **Re-run command:** `python -m mostlyright.weather.satellite probe --mirror aws --out .planning/research` **Valid until:** the first live throughput probe (D10 SAT-25-11) overwrites this with the measured anonymous-throttle / diminishing-returns knee. @@ -37,8 +41,8 @@ conservative) is always safe. **Conservative-pending derived defaults** (the values the shipped constants equal): -- derive_rate_cap: **20.0** Hz -- derive_max_workers: **8** +- derive_rate_cap: **118.0** Hz +- derive_max_workers: **32** ## Inputs to other plans diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index cb39477a..af5e55af 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -73,6 +73,7 @@ _AVAILABLE_SINCE, _AVAILABLE_UNTIL, _GOES_ACTUAL_DATA_END, + _available_since_for, extract_pixel, list_product_keys, ) @@ -112,8 +113,12 @@ #: ``_GOES_S3_RATE_HZ = 20.0``. Its value is the probe-derived ``derive_rate_cap`` #: result recorded in the SOURCE-LIMITS.md satellite entry (provenance: #: SOURCE-LIMITS-satellite.md; conservative-pending the first live probe run). -#: The 25-03 transport's ``_RateLimiter`` consumes this cap. -_GOES_S3_RATE_HZ: float = 20.0 +#: The 25-03 transport's ``_RateLimiter`` consumes this cap. MEASURED 2026-07-05 +#: by the live in-region probe (Batch job probe-goes-ondemand-121450, +#: us-central1 n2-standard-8, gcp mirror, sweep 1..64): knee at 32 workers, +#: derive_rate_cap=118.0 Hz. Shipped at 100.0 (margin below the knee); the +#: provenance-lock test asserts shipped <= the SOURCE-LIMITS record. +_GOES_S3_RATE_HZ: float = 100.0 #: Default fan-out width for ``bulk_backfill``. Its value is the probe-derived #: ``derive_max_workers`` result (the knee of the diminishing-returns curve) @@ -121,7 +126,7 @@ #: SOURCE-LIMITS-satellite.md). Re-measure with #: ``python -m mostlyright.weather.satellite probe``. Conservative-pending the #: first live probe run; the provenance-lock test asserts shipped <= recorded. -_DEFAULT_MAX_WORKERS: int = 8 +_DEFAULT_MAX_WORKERS: int = 32 # --------------------------------------------------------------------------- # Resume-layer constants (ported from satellite_backfill_plan_2i.md:40-56). @@ -288,7 +293,10 @@ def backfill_goes_satellite( # available_since clamp: skip a whole slice before the satellite's # first-light date with no I/O (2i 1320, collapsed to the month grain). if source == _GOES_SOURCE: - available_since = _AVAILABLE_SINCE.get(satellite) + # max(satellite first-light, per-product rollout floor) — several + # enterprise products rolled out years after first light (parity + # audit P2); the product floor kills those guaranteed-empty months. + available_since = _available_since_for(satellite, product) else: available_since = _anon_available_since(source, satellite) last_day_of_month = _last_day_of_month(year, month) diff --git a/packages/weather/tests/test_satellite_probe.py b/packages/weather/tests/test_satellite_probe.py index 3b82731d..77fc7078 100644 --- a/packages/weather/tests/test_satellite_probe.py +++ b/packages/weather/tests/test_satellite_probe.py @@ -174,14 +174,15 @@ def test_read_source_limits_missing_section_returns_none(self, tmp_path) -> None assert _probe.read_source_limits_satellite(no_sat) is None def test_read_source_limits_parses_package_seed(self) -> None: - """The package-co-located conservative-pending seed parses (the file the - Task-1 provenance lock reads).""" + """The package-co-located seed parses (the file the Task-1 provenance + lock reads). Values are the MEASURED 2026-07-05 in-region probe record + (knee 32 workers, 118.0 Hz — job probe-goes-ondemand-121450).""" from mostlyright.weather.satellite import _backfill recorded = _probe.read_source_limits_satellite(_backfill._SOURCE_LIMITS_PATH) assert recorded is not None - assert recorded["rate_hz"] == 20.0 - assert recorded["max_workers"] == 8 + assert recorded["rate_hz"] == 118.0 + assert recorded["max_workers"] == 32 # --------------------------------------------------------------------------- diff --git a/packages/weather/tests/test_satellite_s3.py b/packages/weather/tests/test_satellite_s3.py index d2dd6dc3..7dd88e31 100644 --- a/packages/weather/tests/test_satellite_s3.py +++ b/packages/weather/tests/test_satellite_s3.py @@ -418,6 +418,75 @@ def _latlon_ds(var_name: str, shape) -> mock.MagicMock: return ds +# --------------------------------------------------------------------------- +# Parity-audit P1 regressions (live-verified 2026-07-05): +# - goes17 DSRF is COARSE (326,326) lat/lon for its ENTIRE 2019-2023 life +# (it retired before the 2024 day-108 full-disk overhaul). +# - goes18 DSRF is BIMODAL exactly like goes16 (coarse pre day-108-2024, +# full-disk (5424,5424) after — cutover confirmed at 2024/108). +# - goes19 remains full-disk-only (born after the overhaul). +# - LVMPC/LVTPC pre-Feb-2024 files are ~29.4 MiB — the 16 MiB default cap +# zeroed the entire 3D-profile era; they need an explicit larger cap. +# --------------------------------------------------------------------------- +def test_shape_validation_dsrf_goes17_coarse_accepted() -> None: + ds = mock.MagicMock() + var = mock.MagicMock() + var.dims = ("lat", "lon") + var.shape = (326, 326) + ds.variables = {"DSR": var, "goes_lat_lon_projection": mock.MagicMock()} + ds.__contains__ = lambda self, k: k in ds.variables + ds.__getitem__ = lambda self, k: ds.variables[k] + _goes_s3._validate_dataset_shape(ds, "ABI-L2-DSRF", "goes17") + + +def test_shape_validation_dsrf_goes17_fulldisk_rejected() -> None: + """goes17 died before the overhaul — a full-disk goes17 file is mislabeled.""" + ds = _ds_with_var("DSR", ("y", "x"), (5424, 5424)) + with pytest.raises(GoesDataCorruptError): + _goes_s3._validate_dataset_shape(ds, "ABI-L2-DSRF", "goes17") + + +def test_shape_validation_dsrf_goes18_bimodal_accepted() -> None: + # coarse era (2022 -> 2024 day-107) + ds = mock.MagicMock() + var = mock.MagicMock() + var.dims = ("lat", "lon") + var.shape = (326, 326) + ds.variables = {"DSR": var, "goes_lat_lon_projection": mock.MagicMock()} + ds.__contains__ = lambda self, k: k in ds.variables + ds.__getitem__ = lambda self, k: ds.variables[k] + _goes_s3._validate_dataset_shape(ds, "ABI-L2-DSRF", "goes18") + # full-disk era (2024 day-108 ->) + ds2 = _ds_with_var("DSR", ("y", "x"), (5424, 5424)) + _goes_s3._validate_dataset_shape(ds2, "ABI-L2-DSRF", "goes18") + + +def test_lvm_lvt_size_cap_admits_pre_2024_profiles() -> None: + """Pre-Feb-2024 LVM/LVT files are ~30,830,000 B (29.4 MiB) — live-measured. + The 16 MiB default cap rejected the whole era 'before read'.""" + observed = 30_830_368 # largest live-listed pre-cutover LVTPC/LVMPC size + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-LVMPC"] > observed + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-LVTPC"] > observed + + +def test_product_availability_floor_clamps_phantom_windows() -> None: + """Per-(product, satellite) availability floors (audit P2): TPWC/ACHAC/CTPC/ + DSIC/DSRC/LVMPC/LVTPC do not exist on goes16 before ~Dec 2019 — the + satellite-level 2017 floor alone enumerates ~2.5 years of empty listings.""" + from datetime import date as _d + + f = _goes_s3._available_since_for + # product floor dominates the satellite floor + assert f("goes16", "ABI-L2-TPWC") == _d(2019, 12, 1) + assert f("goes16", "ABI-L2-ACHAC") == _d(2019, 12, 1) + assert f("goes16", "ABI-L2-DSRC") == _d(2019, 12, 1) + assert f("goes16", "ABI-L2-LVMPC") == _d(2019, 12, 1) + # satellite floor dominates when the product predates the satellite + assert f("goes19", "ABI-L2-TPWC") == _d(2024, 11, 15) + # products with full-life coverage keep the satellite floor + assert f("goes16", "ABI-L2-ACMC") == _d(2017, 5, 24) + + def test_shape_validation_dsrc_goes16_conus_accepted() -> None: _goes_s3._validate_dataset_shape(_latlon_ds("DSR", (166, 394)), "ABI-L2-DSRC", "goes16") From 65a90a0d024e9a4524acb5ee57673a3b6740ec5a Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 18:25:16 +0200 Subject: [PATCH 4/5] test(weather): gate the fallback drift guard behind the satellite extra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P1: the drift-guard test imported the heavy _goes_extract registry unguarded inside test_satellite_sources.py — a module that runs in the BASE no-extra fast suite (it tests the dep-free spine), so the default `uv run pytest -m "not live"` broke without the [satellite] extra. pytest.importorskip: skips on base installs, enforces in the satellite-coverage lane — where a drifted fallback would otherwise ship. Co-Authored-By: Claude Opus 4.8 --- packages/weather/tests/test_satellite_sources.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/weather/tests/test_satellite_sources.py b/packages/weather/tests/test_satellite_sources.py index 61885ee2..ea59c112 100644 --- a/packages/weather/tests/test_satellite_sources.py +++ b/packages/weather/tests/test_satellite_sources.py @@ -169,7 +169,15 @@ def test_goes_fallback_product_set_matches_heavy_registry() -> None: import inspect import textwrap - from mostlyright.weather._fetchers._goes_extract import _KNOWN_PRODUCTS + # The heavy registry lives behind the [satellite] extra; this module runs in + # the BASE no-extra fast suite, so skip (not fail) there — the drift guard + # enforces in the satellite-coverage lane, which is where a drifted fallback + # would otherwise ship (Codex P1: an unguarded import broke the base suite). + _goes_extract = pytest.importorskip( + "mostlyright.weather._fetchers._goes_extract", + reason="drift guard needs the [satellite] extra (heavy PRODUCTS registry)", + ) + _KNOWN_PRODUCTS = _goes_extract._KNOWN_PRODUCTS # Parse the literal fallback set out of the function source so the # ImportError branch is checked WITHOUT having to simulate a missing extra. From 29271ba165f4f612bb2d08099b65e47705c4da14 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 18:37:21 +0200 Subject: [PATCH 5/5] fix(weather): share the aggregate GOES rate cap across spawn workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2: _GOES_S3_RATE_HZ is the measured PER-TASK AGGREGATE cap (probe-derived SOURCE-LIMITS record), but the limiter is module-local — under executor=process every spawn worker re-imports _goes_s3 and ran the FULL cap (station workflow's 16 workers ~= 1600 Hz nominal, 13x the measured knee and far past the documented limit). _make_executor's process branch now divides the cap across workers via _MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE (floor 1 Hz), set BEFORE pool creation so spawn children inherit it; _goes_s3 builds its limiter from _effective_rate_hz() at import. Thread mode is unchanged (single process, full cap, shared limiter). Malformed overrides log and fall back to the aggregate cap. Co-Authored-By: Claude Opus 4.8 --- .../mostlyright/weather/_fetchers/_goes_s3.py | 23 ++++++++- .../weather/satellite/_backfill.py | 8 ++++ .../weather/tests/test_satellite_backfill.py | 48 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index d6d803f0..627d18fe 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -298,7 +298,28 @@ def acquire(self) -> None: self._last = time.monotonic() -_DEFAULT_LIMITER = _RateLimiter(1.0 / _GOES_S3_RATE_HZ) +def _effective_rate_hz() -> float: + """The rate this PROCESS may use: the aggregate cap, or its worker share. + + _GOES_S3_RATE_HZ is the measured PER-TASK AGGREGATE cap (SOURCE-LIMITS, + probe 2026-07-05). Under ``executor=process`` every spawn worker re-imports + this module — an unscaled limiter would multiply the cap by the worker + count (Codex P2: 16 workers ~= 16x the documented limit). The parent + divides the cap across workers via this env override BEFORE creating the + pool; spawn children inherit env and each takes its share at import. + """ + import os + + raw = os.environ.get("_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE") + if raw: + try: + return float(raw) + except ValueError: + log.warning("ignoring malformed _MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE=%r", raw) + return _GOES_S3_RATE_HZ + + +_DEFAULT_LIMITER = _RateLimiter(1.0 / _effective_rate_hz()) # --------------------------------------------------------------------------- diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index af5e55af..824beef8 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -801,6 +801,14 @@ def _make_executor(executor: str, max_workers: int) -> Executor: why this never reproduced locally. """ if executor == "process": + # Divide the PER-TASK aggregate GOES rate cap across the workers: each + # spawn child re-imports _goes_s3 and builds its limiter from the env + # override at import (Codex P2 — unscaled, N workers ran N x the + # documented cap). Floor 1 Hz so a huge worker count cannot starve + # children. Set BEFORE pool creation so every child inherits it. + os.environ["_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE"] = str( + max(_GOES_S3_RATE_HZ / max_workers, 1.0) + ) return ProcessPoolExecutor( max_workers=max_workers, mp_context=multiprocessing.get_context("spawn") ) diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 91bd58b8..2ef02414 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1647,6 +1647,54 @@ def test_partial_year_clamps_only_post_until_months(self) -> None: # unless NO native platform was requested at all (explicit oblique study), # in which case the request is honored fully. # --------------------------------------------------------------------------- +class TestProcessModeRateSharing: + """Codex P2 (parity/speed branch): _GOES_S3_RATE_HZ is the measured + PER-TASK AGGREGATE cap, but the limiter is module-local — every spawn + worker re-imports _goes_s3 and would run the FULL cap (16 workers ~= 16x + the documented limit). The process pool must divide the cap across + workers via the env override that spawn children inherit.""" + + def test_process_pool_sets_rate_share_env(self, monkeypatch) -> None: + import os + + monkeypatch.delenv("_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE", raising=False) + pool = _backfill._make_executor("process", 16) + try: + got = float(os.environ["_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE"]) + assert abs(got - _backfill._GOES_S3_RATE_HZ / 16) < 1e-9 + finally: + pool.shutdown(wait=False, cancel_futures=True) + + def test_thread_pool_does_not_set_override(self, monkeypatch) -> None: + import os + + monkeypatch.delenv("_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE", raising=False) + pool = _backfill._make_executor("thread", 16) + try: + assert "_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE" not in os.environ + finally: + pool.shutdown(wait=False, cancel_futures=True) + + def test_effective_rate_honors_override(self, monkeypatch) -> None: + from mostlyright.weather._fetchers import _goes_s3 + + monkeypatch.setenv("_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE", "6.25") + assert _goes_s3._effective_rate_hz() == 6.25 + monkeypatch.delenv("_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE") + assert _goes_s3._effective_rate_hz() == _goes_s3._GOES_S3_RATE_HZ + + def test_override_floor_is_one_hz(self, monkeypatch) -> None: + """A huge worker count must not starve children to ~0 Hz.""" + import os + + monkeypatch.delenv("_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE", raising=False) + pool = _backfill._make_executor("process", 1000) + try: + assert float(os.environ["_MOSTLYRIGHT_GOES_RATE_HZ_OVERRIDE"]) >= 1.0 + finally: + pool.shutdown(wait=False, cancel_futures=True) + + class TestDsrcSectorSemantics: def test_dsrc_is_not_full_disk(self) -> None: """DSRC is a SECTOR product (CONUS/PACUS lat/lon grids), so the