From e49f128f721da75cb5961ff6e6a5c316bd64be74 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 11:05:08 +0200 Subject: [PATCH 1/9] fix(weather): era-complete GOES coverage + hemisphere visibility filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observed live (station backfill 28734982038): KNYC 2026 completed ALL-EMPTY because the fleet requests goes16 only, and GOES-16 stopped being GOES-East in 2025 (goes19 took over). The West side has the mirror hole: goes18 only reaches back to 2022-07, so West stations were missing the 2018-2022 goes17 era. And the roster defaults ("goes16","goes18") carried a latent fleet-cost pathology: _enumerate_slices was a plain cartesian product, so every East station would fetch the ENTIRE West archive (and vice versa) — whole-month downloads whose extraction fails out-of-grid only AFTER the bytes are fetched. - _sources: add goes_satellites_for_longitude() — East {goes16,goes19} / West {goes17,goes18}, keyed on the SAME _GOES_EAST_WEST_SPLIT_LON_DEG as the live router (single source of truth; boundary test pins the agreement). - _backfill._enumerate_slices: GOES hemisphere visibility filter — a GOES satellite enumerates for a station only on its own side of the split. Skips are logged loudly (once per satellite), never silent. Non-GOES sources are untouched (they own their coverage). - __main__._DEFAULT_ROSTER_SATELLITES -> (goes16, goes17, goes18, goes19): era-complete; the filter keeps each station on its side pair and the available_since clamp skips pre-first-light slices with no I/O. - run-weather-backfill.yml station+pilot modes pass the full four (NOTE: requires an image built after this commit — an old image would cartesian-product them). Transport already supported all four platforms (buckets + available_since for goes17/goes19 were registered); only the request layer was era-blind. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/run-weather-backfill.yml | 18 ++++++-- .../mostlyright/weather/satellite/__main__.py | 21 ++++++--- .../weather/satellite/_backfill.py | 35 ++++++++++++++- .../mostlyright/weather/satellite/_sources.py | 28 ++++++++++++ .../weather/tests/test_satellite_backfill.py | 44 +++++++++++++++++++ .../weather/tests/test_satellite_sources.py | 26 +++++++++++ 6 files changed, 160 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index 723f134b..c347935d 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -226,9 +226,17 @@ jobs: # proven prod path (the 2024 pilot processed + uploaded DSRF with # threads); numpy/h5py release the GIL for the heavy decode. Re-enable # process only after the spawn-context fix is proven in a container. + # Era-complete GOES set (goes19-era coverage fix): all four GOES + # platforms so East stations keep coverage past the 2025 + # goes16->goes19 handover and West stations gain the goes17 archive + # era. Requires an image built AFTER the hemisphere filter landed + # (the CLI enumerates only each station's own side pair; an OLD + # image would cartesian-product all four = wrong-hemisphere + # downloads). available_since clamps make the off-era platform + # nearly free. COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" \ - '["--mirror","gcp","--satellites","goes16","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') + '["--mirror","gcp","--satellites","goes16,goes17,goes18,goes19","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') else TASK_COUNT=1 # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single @@ -242,11 +250,13 @@ jobs: YEAR_END_EFF="${YEAR_END:-$(date -u +%Y)}" # Pilot: explicit single-station backfill. The CLI's explicit mode # (no --roster) requires satellites/products/year-window/out, so pass - # them all. GOES-East ACMC covers the default KNYC pilot; an - # international pilot station needs --satellites overridden. + # them all. Era-complete GOES set — the CLI's hemisphere filter + # keeps the station on its own side pair (requires an image built + # after the filter landed); an international pilot station still + # needs --satellites overridden (Himawari/Meteosat). COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" \ - '["--mirror","gcp","--satellites","goes16","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') + '["--mirror","gcp","--satellites","goes16,goes17,goes18,goes19","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') fi jq -n \ --arg img "${IMAGE}" \ diff --git a/packages/weather/src/mostlyright/weather/satellite/__main__.py b/packages/weather/src/mostlyright/weather/satellite/__main__.py index 0f54322a..e403f456 100644 --- a/packages/weather/src/mostlyright/weather/satellite/__main__.py +++ b/packages/weather/src/mostlyright/weather/satellite/__main__.py @@ -50,11 +50,17 @@ #: Default satellites for roster/incremental mode. batch.tf passes NO #: ``--satellites``, so roster mode must supply the canonical native-ring -#: default. The ACMC-only MVP (project memory) runs on the GOES East/West pair — -#: the two operational GOES platforms — so the fleet covers the Americas span the -#: settlement stations concentrate in. (Explicit-args mode still requires +#: default. ERA-COMPLETE GOES set (goes19-era coverage fix): the East pair +#: goes16 (operational 2017 -> standby 2025) + goes19 (GOES-East since the 2025 +#: handover) AND the West pair goes17 (archive, -> storage 2023) + goes18. The +#: old ("goes16", "goes18") pair silently lost East coverage after the 2025 +#: handover (observed live: KNYC 2026 backfilled ALL-EMPTY) and never had +#: goes17-era West coverage (2018-2022). Requesting all four is cheap: the +#: backfill hemisphere filter (_backfill._enumerate_slices) enumerates only the +#: station's own side pair, and the ``available_since`` clamp skips +#: pre-first-light slices with no I/O. (Explicit-args mode still requires #: ``--satellites``; this default ONLY applies when ``--roster`` is given.) -_DEFAULT_ROSTER_SATELLITES: tuple[str, ...] = ("goes16", "goes18") +_DEFAULT_ROSTER_SATELLITES: tuple[str, ...] = ("goes16", "goes17", "goes18", "goes19") #: Default products for roster/incremental mode: the ACMC-only MVP cloud mask #: (``ABI-L2-ACMC``, the cheap CONUS product — matches the ``_sources`` GOES @@ -96,9 +102,10 @@ def _build_parser() -> argparse.ArgumentParser: "--satellites", type=_split_csv, default=None, - help="Comma-separated satellites, e.g. goes16,goes19 (GOES-East) or goes18 " - "(GOES-West). REQUIRED in explicit mode; in --roster mode defaults to the " - "native-ring default (goes16,goes18 — the operational GOES pair).", + help="Comma-separated satellites, e.g. goes16,goes19 (GOES-East) or " + "goes17,goes18 (GOES-West). REQUIRED in explicit mode; in --roster mode " + "defaults to the era-complete GOES set (goes16,goes17,goes18,goes19 — " + "the hemisphere filter keeps each station on its own side pair).", ) bf.add_argument( "--products", diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index b498c1a8..e1bf9f88 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -882,14 +882,47 @@ def _enumerate_slices( year_start: int, year_end: int, ) -> list[tuple[str, str, StationInfo, int, int]]: - """Enumerate per-(satellite, product, station, year, month) slices.""" + """Enumerate per-(satellite, product, station, year, month) slices. + + GOES hemisphere visibility filter (goes19-era coverage fix): a GOES + satellite is enumerated for a station ONLY when it sits on the station's + side of the East/West split (``_sources.goes_satellites_for_longitude`` — + the same split the live router uses). Without this, a multi-satellite + request cartesian-produced e.g. KNYC x goes18: whole-month WEST-archive + downloads whose extraction fails out-of-grid only AFTER the bytes are + fetched — pure spend, zero rows. Non-GOES satellites (Himawari / VIIRS / + Meteosat) are NOT filtered here; their sources own their coverage. Skips + are logged LOUDLY (once per satellite) — never silent under-coverage. + """ + from . import _sources + out: list[tuple[str, str, StationInfo, int, int]] = [] + skipped: dict[str, list[str]] = {} + goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites for sat in satellites: for product in products: for info in station_infos: + if sat in goes_sats and sat not in _sources.goes_satellites_for_longitude( + info.longitude + ): + stations_for_sat = skipped.setdefault(sat, []) + if info.icao not in stations_for_sat: + stations_for_sat.append(info.icao) + continue for year in range(year_start, year_end + 1): for month in range(1, 13): out.append((sat, product, info, year, month)) + for sat, icaos in skipped.items(): + log.warning( + "GOES hemisphere filter: NOT enumerating %s for station(s) %s — " + "wrong side of the East/West split (lon %s %.1f); the station's own " + "side pair covers it. This skip avoids whole-archive downloads that " + "fail out-of-grid after fetch.", + sat, + ",".join(icaos), + "<" if sat in _sources._GOES_WEST_SATELLITES else ">=", + _sources._GOES_EAST_WEST_SPLIT_LON_DEG, + ) return out diff --git a/packages/weather/src/mostlyright/weather/satellite/_sources.py b/packages/weather/src/mostlyright/weather/satellite/_sources.py index ef4940eb..985a50d2 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_sources.py +++ b/packages/weather/src/mostlyright/weather/satellite/_sources.py @@ -416,6 +416,34 @@ def _goes_route(longitude: float) -> RouteResult: return RouteResult(source="noaa_goes", satellite=satellite, product=spec.default_product) +#: GOES era pairs by side of the East/West split: every platform that EVER +#: served that side. East = goes16 (operational 2017 -> standby 2025, when +#: goes19 took over as GOES-East) + goes19; West = goes17 (operational +#: 2019 -> storage 2023, goes18 took over) + goes18. Backfills request the +#: whole side pair for era-complete coverage — the per-slice +#: ``available_since`` clamp skips pre-first-light slices with no I/O, so the +#: extra platform is nearly free — while the hemisphere filter +#: (``_backfill._enumerate_slices``) keys on +#: :func:`goes_satellites_for_longitude` so a station NEVER enumerates the +#: other side's archive (whole-month downloads that fail out-of-grid only +#: AFTER the bytes are fetched). +_GOES_EAST_SATELLITES: frozenset[str] = frozenset({"goes16", "goes19"}) +_GOES_WEST_SATELLITES: frozenset[str] = frozenset({"goes17", "goes18"}) + + +def goes_satellites_for_longitude(longitude: float) -> frozenset[str]: + """Return every GOES platform that ever covered ``longitude``'s side. + + Uses the SAME ``_GOES_EAST_WEST_SPLIT_LON_DEG`` split as + :func:`_goes_route` (single source of truth): ``longitude < split`` is + GOES-West territory, else GOES-East — so the live router and the backfill + visibility filter can never disagree on a station's side. + """ + if longitude < _GOES_EAST_WEST_SPLIT_LON_DEG: + return _GOES_WEST_SATELLITES + return _GOES_EAST_SATELLITES + + def _default_route(source: str) -> RouteResult: """Build the spec-default route for a source (no per-platform override).""" spec = _SOURCES[source] diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 9198bbb0..a61462b7 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1521,3 +1521,47 @@ def test_is_month_fully_elapsed_predicate(self) -> None: assert _backfill._is_month_fully_elapsed(now.year, now.month) is False # A far-future month is NOT fully elapsed. assert _backfill._is_month_fully_elapsed(now.year + 5, 1) is False + + +# --------------------------------------------------------------------------- +# GOES hemisphere visibility filter (goes19-era coverage fix): _enumerate_slices +# must intersect GOES satellites with the station's side of the East/West split +# so a KNYC backfill never enumerates goes17/goes18 slices (whole-archive +# downloads that fail per-file out-of-grid AFTER the bytes are fetched) and a +# KSEA backfill never enumerates goes16/goes19. Non-GOES satellites are not +# filtered here (their sources own their coverage). +# --------------------------------------------------------------------------- +class TestGoesHemisphereFilter: + def test_east_station_only_east_satellites(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert sats == {"goes16", "goes19"} + + def test_west_station_only_west_satellites(self) -> None: + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert sats == {"goes17", "goes18"} + + def test_mixed_stations_get_their_own_pairs(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC", "KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + by_station: dict[str, set[str]] = {} + for sat, _product, info, _y, _m in slices: + by_station.setdefault(info.icao, set()).add(sat) + assert by_station["KNYC"] == {"goes16", "goes19"} + assert by_station["KSEA"] == {"goes17", "goes18"} + + def test_single_requested_satellite_matching_hemisphere_unchanged(self) -> None: + # The pre-fix pilot shape (--satellites goes16 for KNYC) is untouched: + # 12 months x 1 product x 1 satellite. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2024, 2024) + assert len(slices) == 12 diff --git a/packages/weather/tests/test_satellite_sources.py b/packages/weather/tests/test_satellite_sources.py index 8effda60..2789cce6 100644 --- a/packages/weather/tests/test_satellite_sources.py +++ b/packages/weather/tests/test_satellite_sources.py @@ -130,3 +130,29 @@ def test_validate_satellite_and_product_rejects_cross_source_product() -> None: # An ABI product on a Himawari satellite is a category error. with pytest.raises(ValueError): _sources.validate_satellite_and_product("himawari9", "ABI-L2-ACMC") + + +# --------------------------------------------------------------------------- +# GOES era-set helper: which GOES platforms EVER covered a longitude. The +# backfill visibility filter (_backfill._enumerate_slices) keys on this so an +# East-CONUS station never enumerates (and downloads!) the West archive and +# vice versa, while era completeness (goes16->goes19 East handover 2025, +# goes17->goes18 West handover 2022/23) comes from requesting the full pair +# and letting the available_since clamp skip pre-first-light slices free. +# --------------------------------------------------------------------------- +def test_goes_satellites_for_longitude_east() -> None: + # KNYC (-73.97): GOES-East era pair — goes16 (2017..2025 standby) + goes19. + assert _sources.goes_satellites_for_longitude(-73.97) == frozenset({"goes16", "goes19"}) + + +def test_goes_satellites_for_longitude_west() -> None: + # KSEA (-122.31): GOES-West era pair — goes17 (archive) + goes18. + assert _sources.goes_satellites_for_longitude(-122.31) == frozenset({"goes17", "goes18"}) + + +def test_goes_satellites_for_longitude_split_boundary_matches_router() -> None: + # Exactly at the split: _goes_route sends lon < SPLIT to goes18 (West), so + # the era-set helper must agree — lon == SPLIT is EAST on both. + split = _sources._GOES_EAST_WEST_SPLIT_LON_DEG + assert _sources.goes_satellites_for_longitude(split) == frozenset({"goes16", "goes19"}) + assert _sources.goes_satellites_for_longitude(split - 0.01) == frozenset({"goes17", "goes18"}) From 2d95d34d31e89bdeee3e3da836bee969ccc72940 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 11:15:30 +0200 Subject: [PATCH 2/9] fix(weather): operational-era END clamp for retired GOES platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2 on the era-coverage branch: goes_satellites_for_longitude returns every platform that EVER served a side, and the only downstream clamp was _AVAILABLE_SINCE — so retired platforms (goes17 post-2023 storage, goes16 post-2025 handover) kept enumerating post-retirement months and burning empty per-hour listing calls, growing every future fleet year. - _goes_s3._AVAILABLE_UNTIL: the symmetric counterpart of _AVAILABLE_SINCE, RETIRED platforms only (goes17 -> 2024-01-01, goes16 -> 2026-01-01). SAFETY over savings: ~1 yr margin past the public retirement dates (a too-early date silently skips real settlement-adjacent data; a too-late one only wastes cheap listings), and the goes16 date is observation-backed (2026 Jan-Jun measured ALL-EMPTY live). goes18/goes19 deliberately absent. - _enumerate_slices: month-granular clamp at ENUMERATION — clamped slices are never submitted, write NO progress marker (raising/removing a date later backfills them normally), and are logged loudly per satellite. - Tests: 6-case TestGoesOperationalUntilClamp (per-era keep/drop, operational platforms never clamped, month-granular partial window); hemisphere tests re-anchored to 2022 (in-era for all four platforms); the two future-slice resume tests switched goes16->goes19 so the 2099 window still genuinely exercises resume semantics instead of passing vacuously under the clamp. Co-Authored-By: Claude Opus 4.8 --- .../mostlyright/weather/_fetchers/_goes_s3.py | 26 +++++++ .../weather/satellite/_backfill.py | 23 ++++++ .../weather/tests/test_satellite_backfill.py | 76 +++++++++++++++++-- 3 files changed, 120 insertions(+), 5 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index df777d61..25efd8ff 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -120,6 +120,32 @@ "goes19": date(2024, 11, 15), } +# Operational-era END clamps for RETIRED platforms (goes19-era coverage fix, +# Codex P2) — the symmetric counterpart of _AVAILABLE_SINCE, consumed by the +# backfill's ENUMERATION filter so post-retirement (satellite, month) slices are +# never even submitted (each otherwise burns ~a task-minute of empty per-hour +# listing calls, growing every future fleet year). +# +# SAFETY over savings: a too-early date here would SILENTLY skip real +# settlement-adjacent data, while a too-late date only wastes cheap empty +# listings — so every entry carries ~1 YEAR of margin past the public +# retirement date, skips are logged loudly at enumeration, and enumeration-skip +# writes NO progress marker (removing/raising a date later backfills those +# months normally). Amend on any NOAA reactivation (standby platforms have +# historically been reactivated during operational anomalies). +# +# - goes17: operational GOES-West ended 2023-01-10 (goes18 took over +# 2023-01-04; goes17 to storage). Clamp 2024-01-01 = ~1 yr margin. +# - goes16: goes19 became the operational GOES-East 2025-04-07 (goes16 to +# standby). Clamp 2026-01-01 = ~9 mo margin, AND 2026 Jan-Jun was measured +# ALL-EMPTY live (KNYC station backfill, 2026-07-05) — the clamp is +# observation-backed, not date-arithmetic alone. +# - goes18 / goes19: OPERATIONAL — deliberately absent (no end clamp). +_AVAILABLE_UNTIL: dict[str, date] = { + "goes16": date(2026, 1, 1), + "goes17": date(2024, 1, 1), +} + # Conservative top-level S3/GCS call-rate cap (2i ``_GOES_S3_RATE_HZ``). The # empirical concurrency probe (25-05 D10) may later derive a tuned value. _GOES_S3_RATE_HZ = 20.0 diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index e1bf9f88..bbe9f9cc 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -70,6 +70,7 @@ # below references them by bare name so a patched binding is honored. from mostlyright.weather._fetchers._goes_s3 import ( _AVAILABLE_SINCE, + _AVAILABLE_UNTIL, extract_pixel, list_product_keys, ) @@ -898,6 +899,7 @@ def _enumerate_slices( out: list[tuple[str, str, StationInfo, int, int]] = [] skipped: dict[str, list[str]] = {} + clamped: dict[str, int] = {} goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites for sat in satellites: for product in products: @@ -909,9 +911,30 @@ def _enumerate_slices( if info.icao not in stations_for_sat: stations_for_sat.append(info.icao) continue + until = _AVAILABLE_UNTIL.get(sat) if sat in goes_sats else None for year in range(year_start, year_end + 1): for month in range(1, 13): + # Operational-era END clamp (Codex P2): a retired GOES + # platform is not enumerated past its margined + # until-date (see _AVAILABLE_UNTIL provenance) — the + # slice would only burn empty per-hour listing calls. + # Enumeration-skip writes NO progress marker, so + # raising/removing the date later backfills these + # months normally. Month-granular, loud (logged below). + if until is not None and date(year, month, 1) >= until: + clamped[sat] = clamped.get(sat, 0) + 1 + continue out.append((sat, product, info, year, month)) + for sat, n_months in clamped.items(): + log.warning( + "GOES operational-era clamp: NOT enumerating %d post-retirement " + "month-slice(s) for %s (until %s, margined — see _AVAILABLE_UNTIL). " + "Raise/remove the date to backfill them (no progress marker is " + "written for clamped slices).", + n_months, + sat, + _AVAILABLE_UNTIL[sat].isoformat(), + ) for sat, icaos in skipped.items(): log.warning( "GOES hemisphere filter: NOT enumerating %s for station(s) %s — " diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index a61462b7..9654cd03 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1402,9 +1402,12 @@ def _slice(**kw): # Future months legitimately fetch + write nothing (no data yet). return _stub_slice(**kw) + # goes19 (OPERATIONAL, no _AVAILABLE_UNTIL clamp): a 2099 goes16 window + # would now be era-clamped at enumeration and this test would pass + # vacuously without exercising the resume semantics. with mock.patch.object(_backfill, "backfill_goes_satellite", _slice): _backfill.bulk_backfill( - satellites=["goes16"], + satellites=["goes19"], products=["ABI-L2-ACMC"], stations=["KNYC"], year_start=2099, @@ -1415,7 +1418,7 @@ def _slice(**kw): progress = _backfill._load_progress(progress_path) # No 2099 month may be marked completed — they are all in the future. future_keys = [ - _backfill._progress_key("goes16", "ABI-L2-ACMC", "KNYC", 2099, m) for m in range(1, 13) + _backfill._progress_key("goes19", "ABI-L2-ACMC", "KNYC", 2099, m) for m in range(1, 13) ] assert all(progress.get(k) != _backfill._PROGRESS_COMPLETED for k in future_keys) @@ -1432,9 +1435,11 @@ def _spy(**kw): ran.append(kw["month"]) return _slice(**kw) + # goes19: operational (no era clamp) so the 2099 future window + # actually enumerates — see the note in the sibling test above. with mock.patch.object(_backfill, "backfill_goes_satellite", _spy): _backfill.bulk_backfill( - satellites=["goes16"], + satellites=["goes19"], products=["ABI-L2-ACMC"], stations=["KNYC"], year_start=2099, @@ -1541,17 +1546,20 @@ def test_east_station_only_east_satellites(self) -> None: assert sats == {"goes16", "goes19"} def test_west_station_only_west_satellites(self) -> None: + # Probe year 2022: BOTH West platforms in-era (goes17 until-clamps at + # 2024 — that axis is TestGoesOperationalUntilClamp's, not this one's). infos = _backfill._resolve_station_infos(["KSEA"]) slices = _backfill._enumerate_slices( - ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2022, 2022 ) sats = {s[0] for s in slices} assert sats == {"goes17", "goes18"} def test_mixed_stations_get_their_own_pairs(self) -> None: + # Probe year 2022 — inside every platform's era (see note above). infos = _backfill._resolve_station_infos(["KNYC", "KSEA"]) slices = _backfill._enumerate_slices( - ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2022, 2022 ) by_station: dict[str, set[str]] = {} for sat, _product, info, _y, _m in slices: @@ -1565,3 +1573,61 @@ def test_single_requested_satellite_matching_hemisphere_unchanged(self) -> None: infos = _backfill._resolve_station_infos(["KNYC"]) slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2024, 2024) assert len(slices) == 12 + + +# --------------------------------------------------------------------------- +# GOES operational-era END clamp (Codex P2 on the era-coverage fix): a retired +# platform must not be enumerated past its conservative until-date, or every +# future fleet year burns empty per-hour listing calls on it (goes17 after its +# 2023 storage, goes16 after the 2025 goes19 handover). The until-dates carry +# ~1 year of safety margin and skips are LOUD — a wrong date would silently +# lose real settlement-adjacent data, so margin + logging are load-bearing. +# --------------------------------------------------------------------------- +class TestGoesOperationalUntilClamp: + def test_goes16_not_enumerated_after_retirement_margin(self) -> None: + # 2026: measured ALL-EMPTY live (2026-07-05 station run) — past the + # margined clamp, so the slices must not even enumerate. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes19"], ["ABI-L2-ACMC"], infos, 2026, 2026 + ) + sats = {s[0] for s in slices} + assert sats == {"goes19"} + + def test_goes16_still_enumerated_inside_margin(self) -> None: + # 2025: real data through the ~April handover — inside margin, keep. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2025, 2025) + assert len(slices) == 12 + + def test_goes17_not_enumerated_after_retirement_margin(self) -> None: + # goes17 went to storage Jan 2023; clamp (with margin) cuts 2024+. + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes17", "goes18"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert sats == {"goes18"} + + def test_goes17_era_still_enumerated(self) -> None: + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices(["goes17"], ["ABI-L2-ACMC"], infos, 2022, 2022) + assert len(slices) == 12 + + def test_operational_platforms_have_no_until(self) -> None: + # goes18/goes19 are operational: never clamped, any current/future year. + infos = _backfill._resolve_station_infos(["KNYC", "KSEA"]) + slices = _backfill._enumerate_slices( + ["goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2026, 2026 + ) + sats = {s[0] for s in slices} + assert sats == {"goes18", "goes19"} + + def test_partial_year_clamps_only_post_until_months(self) -> None: + # If the until-date falls mid-window the clamp is month-granular, not + # year-granular: a 2025-2026 goes16 request keeps all 12 months of 2025 + # and drops all of 2026 (until 2026-01-01). + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2025, 2026) + months = sorted((y, m) for _s, _p, _i, y, m in slices) + assert months == [(2025, m) for m in range(1, 13)] From fbca971841be698d944098d9db3ccbb66f9c0007 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 11:23:57 +0200 Subject: [PATCH 3/9] feat(weather): scope-keyed backfill progress + workflow satellites input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable concurrent multi-station measurement runs (KNYC + Asia + Europe): - Progress marker objects are now keyed on the RESOLVED (stations, year-window) scope instead of the Batch shard index. Shard-index keys COLLIDED across concurrent jobs: two station-mode jobs both wrote progress/shard-00000/progress.json — a concurrent read-modify-write clobber on the same GCS object. The scope key is stable across Spot retries (the shard->(station,year) mapping is pure), disjoint across array tasks AND across concurrent different-station jobs, and resume-stable across re-submissions. Whole-roster scopes (>3 stations, e.g. the incremental Cloud Run Job) use a bounded stable digest. Old shard-NNNNN objects are orphaned (one idempotent re-fetch; slice keys inside the files are full-identity so no wrong-skip either way). - run-weather-backfill.yml: new `satellites` input for pilot/station modes (default: the era-complete GOES set). Enables Asia (himawari8,himawari9 + AHI-L2-FLDK-Clouds) and Europe (viirs-* + VIIRS-JRR-CloudMask) station measurements; Meteosat backfill stays unwired (GRIB reader seam) per the _sources registry. Tests: scope-key disjointness across stations/year-shards, multi-station digest bounding, and the C4 wiring test updated to the scope scheme. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/run-weather-backfill.yml | 14 ++- .../mostlyright/weather/satellite/__main__.py | 26 ++++- .../tests/satellite/test_cli_roster.py | 95 ++++++++++++++++++- 3 files changed, 125 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index c347935d..abef149e 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -58,6 +58,11 @@ on: required: true default: "ABI-L2-ACMC" type: string + satellites: + description: "Comma-separated satellites for pilot/station modes (default: the era-complete GOES set; the CLI's hemisphere filter keeps each station on its own side pair). Override for non-GOES stations, e.g. himawari8,himawari9 + products=AHI-L2-FLDK-Clouds (Asia/RJTT) or viirs-npp,viirs-n20,viirs-n21 + products=VIIRS-JRR-CloudMask (Europe — Meteosat backfill is not wired yet). mode=full ignores this (roster defaults)." + required: true + default: "goes16,goes17,goes18,goes19" + type: string year_start: description: "First year (inclusive) of the backfill window (mode=pilot and mode=station)." required: true @@ -120,6 +125,7 @@ jobs: MODE: ${{ inputs.mode }} PILOT_STATION: ${{ inputs.pilot_station }} PRODUCTS: ${{ inputs.products }} + SATELLITES: ${{ inputs.satellites }} YEAR_START: ${{ inputs.year_start }} YEAR_END: ${{ inputs.year_end }} R2_BUCKET: ${{ vars.R2_BUCKET }} @@ -235,8 +241,8 @@ jobs: # downloads). available_since clamps make the off-era platform # nearly free. COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ - --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" \ - '["--mirror","gcp","--satellites","goes16,goes17,goes18,goes19","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') + --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') else TASK_COUNT=1 # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single @@ -255,8 +261,8 @@ jobs: # after the filter landed); an international pilot station still # needs --satellites overridden (Himawari/Meteosat). COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ - --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" \ - '["--mirror","gcp","--satellites","goes16,goes17,goes18,goes19","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') + --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') fi jq -n \ --arg img "${IMAGE}" \ diff --git a/packages/weather/src/mostlyright/weather/satellite/__main__.py b/packages/weather/src/mostlyright/weather/satellite/__main__.py index e403f456..6e2193bb 100644 --- a/packages/weather/src/mostlyright/weather/satellite/__main__.py +++ b/packages/weather/src/mostlyright/weather/satellite/__main__.py @@ -471,14 +471,32 @@ def _run_backfill(args: argparse.Namespace) -> int: # crash-safe resume. Wire it to the GCS-backed progress store so a preempted # Spot slice (or a replacement VM) rehydrates its markers from GCS and SKIPS # already-uploaded partitions instead of reprocessing under a fresh local disk. - # Each shard writes a DISJOINT marker object (shard-index in the key) so array - # tasks never clobber each other's progress. + # + # SCOPE-keyed marker object (28-29 multi-station fix; was shard-index-keyed). + # The key is the RESOLVED (stations, year-window) scope, which is: + # - stable across Spot retries (the shard->(station,year) mapping is pure), + # - disjoint across array tasks (each owns a distinct station-year), + # - disjoint across CONCURRENT jobs for DIFFERENT stations — shard-index + # keys collided there: two station-mode jobs both wrote + # progress/shard-00000/, a read-modify-write clobber on the same object, + # - resume-stable across re-submissions of the same scope. + # Old shard-NNNNN objects are simply orphaned (one idempotent re-fetch for + # any scope whose markers lived there; slice keys inside files are already + # full-identity so no wrong-skip is possible either way). if args.progress_bucket is not None: from ._progress import GcsProgressStore - shard_index, _shard_count = _resolve_shard_index_count(args) bucket = args.progress_bucket.rstrip("/") - gcs_uri = f"gs://{bucket}/progress/shard-{shard_index:05d}/progress.json" + if len(stations) <= 3: + scope_stations = "-".join(stations) + else: + # Whole-roster scopes (e.g. the incremental Cloud Run Job) would make + # an unwieldy object name — use a stable digest of the station tuple. + import hashlib + + digest = hashlib.sha1("_".join(stations).encode()).hexdigest()[:10] + scope_stations = f"multi{len(stations)}-{digest}" + gcs_uri = f"gs://{bucket}/progress/{scope_stations}_{year_start}-{year_end}/progress.json" kwargs["progress_store"] = GcsProgressStore(gcs_uri) print(f"using durable GCS progress store for crash-safe resume: {gcs_uri}") diff --git a/packages/weather/tests/satellite/test_cli_roster.py b/packages/weather/tests/satellite/test_cli_roster.py index 15104a35..767e2f31 100644 --- a/packages/weather/tests/satellite/test_cli_roster.py +++ b/packages/weather/tests/satellite/test_cli_roster.py @@ -323,8 +323,15 @@ def __init__(self, gcs_uri, *, fs=None): assert len(captured) == 1 # A durable GCS progress store was wired into bulk_backfill's kwargs... assert isinstance(captured[0]["progress_store"], _FakeStore) - # ...at a shard-disjoint marker URI. - assert built == ["gs://marker-bkt/progress/shard-00003/progress.json"] + # ...at a SCOPE-keyed marker URI (28-29): the resolved (stations, year + # window), NOT the shard index — shard-index keys collided across + # concurrent station-mode jobs (both wrote progress/shard-00000/). + year_start, year_end, _ = _roster_year_window() + stations, y_s, y_e = shard_roster( + resolve_roster("kalshi,polymarket"), 3, 65, year_start=year_start, year_end=year_end + ) + expected = f"gs://marker-bkt/progress/{'-'.join(stations)}_{y_s}-{y_e}/progress.json" + assert built == [expected] def test_explicit_mode_unchanged(captured, tmp_path): @@ -465,3 +472,87 @@ def test_explicit_multi_station_shard(captured): assert kw["stations"] == ["KDEN"] assert kw["year_start"] == 2017 assert kw["year_end"] == 2017 + + +def test_progress_scope_keys_disjoint_across_stations_and_years(captured, monkeypatch): + """Two station-mode jobs for DIFFERENT stations (or year shards) must write + DISJOINT progress objects — the old shard-index key collided (both jobs' + task 0 wrote progress/shard-00000/, a concurrent read-modify-write clobber).""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + + def _dispatch(station: str, index: str) -> None: + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--satellites", + "goes16", + "--products", + "ABI-L2-ACMC", + "--stations", + station, + "--year-start", + "2017", + "--year-end", + "2026", + "--out", + "/tmp/derived", + "--progress-bucket", + "marker-bkt", + "--shard-index", + index, + "--shard-count", + "10", + ] + ) + assert rc == 0 + + _dispatch("KNYC", "0") # KNYC 2017 (year shard 0) + _dispatch("RJTT", "0") # RJTT 2017 — same shard index, different station + _dispatch("KNYC", "1") # KNYC 2018 — same station, different year shard + assert built == [ + "gs://marker-bkt/progress/KNYC_2017-2017/progress.json", + "gs://marker-bkt/progress/RJTT_2017-2017/progress.json", + "gs://marker-bkt/progress/KNYC_2018-2018/progress.json", + ] + assert len(set(built)) == 3 # fully disjoint + + +def test_progress_scope_key_multi_station_uses_digest(captured, monkeypatch): + """A whole-roster scope (>3 stations) digests the station tuple so the + object name stays bounded but deterministic.""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--roster", + "kalshi,polymarket", + "--satellites", + "goes16", + "--progress-bucket", + "marker-bkt", + ] + ) + assert rc == 0 + assert len(built) == 1 + assert built[0].startswith("gs://marker-bkt/progress/multi65-") + assert built[0].endswith("/progress.json") From 9c6911df1c53a611b3ae797bf602cf9598e655a8 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 11:40:53 +0200 Subject: [PATCH 4/9] =?UTF-8?q?fix(weather):=20East-for-West=20era=20fallb?= =?UTF-8?q?ack=20=E2=80=94=20routing=20split=20is=20not=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P1 on the hemisphere filter: treating the East/West routing split as a coverage boundary silently lost the West side's 2017-2018 era. The sector geometry is ASYMMETRIC: GOES-West PACUS truly cannot see East stations, but GOES-East CONUS/full-disk genuinely covers the whole Americas (oblique at the far West) — and for 2017 -> 2018-08-27 (goes17 first light) goes16 was the ONLY ABI platform in existence, so a pure hemisphere filter left KSEA-class stations with zero 2017-2018 coverage that the previous defaults DID fetch. _enumerate_slices now applies direction-correct rules: - West platforms x East stations: still excluded (true out-of-grid). - East platforms x West stations: month-granular era FALLBACK — kept ONLY for months no REQUESTED native West platform covers (per the same since/until month semantics as the child-side clamp). August 2018 counts as goes17-covered (last day >= first light), so the fallback is Jan-Jul 2018 + all of 2017. - Explicit oblique requests (e.g. --satellites goes16 --stations KSEA, no native platform requested) are honored IN FULL — never silently narrowed. All three skip classes (out-of-grid exclusion, retirement clamp, fallback clip) log loudly. Tests: 6-case TestGoesEastForWestFallback (2017 kept, 2019 dropped, month-granular 2018 boundary, explicit-request honored, East exclusion unchanged, 2024 native-gap handling). Co-Authored-By: Claude Opus 4.8 --- .../weather/satellite/_backfill.py | 111 ++++++++++++------ .../weather/tests/test_satellite_backfill.py | 74 ++++++++++++ 2 files changed, 151 insertions(+), 34 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index bbe9f9cc..6db09331 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -885,45 +885,83 @@ def _enumerate_slices( ) -> list[tuple[str, str, StationInfo, int, int]]: """Enumerate per-(satellite, product, station, year, month) slices. - GOES hemisphere visibility filter (goes19-era coverage fix): a GOES - satellite is enumerated for a station ONLY when it sits on the station's - side of the East/West split (``_sources.goes_satellites_for_longitude`` — - the same split the live router uses). Without this, a multi-satellite - request cartesian-produced e.g. KNYC x goes18: whole-month WEST-archive - downloads whose extraction fails out-of-grid only AFTER the bytes are - fetched — pure spend, zero rows. Non-GOES satellites (Himawari / VIIRS / - Meteosat) are NOT filtered here; their sources own their coverage. Skips - are logged LOUDLY (once per satellite) — never silent under-coverage. + GOES coverage filter (goes19-era coverage fix + Codex P1 revision). The + East/West longitude split is a ROUTING preference, not a coverage boundary, + and the sector geometry is ASYMMETRIC: + + - GOES-West (goes17/goes18) publishes the PACUS sector, which truly + CANNOT see East-side stations -> a West platform is NEVER enumerated + for an East station (whole-month downloads would fail out-of-grid only + AFTER the bytes are fetched — pure spend, zero rows). + - GOES-East (goes16/goes19) CONUS/full-disk genuinely covers the whole + Americas (oblique view at the far West). An East platform IS valid + coverage for a West station — and for 2017 -> 2018-08-27 (goes17 first + light) goes16 was the ONLY ABI platform in existence. So East + platforms are kept for West stations as a month-granular era FALLBACK: + only for months that NO requested native West platform covers. If the + request contains NO native West platform at all (an explicit + oblique-view study, e.g. ``--satellites goes16 --stations KSEA``), the + explicit request is honored in full — never silently narrowed. + + Retired platforms are clamped past their margined _AVAILABLE_UNTIL date + (post-retirement months would only burn empty listing calls). Non-GOES + satellites (Himawari / VIIRS / Meteosat) are NOT filtered here; their + sources own their coverage. Every skip class is logged LOUDLY — never + silent under-coverage. """ from . import _sources out: list[tuple[str, str, StationInfo, int, int]] = [] - skipped: dict[str, list[str]] = {} - clamped: dict[str, int] = {} + excluded: dict[str, list[str]] = {} # West platform x East station (out-of-grid) + clamped: dict[str, int] = {} # post-retirement months (_AVAILABLE_UNTIL) + fallback_clipped: dict[str, int] = {} # East-for-West months a native platform covers goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites + + def _covers(sat: str, year: int, month: int) -> bool: + """True iff ``sat`` has data for (year, month) per the since/until clamps. + + Month availability matches the child-side clamp semantics: a month + counts as covered when its LAST day >= available_since and its FIRST + day < the (margined) available_until. + """ + since = _AVAILABLE_SINCE.get(sat) + if since is not None and _last_day_of_month(year, month) < since: + return False + until = _AVAILABLE_UNTIL.get(sat) + return not (until is not None and date(year, month, 1) >= until) + for sat in satellites: for product in products: for info in station_infos: - if sat in goes_sats and sat not in _sources.goes_satellites_for_longitude( - info.longitude - ): - stations_for_sat = skipped.setdefault(sat, []) - if info.icao not in stations_for_sat: - stations_for_sat.append(info.icao) - continue + fallback_native: list[str] = [] + if sat in goes_sats: + native = _sources.goes_satellites_for_longitude(info.longitude) + if sat not in native: + if sat in _sources._GOES_WEST_SATELLITES: + # West platform x East station: PACUS cannot see it. + stations_for_sat = excluded.setdefault(sat, []) + if info.icao not in stations_for_sat: + stations_for_sat.append(info.icao) + continue + # East platform x West station: era fallback — clip to + # months no REQUESTED native platform covers (empty + # list == explicit oblique request, honored in full). + fallback_native = [s for s in satellites if s in native] until = _AVAILABLE_UNTIL.get(sat) if sat in goes_sats else None for year in range(year_start, year_end + 1): for month in range(1, 13): - # Operational-era END clamp (Codex P2): a retired GOES - # platform is not enumerated past its margined - # until-date (see _AVAILABLE_UNTIL provenance) — the - # slice would only burn empty per-hour listing calls. - # Enumeration-skip writes NO progress marker, so - # raising/removing the date later backfills these - # months normally. Month-granular, loud (logged below). + # Operational-era END clamp: a retired GOES platform is + # not enumerated past its margined until-date. NO + # progress marker is written for clamped slices, so + # raising/removing the date later backfills them. if until is not None and date(year, month, 1) >= until: clamped[sat] = clamped.get(sat, 0) + 1 continue + if fallback_native and any( + _covers(s, year, month) for s in fallback_native + ): + fallback_clipped[sat] = fallback_clipped.get(sat, 0) + 1 + continue out.append((sat, product, info, year, month)) for sat, n_months in clamped.items(): log.warning( @@ -935,22 +973,27 @@ def _enumerate_slices( sat, _AVAILABLE_UNTIL[sat].isoformat(), ) - for sat, icaos in skipped.items(): + for sat, icaos in excluded.items(): log.warning( - "GOES hemisphere filter: NOT enumerating %s for station(s) %s — " - "wrong side of the East/West split (lon %s %.1f); the station's own " - "side pair covers it. This skip avoids whole-archive downloads that " - "fail out-of-grid after fetch.", + "GOES coverage filter: NOT enumerating West platform %s for " + "East-side station(s) %s — the PACUS sector cannot see them " + "(out-of-grid only AFTER a full download). GOES-East covers them.", sat, ",".join(icaos), - "<" if sat in _sources._GOES_WEST_SATELLITES else ">=", - _sources._GOES_EAST_WEST_SPLIT_LON_DEG, + ) + for sat, n_months in fallback_clipped.items(): + log.warning( + "GOES East-for-West fallback: clipped %d month-slice(s) of %s for " + "West-side station(s) — a requested native West platform covers " + "those months (East oblique kept ONLY for the pre-native era). " + "Request the East platform WITHOUT native platforms to fetch the " + "oblique view explicitly.", + n_months, + sat, ) return out -# --------------------------------------------------------------------------- -# Resume layer — durable progress (fsync) + .bak fallback + validation. # --------------------------------------------------------------------------- def _progress_key(satellite: str, product: str, station: str, year: int, month: int) -> str: """Return the resume key encoding the FULL slice identity (P1-2). diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 9654cd03..54cfa507 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1631,3 +1631,77 @@ def test_partial_year_clamps_only_post_until_months(self) -> None: slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2025, 2026) months = sorted((y, m) for _s, _p, _i, y, m in slices) assert months == [(2025, m) for m in range(1, 13)] + + +# --------------------------------------------------------------------------- +# GOES East-for-West era FALLBACK (Codex P1 on the hemisphere filter): the +# East/West split is a ROUTING preference, not a coverage boundary. GOES-East +# CONUS/full-disk genuinely covers the whole Americas (oblique view), and for +# 2017 -> 2018-08-27 (goes17 first light) goes16 was the ONLY ABI platform — +# a pure hemisphere filter silently loses the West side's 2017-2018 era. +# Direction-correct rules: +# - West platforms (goes17/goes18 PACUS) for EAST stations: still excluded +# (true out-of-grid). +# - East platforms (goes16/goes19) for WEST stations: kept ONLY for months +# no REQUESTED native West platform covers (the pre-native fallback era) — +# unless NO native platform was requested at all (explicit oblique study), +# in which case the request is honored fully. +# --------------------------------------------------------------------------- +class TestGoesEastForWestFallback: + def test_west_station_2017_keeps_goes16(self) -> None: + """KSEA 2017: no West platform existed -> goes16 fallback must stay.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2017, 2017 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 13)) + + def test_west_station_2019_drops_goes16(self) -> None: + """KSEA 2019: goes17 (native) covers every month -> no East fallback.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2019, 2019 + ) + sats = {s[0] for s in slices} + assert "goes16" not in sats + assert {"goes17", "goes18"} <= sats + + def test_west_station_2018_fallback_is_month_granular(self) -> None: + """2018: goes17 first light 2018-08-27 -> goes16 covers Jan-Jul only + (August itself counts as goes17-covered: the child-side clamp treats a + month as available when its LAST day >= available_since).""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2018, 2018 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 8)) # Jan..Jul + + def test_west_station_explicit_east_only_request_honored(self) -> None: + """--satellites goes16 --stations KSEA (no native platform requested): + an explicit oblique-view request runs FULLY — never silently narrowed.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2024, 2024) + assert len(slices) == 12 + + def test_east_station_still_never_gets_west_platforms(self) -> None: + """KNYC x goes17/goes18 stays excluded — PACUS truly cannot see the + East coast (out-of-grid after full download; the original pathology).""" + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2017, 2017 + ) + sats = {s[0] for s in slices} + assert sats == {"goes16", "goes19"} + + def test_west_station_2024_native_gap_falls_back_to_east(self) -> None: + """2024: goes17 is until-clamped (2024-01-01) but goes18 (native, since + 2022-07-28) covers every month -> STILL no East fallback.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert "goes16" not in sats + assert sats == {"goes18"} From 3410c167e0d20b27bef2a9fc2200aacef46449f5 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 12:09:52 +0200 Subject: [PATCH 5/9] fix(weather): full-month un-margined coverage claims; product/satellite scope keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial exit panel (18 agents, findings verified by EXECUTING the enumeration code) caught 4 blocking defects in the fallback/scope work: - P1: _covers() counted a month as native-covered from a SINGLE day of data. goes17's Aug 27-31 2018 first-light tail clipped the goes16 fallback for all of August while the child-side clamp fetched only the tail -> Aug 1-26 2018 fetched by NO platform for all 8 West roster stations, on the DEFAULT request (and Jul 1-27 2022 under the legacy goes16,goes18 pair). The old boundary test enshrined the hole. Positive coverage now requires the FULL month (first day >= available_since); partial first-light months keep the East fallback (~5 duplicate days in separate per-satellite partitions — the safe direction). - P2: _covers() consumed the MARGINED _AVAILABLE_UNTIL as a positive coverage claim, inverting the margin's "too-late only wastes" safety contract (goes17 "covered" Feb-Dec 2023 with zero real data -> 11-month hole for goes16,goes17 requests; margin widening would silently widen the suppression window). New _GOES_ACTUAL_DATA_END map (un-margined real last-data dates, verified against the live archive) is used for claims; the margined map stays fetch-side only. - P2: the scope-keyed progress object omitted the products/satellites axes -> concurrent ACMC vs DSRF jobs for the same station+window still clobbered one GCS object (full-map PUTs, last-write-wins). The scope now appends a SORTED satellites#products digest (order-insensitive). - P2: the stale-image danger (all-four satellites x pre-filter image = cartesian wrong-hemisphere downloads) was documented only in shell comments invisible at dispatch; it is now in the image_tag and satellites INPUT descriptions the operator actually sees. Regressions added for every scenario the panel reproduced: Aug-2018 partial-month keep, Jul-2022 legacy-pair keep, 2023 margined-until non-claim, default-2023 goes18 clip, and scope disjointness across products/satellites with order-insensitivity. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/run-weather-backfill.yml | 4 +- .../mostlyright/weather/_fetchers/_goes_s3.py | 18 +++++ .../mostlyright/weather/satellite/__main__.py | 17 ++++- .../weather/satellite/_backfill.py | 34 +++++---- .../tests/satellite/test_cli_roster.py | 69 +++++++++++++++++-- .../weather/tests/test_satellite_backfill.py | 48 +++++++++++-- 6 files changed, 164 insertions(+), 26 deletions(-) diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index abef149e..7bb02fb2 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -35,7 +35,7 @@ on: workflow_dispatch: inputs: image_tag: - description: "weather-backfill image tag to run (must already be pushed by deploy-weather-ingest)." + description: "weather-backfill image tag to run (must already be pushed by deploy-weather-ingest). MUST be built from a commit AFTER the GOES coverage filter (PR #101) when using the multi-satellite default below — an older image cartesian-products all requested satellites (wrong-hemisphere downloads)." required: true default: "latest" type: string @@ -59,7 +59,7 @@ on: default: "ABI-L2-ACMC" type: string satellites: - description: "Comma-separated satellites for pilot/station modes (default: the era-complete GOES set; the CLI's hemisphere filter keeps each station on its own side pair). Override for non-GOES stations, e.g. himawari8,himawari9 + products=AHI-L2-FLDK-Clouds (Asia/RJTT) or viirs-npp,viirs-n20,viirs-n21 + products=VIIRS-JRR-CloudMask (Europe — Meteosat backfill is not wired yet). mode=full ignores this (roster defaults)." + description: "Comma-separated satellites for pilot/station modes (default: the era-complete GOES set — REQUIRES an image built after the GOES coverage filter, PR #101; older images cartesian-product all four = wrong-hemisphere downloads). Override for non-GOES stations, e.g. himawari8,himawari9 + products=AHI-L2-FLDK-Clouds (Asia/RJTT) or viirs-npp,viirs-n20,viirs-n21 + products=VIIRS-JRR-CloudMask (Europe — Meteosat backfill is not wired yet). mode=full IGNORES this input entirely (roster defaults apply)." required: true default: "goes16,goes17,goes18,goes19" type: string diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index 25efd8ff..753b310a 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -146,6 +146,24 @@ "goes17": date(2024, 1, 1), } +# UN-MARGINED actual last-data dates for retired platforms — used ONLY for +# POSITIVE coverage claims (the East-for-West fallback clip asks "does this +# native platform REALLY cover this month?"), NEVER for fetch clamps. The two +# maps deliberately lean opposite ways so both stay safe: _AVAILABLE_UNTIL is +# margined LATE (fetch side: a too-late date only wastes cheap empty listings), +# while claiming coverage from a margined date would suppress the East fallback +# for months with NO real native data (verified live: s3://noaa-goes17/ has no +# objects past 2023-01-10). Widening an _AVAILABLE_UNTIL margin must therefore +# never touch this map. +# - goes17: last data 2023-01-10 (went to storage; goes18 took over 2023-01-04). +# - goes16: goes19 became operational GOES-East 2025-04-07 (kept for +# documentation; East platforms are never consulted as NATIVE coverage — +# no fallback direction exists toward the East side). +_GOES_ACTUAL_DATA_END: dict[str, date] = { + "goes16": date(2025, 4, 7), + "goes17": date(2023, 1, 10), +} + # Conservative top-level S3/GCS call-rate cap (2i ``_GOES_S3_RATE_HZ``). The # empirical concurrency probe (25-05 D10) may later derive a tuned value. _GOES_S3_RATE_HZ = 20.0 diff --git a/packages/weather/src/mostlyright/weather/satellite/__main__.py b/packages/weather/src/mostlyright/weather/satellite/__main__.py index 6e2193bb..109a60e6 100644 --- a/packages/weather/src/mostlyright/weather/satellite/__main__.py +++ b/packages/weather/src/mostlyright/weather/satellite/__main__.py @@ -484,6 +484,8 @@ def _run_backfill(args: argparse.Namespace) -> int: # any scope whose markers lived there; slice keys inside files are already # full-identity so no wrong-skip is possible either way). if args.progress_bucket is not None: + import hashlib + from ._progress import GcsProgressStore bucket = args.progress_bucket.rstrip("/") @@ -492,11 +494,20 @@ def _run_backfill(args: argparse.Namespace) -> int: else: # Whole-roster scopes (e.g. the incremental Cloud Run Job) would make # an unwieldy object name — use a stable digest of the station tuple. - import hashlib - digest = hashlib.sha1("_".join(stations).encode()).hexdigest()[:10] scope_stations = f"multi{len(stations)}-{digest}" - gcs_uri = f"gs://{bucket}/progress/{scope_stations}_{year_start}-{year_end}/progress.json" + # The scope must ALSO carry the satellites+products axes: an ACMC job + # and a DSRF job for the same station+window are separate concurrent + # work-scopes, and GcsProgressStore's mark_complete PUTs the full + # in-memory map — same-object concurrent writers are last-write-wins. + # SORTED so goes16,goes19 vs goes19,goes16 map to the same scope. + sp_digest = hashlib.sha1( + (",".join(sorted(satellites)) + "#" + ",".join(sorted(products))).encode() + ).hexdigest()[:8] + gcs_uri = ( + f"gs://{bucket}/progress/" + f"{scope_stations}_{year_start}-{year_end}_{sp_digest}/progress.json" + ) kwargs["progress_store"] = GcsProgressStore(gcs_uri) print(f"using durable GCS progress store for crash-safe resume: {gcs_uri}") diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index 6db09331..293ec51c 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -71,6 +71,7 @@ from mostlyright.weather._fetchers._goes_s3 import ( _AVAILABLE_SINCE, _AVAILABLE_UNTIL, + _GOES_ACTUAL_DATA_END, extract_pixel, list_product_keys, ) @@ -917,18 +918,27 @@ def _enumerate_slices( fallback_clipped: dict[str, int] = {} # East-for-West months a native platform covers goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites - def _covers(sat: str, year: int, month: int) -> bool: - """True iff ``sat`` has data for (year, month) per the since/until clamps. - - Month availability matches the child-side clamp semantics: a month - counts as covered when its LAST day >= available_since and its FIRST - day < the (margined) available_until. + def _native_fully_covers(sat: str, year: int, month: int) -> bool: + """POSITIVE coverage claim: does ``sat`` cover EVERY day of the month? + + Used only to decide whether the East-for-West fallback may be clipped, + so both bounds lean CONSERVATIVE (a false "covered" silently loses real + data; a false "not covered" only over-fetches a few duplicate days into + separate per-satellite partitions — the stated-safe direction): + - FULL month: first day >= available_since. The child-side fetch + clamp skips days BEFORE first light, so a first-light month (e.g. + goes17 Aug 2018, since 08-27) yields only its 5-day tail — it must + NOT clip the East platform that covers the other 26 days. + - UN-MARGINED end: last day < _GOES_ACTUAL_DATA_END (the real + last-data date), NOT the margined _AVAILABLE_UNTIL — the margin is + fetch-side slack and claiming coverage from it would suppress the + fallback for months with no real native data (goes17 Feb-Dec 2023). """ since = _AVAILABLE_SINCE.get(sat) - if since is not None and _last_day_of_month(year, month) < since: + if since is not None and date(year, month, 1) < since: return False - until = _AVAILABLE_UNTIL.get(sat) - return not (until is not None and date(year, month, 1) >= until) + data_end = _GOES_ACTUAL_DATA_END.get(sat) + return not (data_end is not None and _last_day_of_month(year, month) >= data_end) for sat in satellites: for product in products: @@ -958,7 +968,7 @@ def _covers(sat: str, year: int, month: int) -> bool: clamped[sat] = clamped.get(sat, 0) + 1 continue if fallback_native and any( - _covers(s, year, month) for s in fallback_native + _native_fully_covers(s, year, month) for s in fallback_native ): fallback_clipped[sat] = fallback_clipped.get(sat, 0) + 1 continue @@ -984,8 +994,8 @@ def _covers(sat: str, year: int, month: int) -> bool: for sat, n_months in fallback_clipped.items(): log.warning( "GOES East-for-West fallback: clipped %d month-slice(s) of %s for " - "West-side station(s) — a requested native West platform covers " - "those months (East oblique kept ONLY for the pre-native era). " + "West-side station(s) — a requested native West platform FULLY " + "covers those months (first-light/partial months are NOT clipped). " "Request the East platform WITHOUT native platforms to fetch the " "oblique view explicitly.", n_months, diff --git a/packages/weather/tests/satellite/test_cli_roster.py b/packages/weather/tests/satellite/test_cli_roster.py index 767e2f31..715d7d90 100644 --- a/packages/weather/tests/satellite/test_cli_roster.py +++ b/packages/weather/tests/satellite/test_cli_roster.py @@ -330,7 +330,10 @@ def __init__(self, gcs_uri, *, fs=None): stations, y_s, y_e = shard_roster( resolve_roster("kalshi,polymarket"), 3, 65, year_start=year_start, year_end=year_end ) - expected = f"gs://marker-bkt/progress/{'-'.join(stations)}_{y_s}-{y_e}/progress.json" + # Scope carries the satellites+products axes too (sorted digest) so + # concurrent same-station jobs with different products never clobber. + sp = _sp_digest(["goes16"], list(cli._DEFAULT_ROSTER_PRODUCTS)) + expected = f"gs://marker-bkt/progress/{'-'.join(stations)}_{y_s}-{y_e}_{sp}/progress.json" assert built == [expected] @@ -519,10 +522,11 @@ def _dispatch(station: str, index: str) -> None: _dispatch("KNYC", "0") # KNYC 2017 (year shard 0) _dispatch("RJTT", "0") # RJTT 2017 — same shard index, different station _dispatch("KNYC", "1") # KNYC 2018 — same station, different year shard + sp = _sp_digest(["goes16"], ["ABI-L2-ACMC"]) assert built == [ - "gs://marker-bkt/progress/KNYC_2017-2017/progress.json", - "gs://marker-bkt/progress/RJTT_2017-2017/progress.json", - "gs://marker-bkt/progress/KNYC_2018-2018/progress.json", + f"gs://marker-bkt/progress/KNYC_2017-2017_{sp}/progress.json", + f"gs://marker-bkt/progress/RJTT_2017-2017_{sp}/progress.json", + f"gs://marker-bkt/progress/KNYC_2018-2018_{sp}/progress.json", ] assert len(set(built)) == 3 # fully disjoint @@ -556,3 +560,60 @@ def __init__(self, gcs_uri, *, fs=None): assert len(built) == 1 assert built[0].startswith("gs://marker-bkt/progress/multi65-") assert built[0].endswith("/progress.json") + + +def _sp_digest(satellites: list[str], products: list[str]) -> str: + """Mirror of the CLI's scope digest: sorted satellites + sorted products.""" + import hashlib + + return hashlib.sha1( + (",".join(sorted(satellites)) + "#" + ",".join(sorted(products))).encode() + ).hexdigest()[:8] + + +def test_progress_scope_disjoint_across_products_and_satellites(captured, monkeypatch): + """Exit-panel P2 regression: an ACMC job and a DSRF job for the SAME + station+window are separate concurrent work-scopes — their progress objects + must differ (GcsProgressStore full-map PUTs are last-write-wins on a shared + object). Satellite-set changes must also re-scope; ordering must NOT.""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + + def _dispatch(products: str, satellites: str) -> None: + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--satellites", + satellites, + "--products", + products, + "--stations", + "KNYC", + "--year-start", + "2017", + "--year-end", + "2017", + "--out", + "/tmp/derived", + "--progress-bucket", + "marker-bkt", + ] + ) + assert rc == 0 + + _dispatch("ABI-L2-ACMC", "goes16") + _dispatch("ABI-L2-DSRF", "goes16") # product axis -> new scope + _dispatch("ABI-L2-ACMC", "goes16,goes19") # satellite axis -> new scope + _dispatch("ABI-L2-ACMC", "goes19,goes16") # ORDER only -> same scope as #3 + assert len(built) == 4 + assert len({built[0], built[1], built[2]}) == 3 # three distinct scopes + assert built[2] == built[3] # sorted digest: order-insensitive diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 54cfa507..b03bc0ba 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1667,16 +1667,21 @@ def test_west_station_2019_drops_goes16(self) -> None: assert "goes16" not in sats assert {"goes17", "goes18"} <= sats - def test_west_station_2018_fallback_is_month_granular(self) -> None: - """2018: goes17 first light 2018-08-27 -> goes16 covers Jan-Jul only - (August itself counts as goes17-covered: the child-side clamp treats a - month as available when its LAST day >= available_since).""" + def test_west_station_2018_fallback_keeps_partial_first_light_month(self) -> None: + """2018: goes17 first light 2018-08-27 -> goes16 must cover Jan-AUG. + + Exit-panel P1 regression: the first cut treated Aug 2018 as + goes17-covered because the month's LAST day >= first light — but the + child-side fetch clamp only pulls Aug 27-31, so clipping goes16 left + Aug 1-26 fetched by NO platform for every West station. A partial + native month must NOT clip the East fallback (the ~5 overlap days land + in separate per-satellite partitions — the safe direction).""" infos = _backfill._resolve_station_infos(["KSEA"]) slices = _backfill._enumerate_slices( ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2018, 2018 ) goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") - assert goes16_months == list(range(1, 8)) # Jan..Jul + assert goes16_months == list(range(1, 9)) # Jan..Aug (Aug = partial goes17) def test_west_station_explicit_east_only_request_honored(self) -> None: """--satellites goes16 --stations KSEA (no native platform requested): @@ -1695,6 +1700,39 @@ def test_east_station_still_never_gets_west_platforms(self) -> None: sats = {s[0] for s in slices} assert sats == {"goes16", "goes19"} + def test_legacy_pair_request_keeps_goes16_for_partial_jul_2022(self) -> None: + """Exit-panel P1 sibling: goes16,goes18 request (the OLD default pair) + for a West station in 2022 — goes18 first light 2022-07-28 makes July a + partial month, so goes16 must stay for Jan..Jul (not Jan..Jun).""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes18"], ["ABI-L2-ACMC"], infos, 2022, 2022 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 8)) # Jan..Jul + + def test_margined_until_is_not_a_coverage_claim(self) -> None: + """Exit-panel P2 regression: goes17's margined _AVAILABLE_UNTIL + (2024-01-01) must NOT count as coverage for 2023 — its ACTUAL data ends + 2023-01-10. A goes16,goes17 request for KSEA 2023 keeps the goes16 + fallback for ALL 12 months (Jan is partial: data ends mid-month).""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17"], ["ABI-L2-ACMC"], infos, 2023, 2023 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 13)) + + def test_default_request_2023_still_clips_via_goes18(self) -> None: + """Default all-four request for KSEA 2023: goes18 (operational, full + coverage) legitimately clips the goes16 fallback for every month.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2023, 2023 + ) + sats = {s[0] for s in slices} + assert "goes16" not in sats + def test_west_station_2024_native_gap_falls_back_to_east(self) -> None: """2024: goes17 is until-clamped (2024-01-01) but goes18 (native, since 2022-07-28) covers every month -> STILL no East fallback.""" From 0f742ddea040c85d482c66747b8a1113f2f0d28c Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 12:28:18 +0200 Subject: [PATCH 6/9] fix(weather): raise CONUS-class size cap 2->16 MiB for modern ACMC files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-fleet finding (2026-07-05, KNYC station run, tasks 2022/2023): every modern ABI-L2-ACMC file was rejected "before read (P2-d resource limit)" — listed sizes 4.3-4.6 MB vs the 2 MiB default cap. The cap was calibrated on 2017-era 2-variable files (~0.3-1.5 MB); NOAA later added ACM + Cloud_Probabilities to the product, roughly tripling file sizes, so the guard zeroed ACMC coverage for the modern era (2021+) fleet-wide. Fail-safe held (per-file errors block completion markers — no wrong "done" state), but the affected year-tasks burned full listing time producing zero ACMC rows. Third instance of the same failure family today: a guard hardcoded from old-era data (DSRF grid, goes16-only satellites, now file sizes). Default cap -> 16 MiB (>3x headroom over observed modern sizes, still bounds the in-memory BytesIO well below DSRF's 64 MiB). Regression test pins: modern observed size admitted with margin, cap still < DSRF cap, DSRF cap unchanged. Co-Authored-By: Claude Opus 4.8 --- .../mostlyright/weather/_fetchers/_goes_s3.py | 16 ++++++++++++---- packages/weather/tests/test_satellite_s3.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index 753b310a..338c8a3b 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -178,10 +178,18 @@ _S3_FAIL_FAST_CODES = frozenset({"404", "NoSuchKey", "NoSuchBucket", "AccessDenied"}) # Per-product size cap (P2-d). Rejects an over-cap object BEFORE the full-object -# read so the in-memory BytesIO buffer is bounded. CONUS files run ~0.3-1.5 MB; -# DSRF full-disk runs ~50 MB. Caps are generous over the documented sizes. -# Default cap (CONUS-class) applies to any product not listed explicitly. -_DEFAULT_SIZE_CAP_BYTES = 2 * 1024 * 1024 # ~2 MB +# read so the in-memory BytesIO buffer is bounded. Default cap (CONUS-class) +# applies to any product not listed explicitly; DSRF full-disk (~50 MB) carries +# its own larger cap. +# +# 16 MiB, NOT the original 2 MiB: the 2 MiB cap was calibrated on 2017-era +# 2-variable ACMC files (~0.3-1.5 MB), but NOAA later added ACM + +# Cloud_Probabilities to the product and modern ACMC files run ~4.3-4.6 MB — +# observed LIVE 2026-07-05 (KNYC station backfill): the old cap rejected EVERY +# 2022+ ACMC file "before read", zeroing ACMC coverage for the modern era +# fleet-wide. 16 MiB = >3x headroom over the observed modern sizes while still +# bounding the in-memory buffer well below the DSRF cap. +_DEFAULT_SIZE_CAP_BYTES = 16 * 1024 * 1024 # 16 MiB _SIZE_CAP_BYTES: dict[str, int] = { p: (64 * 1024 * 1024 if p == "ABI-L2-DSRF" else _DEFAULT_SIZE_CAP_BYTES) for (p, _v) in PRODUCTS } diff --git a/packages/weather/tests/test_satellite_s3.py b/packages/weather/tests/test_satellite_s3.py index 087a14dd..16900de0 100644 --- a/packages/weather/tests/test_satellite_s3.py +++ b/packages/weather/tests/test_satellite_s3.py @@ -286,6 +286,21 @@ def test_open_and_extract_reads_bytesio_not_lazy_file(knyc: StationInfo) -> None fs.open.assert_not_called() +def test_size_cap_accepts_modern_acmc_file_sizes() -> None: + """Live-fleet regression (2026-07-05, KNYC station run): modern ACMC files + are ~4.3-4.6 MB — NOAA added ACM + Cloud_Probabilities to the product + (~2021+), roughly doubling 2017-era 2-variable file sizes. The old 2 MiB + default cap rejected EVERY modern ACMC file "before read", so ACMC 2021+ + produced zero rows fleet-wide. The cap must comfortably admit the observed + modern sizes while still rejecting pathological objects.""" + observed_modern_acmc = 4_614_061 # largest size seen live (2022/002) + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-ACMC"] > observed_modern_acmc * 2 + # Still a real guard: pathological objects stay rejected. + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-ACMC"] < 64 * 1024 * 1024 + # DSRF full-disk keeps its own larger cap. + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-DSRF"] == 64 * 1024 * 1024 + + def test_size_cap_rejects_before_read(knyc: StationInfo) -> None: """An over-cap object is rejected before any object read on either mirror.""" fs = mock.MagicMock() From e23a8bfa7dd7f06812d92fbcab7fdfc7d93fa9df Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 13:39:50 +0200 Subject: [PATCH 7/9] =?UTF-8?q?fix(weather):=20product-aware=20cross-side?= =?UTF-8?q?=20exclusion=20=E2=80=94=20full-disk=20sees=20both=20sides?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2 (final stamp): the West-for-East exclusion was PRODUCT-blind. PACUS/CONUS-sector products truly cannot see East stations, but FULL-DISK products (ABI-L2-DSRF, 5424x5424 registry grid) see most of the Americas from either GOES slot — an explicit goes18+DSRF+KNYC request was silently dropped to zero slices, with the exclusion log falsely claiming "the PACUS sector cannot see them". _enumerate_slices West-for-East branch is now product-aware: - sector product -> always excluded (unchanged; true out-of-grid); - full-disk + a requested native covering the station -> skipped as redundant (default cost control), logged distinctly; - full-disk WITHOUT any native in the request (explicit cross-side study) -> honored in full. Full-disk detection reads grid_shape_expected == (5424, 5424) from the PRODUCTS registry (no hardcoded product list). Tests: explicit cross-side DSRF honored, sector product still excluded, default all-four still skips cross-side DSRF, mixed-products explicit request keeps only the full-disk slices. Co-Authored-By: Claude Opus 4.8 --- .../weather/satellite/_backfill.py | 55 ++++++++++++++++--- .../weather/tests/test_satellite_backfill.py | 37 +++++++++++++ 2 files changed, 85 insertions(+), 7 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index 293ec51c..7db5342d 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -68,6 +68,7 @@ # Transport (25-03). These bare module-level names are what tests monkeypatch # (``mock.patch.object(_backfill, "list_product_keys")``); the bulk/slice code # below references them by bare name so a patched binding is honored. +from mostlyright.weather._fetchers._goes_extract import PRODUCTS as _GOES_PRODUCTS from mostlyright.weather._fetchers._goes_s3 import ( _AVAILABLE_SINCE, _AVAILABLE_UNTIL, @@ -877,6 +878,20 @@ def _validate_partition_components(satellites: list[str], products: list[str]) - _sources.validate_satellite_and_product(sat, product) +def _is_full_disk_goes_product(product: str) -> bool: + """True iff ``product``'s registry grid is the 5424x5424 full ABI disk. + + Full-disk products see most of the Americas from EITHER GOES slot (oblique + at the far edge), so the West-for-East sector exclusion must not apply to + them — only PACUS/CONUS-sector products are truly blind cross-side. + """ + return any( + tuple(pv.grid_shape_expected) == (5424, 5424) + for (p, _v), pv in _GOES_PRODUCTS.items() + if p == product + ) + + def _enumerate_slices( satellites: list[str], products: list[str], @@ -914,6 +929,7 @@ def _enumerate_slices( out: list[tuple[str, str, StationInfo, int, int]] = [] excluded: dict[str, list[str]] = {} # West platform x East station (out-of-grid) + cross_redundant: dict[str, int] = {} # full-disk cross-side skipped: natives cover clamped: dict[str, int] = {} # post-retirement months (_AVAILABLE_UNTIL) fallback_clipped: dict[str, int] = {} # East-for-West months a native platform covers goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites @@ -948,11 +964,26 @@ def _native_fully_covers(sat: str, year: int, month: int) -> bool: native = _sources.goes_satellites_for_longitude(info.longitude) if sat not in native: if sat in _sources._GOES_WEST_SATELLITES: - # West platform x East station: PACUS cannot see it. - stations_for_sat = excluded.setdefault(sat, []) - if info.icao not in stations_for_sat: - stations_for_sat.append(info.icao) - continue + # West platform x East station — PRODUCT-aware + # (Codex P2): PACUS/CONUS-sector products truly + # cannot see an East station (out-of-grid only + # AFTER a full download), but FULL-DISK products + # (DSRF) see most of the Americas from either + # slot. So: sector product -> always excluded; + # full-disk with a requested native covering the + # station -> excluded as redundant (default cost + # control); full-disk WITHOUT any native in the + # request (explicit cross-side study, e.g. + # goes18+DSRF+KNYC) -> honored in full. + if not _is_full_disk_goes_product(product): + stations_for_sat = excluded.setdefault(sat, []) + if info.icao not in stations_for_sat: + stations_for_sat.append(info.icao) + continue + if any(s in native for s in satellites): + cross_redundant[sat] = cross_redundant.get(sat, 0) + 1 + continue + # Explicit cross-side full-disk request: keep. # East platform x West station: era fallback — clip to # months no REQUESTED native platform covers (empty # list == explicit oblique request, honored in full). @@ -986,11 +1017,21 @@ def _native_fully_covers(sat: str, year: int, month: int) -> bool: for sat, icaos in excluded.items(): log.warning( "GOES coverage filter: NOT enumerating West platform %s for " - "East-side station(s) %s — the PACUS sector cannot see them " - "(out-of-grid only AFTER a full download). GOES-East covers them.", + "East-side station(s) %s — its PACUS/CONUS-sector products cannot " + "see them (out-of-grid only AFTER a full download). GOES-East " + "covers them. (Full-disk products are handled separately.)", sat, ",".join(icaos), ) + for sat, n_months in cross_redundant.items(): + log.warning( + "GOES coverage filter: skipped %d cross-side FULL-DISK month-" + "slice(s) of %s — requested native platforms cover the station. " + "Request the cross-side platform WITHOUT natives to fetch its " + "oblique full-disk view explicitly.", + n_months, + sat, + ) for sat, n_months in fallback_clipped.items(): log.warning( "GOES East-for-West fallback: clipped %d month-slice(s) of %s for " diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index b03bc0ba..2c4797a7 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1647,6 +1647,43 @@ def test_partial_year_clamps_only_post_until_months(self) -> None: # unless NO native platform was requested at all (explicit oblique study), # in which case the request is honored fully. # --------------------------------------------------------------------------- +class TestGoesFullDiskCrossSide: + """Codex P2 (final stamp): the West-for-East exclusion must be + PRODUCT-aware. PACUS/CONUS-sector products truly cannot see East stations, + but FULL-DISK products (ABI-L2-DSRF, 5424x5424) see most of the Americas + from either slot — an explicit goes18+DSRF+KNYC request is geometrically + valid (oblique) and must not be silently dropped. Default requests still + exclude the cross-side platform when natives cover it (cost control).""" + + def test_explicit_west_fulldisk_for_east_station_honored(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes18"], ["ABI-L2-DSRF"], infos, 2023, 2023) + assert len(slices) == 12 + + def test_explicit_west_conus_for_east_station_still_excluded(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes18"], ["ABI-L2-ACMC"], infos, 2023, 2023) + assert slices == [] + + def test_default_request_fulldisk_still_excludes_cross_side(self) -> None: + # Natives (goes16/goes19) cover KNYC DSRF — no cross-side redundancy. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-DSRF"], infos, 2023, 2023 + ) + sats = {s[0] for s in slices} + assert "goes17" not in sats and "goes18" not in sats + + def test_explicit_west_mixed_products_keeps_only_fulldisk(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes18"], ["ABI-L2-ACMC", "ABI-L2-DSRF"], infos, 2023, 2023 + ) + prods = {s[1] for s in slices} + assert prods == {"ABI-L2-DSRF"} + assert len(slices) == 12 + + class TestGoesEastForWestFallback: def test_west_station_2017_keeps_goes16(self) -> None: """KSEA 2017: no West platform existed -> goes16 fallback must stay.""" From d1beb084d262260dc8aaff25ed1b1bb34cb8a909 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 13:48:45 +0200 Subject: [PATCH 8/9] =?UTF-8?q?fix(weather):=20month-granular=20cross-side?= =?UTF-8?q?=20clip=20=E2=80=94=20unify=20with=20the=20fallback=20rule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2 (re-stamp): the cross-side full-disk redundancy check was REQUEST-level (any native present in the request suppressed the cross-side platform outright), while the mirror-image East-for-West fallback was already month-granular. goes16,goes18 + ABI-L2-DSRF + KNYC in 2026: goes16 is era-clamped (no 2026 data), yet its mere presence in the request dropped goes18 — zero valid slices remained. Cross-side full-disk platforms now fall through to the SAME month-granular fallback machinery as East-for-West (_native_fully_covers against the requested natives): kept exactly for months no requested native fully covers, honored in full when no native is requested. Sector products remain always-excluded cross-side (true out-of-grid). One unified skip bucket + generalized log wording (both directions). Tests: era-clamped-native 2026 scenario keeps goes18 (12 months) and drops goes16; 2023 control still clips goes18 (goes16 fully covers). Co-Authored-By: Claude Opus 4.8 --- .../weather/satellite/_backfill.py | 44 +++++++------------ .../weather/tests/test_satellite_backfill.py | 24 ++++++++++ 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index 7db5342d..1c6682bc 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -928,8 +928,7 @@ def _enumerate_slices( from . import _sources out: list[tuple[str, str, StationInfo, int, int]] = [] - excluded: dict[str, list[str]] = {} # West platform x East station (out-of-grid) - cross_redundant: dict[str, int] = {} # full-disk cross-side skipped: natives cover + excluded: dict[str, list[str]] = {} # sector-blind West platform x East station clamped: dict[str, int] = {} # post-retirement months (_AVAILABLE_UNTIL) fallback_clipped: dict[str, int] = {} # East-for-West months a native platform covers goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites @@ -967,23 +966,21 @@ def _native_fully_covers(sat: str, year: int, month: int) -> bool: # West platform x East station — PRODUCT-aware # (Codex P2): PACUS/CONUS-sector products truly # cannot see an East station (out-of-grid only - # AFTER a full download), but FULL-DISK products - # (DSRF) see most of the Americas from either - # slot. So: sector product -> always excluded; - # full-disk with a requested native covering the - # station -> excluded as redundant (default cost - # control); full-disk WITHOUT any native in the - # request (explicit cross-side study, e.g. - # goes18+DSRF+KNYC) -> honored in full. + # AFTER a full download), so they are ALWAYS + # excluded. FULL-DISK products (DSRF) see most of + # the Americas from either slot, so they fall + # through to the SAME month-granular fallback rule + # as East-for-West below (Codex P2 re-stamp: a + # request-level redundancy check dropped goes18 + # DSRF for KNYC 2026 just because era-clamped + # goes16 was in the request) — kept only for + # months no requested native fully covers, and + # honored in full when no native is requested. if not _is_full_disk_goes_product(product): stations_for_sat = excluded.setdefault(sat, []) if info.icao not in stations_for_sat: stations_for_sat.append(info.icao) continue - if any(s in native for s in satellites): - cross_redundant[sat] = cross_redundant.get(sat, 0) + 1 - continue - # Explicit cross-side full-disk request: keep. # East platform x West station: era fallback — clip to # months no REQUESTED native platform covers (empty # list == explicit oblique request, honored in full). @@ -1023,22 +1020,13 @@ def _native_fully_covers(sat: str, year: int, month: int) -> bool: sat, ",".join(icaos), ) - for sat, n_months in cross_redundant.items(): - log.warning( - "GOES coverage filter: skipped %d cross-side FULL-DISK month-" - "slice(s) of %s — requested native platforms cover the station. " - "Request the cross-side platform WITHOUT natives to fetch its " - "oblique full-disk view explicitly.", - n_months, - sat, - ) for sat, n_months in fallback_clipped.items(): log.warning( - "GOES East-for-West fallback: clipped %d month-slice(s) of %s for " - "West-side station(s) — a requested native West platform FULLY " - "covers those months (first-light/partial months are NOT clipped). " - "Request the East platform WITHOUT native platforms to fetch the " - "oblique view explicitly.", + "GOES cross-side fallback: clipped %d month-slice(s) of %s — a " + "requested NATIVE platform FULLY covers those months " + "(first-light/partial/era-clamped months are NOT clipped). " + "Request the cross-side platform WITHOUT native platforms to " + "fetch its oblique view explicitly.", n_months, sat, ) diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 2c4797a7..56d3994d 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1674,6 +1674,30 @@ def test_default_request_fulldisk_still_excludes_cross_side(self) -> None: sats = {s[0] for s in slices} assert "goes17" not in sats and "goes18" not in sats + def test_cross_side_fulldisk_kept_when_requested_native_is_era_clamped(self) -> None: + """Codex P2 (re-stamp): the cross-side redundancy clip must be + MONTH-granular, not request-level. goes16,goes18 + DSRF + KNYC in + 2026: goes16 (the only requested native) has no 2026 data — goes18's + full-disk view is the ONLY valid coverage and must be kept.""" + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes18"], ["ABI-L2-DSRF"], infos, 2026, 2026 + ) + by_sat = {} + for sat, _p, _i, _y, m in slices: + by_sat.setdefault(sat, []).append(m) + assert sorted(by_sat.get("goes18", [])) == list(range(1, 13)) + assert "goes16" not in by_sat # era-clamped + + def test_cross_side_fulldisk_still_clipped_when_native_covers(self) -> None: + """Same request in 2023: goes16 fully covers -> goes18 clipped.""" + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes18"], ["ABI-L2-DSRF"], infos, 2023, 2023 + ) + sats = {s[0] for s in slices} + assert sats == {"goes16"} + def test_explicit_west_mixed_products_keeps_only_fulldisk(self) -> None: infos = _backfill._resolve_station_infos(["KNYC"]) slices = _backfill._enumerate_slices( From 7a90ffbc390bb4e504ba859cf32aaa11a448ca82 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sun, 5 Jul 2026 13:58:48 +0200 Subject: [PATCH 9/9] fix(weather): canonical-ICAO progress scopes; collapse SIM102 nested if MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex P2 (round 8): the progress scope key was built from RAW CLI station tokens, but bulk_backfill resolves NWS aliases to canonical ICAOs (NYC -> KNYC) via _resolve_station_infos — so an alias submission and a canonical submission of the SAME work got different progress objects: resume never rehydrated markers across the pair, and concurrent alias/canonical dispatches double-processed partitions. The scope now resolves stations to canonical ICAOs, dedupes, and sorts (order-insensitive) before keying. Also collapses the SIM102 nested-if in the cross-side branch (lint slipped through the previous chained run; behavior identical, verified by the full filter-family suite). Test: alias (NYC) and canonical (KNYC) submissions share one scope URI; multi-station order-insensitivity pinned. Co-Authored-By: Claude Opus 4.8 --- .../mostlyright/weather/satellite/__main__.py | 17 +++++-- .../weather/satellite/_backfill.py | 42 ++++++++-------- .../tests/satellite/test_cli_roster.py | 49 +++++++++++++++++++ 3 files changed, 82 insertions(+), 26 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/satellite/__main__.py b/packages/weather/src/mostlyright/weather/satellite/__main__.py index 109a60e6..62548720 100644 --- a/packages/weather/src/mostlyright/weather/satellite/__main__.py +++ b/packages/weather/src/mostlyright/weather/satellite/__main__.py @@ -489,13 +489,22 @@ def _run_backfill(args: argparse.Namespace) -> int: from ._progress import GcsProgressStore bucket = args.progress_bucket.rstrip("/") - if len(stations) <= 3: - scope_stations = "-".join(stations) + # CANONICAL station identity (Codex P2, round 8): the CLI accepts NWS + # aliases (NYC -> KNYC) which bulk_backfill resolves via + # _resolve_station_infos — so the scope must be built from the SAME + # resolved, deduped, sorted ICAOs or an alias submission and a + # canonical submission of identical work would get different progress + # objects (resume misses; concurrent double-work). + from ._resolve import _resolve_station_infos + + scope_icaos = sorted({i.icao for i in _resolve_station_infos(stations)}) + if len(scope_icaos) <= 3: + scope_stations = "-".join(scope_icaos) else: # Whole-roster scopes (e.g. the incremental Cloud Run Job) would make # an unwieldy object name — use a stable digest of the station tuple. - digest = hashlib.sha1("_".join(stations).encode()).hexdigest()[:10] - scope_stations = f"multi{len(stations)}-{digest}" + digest = hashlib.sha1("_".join(scope_icaos).encode()).hexdigest()[:10] + scope_stations = f"multi{len(scope_icaos)}-{digest}" # The scope must ALSO carry the satellites+products axes: an ACMC job # and a DSRF job for the same station+window are separate concurrent # work-scopes, and GcsProgressStore's mark_complete PUTs the full diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index 1c6682bc..cb39477a 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -962,28 +962,26 @@ def _native_fully_covers(sat: str, year: int, month: int) -> bool: if sat in goes_sats: native = _sources.goes_satellites_for_longitude(info.longitude) if sat not in native: - if sat in _sources._GOES_WEST_SATELLITES: - # West platform x East station — PRODUCT-aware - # (Codex P2): PACUS/CONUS-sector products truly - # cannot see an East station (out-of-grid only - # AFTER a full download), so they are ALWAYS - # excluded. FULL-DISK products (DSRF) see most of - # the Americas from either slot, so they fall - # through to the SAME month-granular fallback rule - # as East-for-West below (Codex P2 re-stamp: a - # request-level redundancy check dropped goes18 - # DSRF for KNYC 2026 just because era-clamped - # goes16 was in the request) — kept only for - # months no requested native fully covers, and - # honored in full when no native is requested. - if not _is_full_disk_goes_product(product): - stations_for_sat = excluded.setdefault(sat, []) - if info.icao not in stations_for_sat: - stations_for_sat.append(info.icao) - continue - # East platform x West station: era fallback — clip to - # months no REQUESTED native platform covers (empty - # list == explicit oblique request, honored in full). + # Cross-side platform — PRODUCT-aware (Codex P2): + # West PACUS/CONUS-sector products truly cannot see an + # East station (out-of-grid only AFTER a full + # download), so those are ALWAYS excluded. FULL-DISK + # products (DSRF) see most of the Americas from either + # slot, and East platforms genuinely cover West + # stations — both fall through to the month-granular + # fallback rule below (Codex P2 re-stamp: a + # request-level redundancy check dropped goes18 DSRF + # for KNYC 2026 just because era-clamped goes16 was in + # the request): kept only for months no requested + # native FULLY covers, honored in full when no native + # is requested. + if sat in _sources._GOES_WEST_SATELLITES and not ( + _is_full_disk_goes_product(product) + ): + stations_for_sat = excluded.setdefault(sat, []) + if info.icao not in stations_for_sat: + stations_for_sat.append(info.icao) + continue fallback_native = [s for s in satellites if s in native] until = _AVAILABLE_UNTIL.get(sat) if sat in goes_sats else None for year in range(year_start, year_end + 1): diff --git a/packages/weather/tests/satellite/test_cli_roster.py b/packages/weather/tests/satellite/test_cli_roster.py index 715d7d90..996633e6 100644 --- a/packages/weather/tests/satellite/test_cli_roster.py +++ b/packages/weather/tests/satellite/test_cli_roster.py @@ -617,3 +617,52 @@ def _dispatch(products: str, satellites: str) -> None: assert len(built) == 4 assert len({built[0], built[1], built[2]}) == 3 # three distinct scopes assert built[2] == built[3] # sorted digest: order-insensitive + + +def test_progress_scope_uses_canonical_station_identity(captured, monkeypatch): + """Codex P2 (round 8): an NWS alias submission (NYC) and the canonical + ICAO submission (KNYC) are the SAME work scope — they must share one + progress object (alias-keyed scopes broke resume rehydration and allowed + concurrent alias/canonical double-work). Order of a multi-station list + must not matter either.""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + + def _dispatch(stations: str) -> None: + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--satellites", + "goes16", + "--products", + "ABI-L2-ACMC", + "--stations", + stations, + "--year-start", + "2017", + "--year-end", + "2017", + "--out", + "/tmp/derived", + "--progress-bucket", + "marker-bkt", + ] + ) + assert rc == 0 + + _dispatch("KNYC") + _dispatch("NYC") # NWS alias of KNYC + _dispatch("KSEA,KNYC") + _dispatch("KNYC,KSEA") # order-insensitive + assert built[0] == built[1], "alias and canonical must share one scope" + assert "KNYC" in built[0] + assert built[2] == built[3], "station order must not change the scope"