diff --git a/.github/workflows/run-weather-backfill.yml b/.github/workflows/run-weather-backfill.yml index 723f134..7bb02fb 100644 --- a/.github/workflows/run-weather-backfill.yml +++ b/.github/workflows/run-weather-backfill.yml @@ -35,7 +35,7 @@ on: workflow_dispatch: inputs: image_tag: - description: "weather-backfill image tag to run (must already be pushed by deploy-weather-ingest)." + description: "weather-backfill image tag to run (must already be pushed by deploy-weather-ingest). MUST be built from a commit AFTER the GOES coverage filter (PR #101) when using the multi-satellite default below — an older image cartesian-products all requested satellites (wrong-hemisphere downloads)." required: true default: "latest" type: string @@ -58,6 +58,11 @@ on: required: true default: "ABI-L2-ACMC" type: string + satellites: + description: "Comma-separated satellites for pilot/station modes (default: the era-complete GOES set — REQUIRES an image built after the GOES coverage filter, PR #101; older images cartesian-product all four = wrong-hemisphere downloads). Override for non-GOES stations, e.g. himawari8,himawari9 + products=AHI-L2-FLDK-Clouds (Asia/RJTT) or viirs-npp,viirs-n20,viirs-n21 + products=VIIRS-JRR-CloudMask (Europe — Meteosat backfill is not wired yet). mode=full IGNORES this input entirely (roster defaults apply)." + required: true + default: "goes16,goes17,goes18,goes19" + type: string year_start: description: "First year (inclusive) of the backfill window (mode=pilot and mode=station)." required: true @@ -120,6 +125,7 @@ jobs: MODE: ${{ inputs.mode }} PILOT_STATION: ${{ inputs.pilot_station }} PRODUCTS: ${{ inputs.products }} + SATELLITES: ${{ inputs.satellites }} YEAR_START: ${{ inputs.year_start }} YEAR_END: ${{ inputs.year_end }} R2_BUCKET: ${{ vars.R2_BUCKET }} @@ -226,9 +232,17 @@ jobs: # proven prod path (the 2024 pilot processed + uploaded DSRF with # threads); numpy/h5py release the GIL for the heavy decode. Re-enable # process only after the spawn-context fix is proven in a container. + # Era-complete GOES set (goes19-era coverage fix): all four GOES + # platforms so East stations keep coverage past the 2025 + # goes16->goes19 handover and West stations gain the goes17 archive + # era. Requires an image built AFTER the hemisphere filter landed + # (the CLI enumerates only each station's own side pair; an OLD + # image would cartesian-product all four = wrong-hemisphere + # downloads). available_since clamps make the off-era platform + # nearly free. COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ - --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" \ - '["--mirror","gcp","--satellites","goes16","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') + --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb,"--max-workers","8"]') else TASK_COUNT=1 # Pilot machine — UNCHANGED (n2-standard-4, parallelism 16): a single @@ -242,11 +256,13 @@ jobs: YEAR_END_EFF="${YEAR_END:-$(date -u +%Y)}" # Pilot: explicit single-station backfill. The CLI's explicit mode # (no --roster) requires satellites/products/year-window/out, so pass - # them all. GOES-East ACMC covers the default KNYC pilot; an - # international pilot station needs --satellites overridden. + # them all. Era-complete GOES set — the CLI's hemisphere filter + # keeps the station on its own side pair (requires an image built + # after the filter landed); an international pilot station still + # needs --satellites overridden (Himawari/Meteosat). COMMANDS=$(jq -nc --arg st "$PILOT_STATION" --arg ys "$YEAR_START_EFF" --arg ye "$YEAR_END_EFF" \ - --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" \ - '["--mirror","gcp","--satellites","goes16","--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') + --arg pb "$PROGRESS_BUCKET" --arg rb "$R2_BUCKET" --arg pr "$PRODUCTS" --arg sats "$SATELLITES" \ + '["--mirror","gcp","--satellites",$sats,"--products",$pr,"--stations",$st,"--year-start",$ys,"--year-end",$ye,"--out","/tmp/derived","--r2-target","--r2-bucket",$rb,"--progress-bucket",$pb]') fi jq -n \ --arg img "${IMAGE}" \ diff --git a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py index df777d6..338c8a3 100644 --- a/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py +++ b/packages/weather/src/mostlyright/weather/_fetchers/_goes_s3.py @@ -120,6 +120,50 @@ "goes19": date(2024, 11, 15), } +# Operational-era END clamps for RETIRED platforms (goes19-era coverage fix, +# Codex P2) — the symmetric counterpart of _AVAILABLE_SINCE, consumed by the +# backfill's ENUMERATION filter so post-retirement (satellite, month) slices are +# never even submitted (each otherwise burns ~a task-minute of empty per-hour +# listing calls, growing every future fleet year). +# +# SAFETY over savings: a too-early date here would SILENTLY skip real +# settlement-adjacent data, while a too-late date only wastes cheap empty +# listings — so every entry carries ~1 YEAR of margin past the public +# retirement date, skips are logged loudly at enumeration, and enumeration-skip +# writes NO progress marker (removing/raising a date later backfills those +# months normally). Amend on any NOAA reactivation (standby platforms have +# historically been reactivated during operational anomalies). +# +# - goes17: operational GOES-West ended 2023-01-10 (goes18 took over +# 2023-01-04; goes17 to storage). Clamp 2024-01-01 = ~1 yr margin. +# - goes16: goes19 became the operational GOES-East 2025-04-07 (goes16 to +# standby). Clamp 2026-01-01 = ~9 mo margin, AND 2026 Jan-Jun was measured +# ALL-EMPTY live (KNYC station backfill, 2026-07-05) — the clamp is +# observation-backed, not date-arithmetic alone. +# - goes18 / goes19: OPERATIONAL — deliberately absent (no end clamp). +_AVAILABLE_UNTIL: dict[str, date] = { + "goes16": date(2026, 1, 1), + "goes17": date(2024, 1, 1), +} + +# UN-MARGINED actual last-data dates for retired platforms — used ONLY for +# POSITIVE coverage claims (the East-for-West fallback clip asks "does this +# native platform REALLY cover this month?"), NEVER for fetch clamps. The two +# maps deliberately lean opposite ways so both stay safe: _AVAILABLE_UNTIL is +# margined LATE (fetch side: a too-late date only wastes cheap empty listings), +# while claiming coverage from a margined date would suppress the East fallback +# for months with NO real native data (verified live: s3://noaa-goes17/ has no +# objects past 2023-01-10). Widening an _AVAILABLE_UNTIL margin must therefore +# never touch this map. +# - goes17: last data 2023-01-10 (went to storage; goes18 took over 2023-01-04). +# - goes16: goes19 became operational GOES-East 2025-04-07 (kept for +# documentation; East platforms are never consulted as NATIVE coverage — +# no fallback direction exists toward the East side). +_GOES_ACTUAL_DATA_END: dict[str, date] = { + "goes16": date(2025, 4, 7), + "goes17": date(2023, 1, 10), +} + # Conservative top-level S3/GCS call-rate cap (2i ``_GOES_S3_RATE_HZ``). The # empirical concurrency probe (25-05 D10) may later derive a tuned value. _GOES_S3_RATE_HZ = 20.0 @@ -134,10 +178,18 @@ _S3_FAIL_FAST_CODES = frozenset({"404", "NoSuchKey", "NoSuchBucket", "AccessDenied"}) # Per-product size cap (P2-d). Rejects an over-cap object BEFORE the full-object -# read so the in-memory BytesIO buffer is bounded. CONUS files run ~0.3-1.5 MB; -# DSRF full-disk runs ~50 MB. Caps are generous over the documented sizes. -# Default cap (CONUS-class) applies to any product not listed explicitly. -_DEFAULT_SIZE_CAP_BYTES = 2 * 1024 * 1024 # ~2 MB +# read so the in-memory BytesIO buffer is bounded. Default cap (CONUS-class) +# applies to any product not listed explicitly; DSRF full-disk (~50 MB) carries +# its own larger cap. +# +# 16 MiB, NOT the original 2 MiB: the 2 MiB cap was calibrated on 2017-era +# 2-variable ACMC files (~0.3-1.5 MB), but NOAA later added ACM + +# Cloud_Probabilities to the product and modern ACMC files run ~4.3-4.6 MB — +# observed LIVE 2026-07-05 (KNYC station backfill): the old cap rejected EVERY +# 2022+ ACMC file "before read", zeroing ACMC coverage for the modern era +# fleet-wide. 16 MiB = >3x headroom over the observed modern sizes while still +# bounding the in-memory buffer well below the DSRF cap. +_DEFAULT_SIZE_CAP_BYTES = 16 * 1024 * 1024 # 16 MiB _SIZE_CAP_BYTES: dict[str, int] = { p: (64 * 1024 * 1024 if p == "ABI-L2-DSRF" else _DEFAULT_SIZE_CAP_BYTES) for (p, _v) in PRODUCTS } diff --git a/packages/weather/src/mostlyright/weather/satellite/__main__.py b/packages/weather/src/mostlyright/weather/satellite/__main__.py index 0f54322..6254872 100644 --- a/packages/weather/src/mostlyright/weather/satellite/__main__.py +++ b/packages/weather/src/mostlyright/weather/satellite/__main__.py @@ -50,11 +50,17 @@ #: Default satellites for roster/incremental mode. batch.tf passes NO #: ``--satellites``, so roster mode must supply the canonical native-ring -#: default. The ACMC-only MVP (project memory) runs on the GOES East/West pair — -#: the two operational GOES platforms — so the fleet covers the Americas span the -#: settlement stations concentrate in. (Explicit-args mode still requires +#: default. ERA-COMPLETE GOES set (goes19-era coverage fix): the East pair +#: goes16 (operational 2017 -> standby 2025) + goes19 (GOES-East since the 2025 +#: handover) AND the West pair goes17 (archive, -> storage 2023) + goes18. The +#: old ("goes16", "goes18") pair silently lost East coverage after the 2025 +#: handover (observed live: KNYC 2026 backfilled ALL-EMPTY) and never had +#: goes17-era West coverage (2018-2022). Requesting all four is cheap: the +#: backfill hemisphere filter (_backfill._enumerate_slices) enumerates only the +#: station's own side pair, and the ``available_since`` clamp skips +#: pre-first-light slices with no I/O. (Explicit-args mode still requires #: ``--satellites``; this default ONLY applies when ``--roster`` is given.) -_DEFAULT_ROSTER_SATELLITES: tuple[str, ...] = ("goes16", "goes18") +_DEFAULT_ROSTER_SATELLITES: tuple[str, ...] = ("goes16", "goes17", "goes18", "goes19") #: Default products for roster/incremental mode: the ACMC-only MVP cloud mask #: (``ABI-L2-ACMC``, the cheap CONUS product — matches the ``_sources`` GOES @@ -96,9 +102,10 @@ def _build_parser() -> argparse.ArgumentParser: "--satellites", type=_split_csv, default=None, - help="Comma-separated satellites, e.g. goes16,goes19 (GOES-East) or goes18 " - "(GOES-West). REQUIRED in explicit mode; in --roster mode defaults to the " - "native-ring default (goes16,goes18 — the operational GOES pair).", + help="Comma-separated satellites, e.g. goes16,goes19 (GOES-East) or " + "goes17,goes18 (GOES-West). REQUIRED in explicit mode; in --roster mode " + "defaults to the era-complete GOES set (goes16,goes17,goes18,goes19 — " + "the hemisphere filter keeps each station on its own side pair).", ) bf.add_argument( "--products", @@ -464,14 +471,52 @@ def _run_backfill(args: argparse.Namespace) -> int: # crash-safe resume. Wire it to the GCS-backed progress store so a preempted # Spot slice (or a replacement VM) rehydrates its markers from GCS and SKIPS # already-uploaded partitions instead of reprocessing under a fresh local disk. - # Each shard writes a DISJOINT marker object (shard-index in the key) so array - # tasks never clobber each other's progress. + # + # SCOPE-keyed marker object (28-29 multi-station fix; was shard-index-keyed). + # The key is the RESOLVED (stations, year-window) scope, which is: + # - stable across Spot retries (the shard->(station,year) mapping is pure), + # - disjoint across array tasks (each owns a distinct station-year), + # - disjoint across CONCURRENT jobs for DIFFERENT stations — shard-index + # keys collided there: two station-mode jobs both wrote + # progress/shard-00000/, a read-modify-write clobber on the same object, + # - resume-stable across re-submissions of the same scope. + # Old shard-NNNNN objects are simply orphaned (one idempotent re-fetch for + # any scope whose markers lived there; slice keys inside files are already + # full-identity so no wrong-skip is possible either way). if args.progress_bucket is not None: + import hashlib + from ._progress import GcsProgressStore - shard_index, _shard_count = _resolve_shard_index_count(args) bucket = args.progress_bucket.rstrip("/") - gcs_uri = f"gs://{bucket}/progress/shard-{shard_index:05d}/progress.json" + # CANONICAL station identity (Codex P2, round 8): the CLI accepts NWS + # aliases (NYC -> KNYC) which bulk_backfill resolves via + # _resolve_station_infos — so the scope must be built from the SAME + # resolved, deduped, sorted ICAOs or an alias submission and a + # canonical submission of identical work would get different progress + # objects (resume misses; concurrent double-work). + from ._resolve import _resolve_station_infos + + scope_icaos = sorted({i.icao for i in _resolve_station_infos(stations)}) + if len(scope_icaos) <= 3: + scope_stations = "-".join(scope_icaos) + else: + # Whole-roster scopes (e.g. the incremental Cloud Run Job) would make + # an unwieldy object name — use a stable digest of the station tuple. + digest = hashlib.sha1("_".join(scope_icaos).encode()).hexdigest()[:10] + scope_stations = f"multi{len(scope_icaos)}-{digest}" + # The scope must ALSO carry the satellites+products axes: an ACMC job + # and a DSRF job for the same station+window are separate concurrent + # work-scopes, and GcsProgressStore's mark_complete PUTs the full + # in-memory map — same-object concurrent writers are last-write-wins. + # SORTED so goes16,goes19 vs goes19,goes16 map to the same scope. + sp_digest = hashlib.sha1( + (",".join(sorted(satellites)) + "#" + ",".join(sorted(products))).encode() + ).hexdigest()[:8] + gcs_uri = ( + f"gs://{bucket}/progress/" + f"{scope_stations}_{year_start}-{year_end}_{sp_digest}/progress.json" + ) kwargs["progress_store"] = GcsProgressStore(gcs_uri) print(f"using durable GCS progress store for crash-safe resume: {gcs_uri}") diff --git a/packages/weather/src/mostlyright/weather/satellite/_backfill.py b/packages/weather/src/mostlyright/weather/satellite/_backfill.py index b498c1a..cb39477 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_backfill.py +++ b/packages/weather/src/mostlyright/weather/satellite/_backfill.py @@ -68,8 +68,11 @@ # Transport (25-03). These bare module-level names are what tests monkeypatch # (``mock.patch.object(_backfill, "list_product_keys")``); the bulk/slice code # below references them by bare name so a patched binding is honored. +from mostlyright.weather._fetchers._goes_extract import PRODUCTS as _GOES_PRODUCTS from mostlyright.weather._fetchers._goes_s3 import ( _AVAILABLE_SINCE, + _AVAILABLE_UNTIL, + _GOES_ACTUAL_DATA_END, extract_pixel, list_product_keys, ) @@ -875,6 +878,20 @@ def _validate_partition_components(satellites: list[str], products: list[str]) - _sources.validate_satellite_and_product(sat, product) +def _is_full_disk_goes_product(product: str) -> bool: + """True iff ``product``'s registry grid is the 5424x5424 full ABI disk. + + Full-disk products see most of the Americas from EITHER GOES slot (oblique + at the far edge), so the West-for-East sector exclusion must not apply to + them — only PACUS/CONUS-sector products are truly blind cross-side. + """ + return any( + tuple(pv.grid_shape_expected) == (5424, 5424) + for (p, _v), pv in _GOES_PRODUCTS.items() + if p == product + ) + + def _enumerate_slices( satellites: list[str], products: list[str], @@ -882,19 +899,138 @@ def _enumerate_slices( year_start: int, year_end: int, ) -> list[tuple[str, str, StationInfo, int, int]]: - """Enumerate per-(satellite, product, station, year, month) slices.""" + """Enumerate per-(satellite, product, station, year, month) slices. + + GOES coverage filter (goes19-era coverage fix + Codex P1 revision). The + East/West longitude split is a ROUTING preference, not a coverage boundary, + and the sector geometry is ASYMMETRIC: + + - GOES-West (goes17/goes18) publishes the PACUS sector, which truly + CANNOT see East-side stations -> a West platform is NEVER enumerated + for an East station (whole-month downloads would fail out-of-grid only + AFTER the bytes are fetched — pure spend, zero rows). + - GOES-East (goes16/goes19) CONUS/full-disk genuinely covers the whole + Americas (oblique view at the far West). An East platform IS valid + coverage for a West station — and for 2017 -> 2018-08-27 (goes17 first + light) goes16 was the ONLY ABI platform in existence. So East + platforms are kept for West stations as a month-granular era FALLBACK: + only for months that NO requested native West platform covers. If the + request contains NO native West platform at all (an explicit + oblique-view study, e.g. ``--satellites goes16 --stations KSEA``), the + explicit request is honored in full — never silently narrowed. + + Retired platforms are clamped past their margined _AVAILABLE_UNTIL date + (post-retirement months would only burn empty listing calls). Non-GOES + satellites (Himawari / VIIRS / Meteosat) are NOT filtered here; their + sources own their coverage. Every skip class is logged LOUDLY — never + silent under-coverage. + """ + from . import _sources + out: list[tuple[str, str, StationInfo, int, int]] = [] + excluded: dict[str, list[str]] = {} # sector-blind West platform x East station + clamped: dict[str, int] = {} # post-retirement months (_AVAILABLE_UNTIL) + fallback_clipped: dict[str, int] = {} # East-for-West months a native platform covers + goes_sats = _sources.spec_for_source(_GOES_SOURCE).satellites + + def _native_fully_covers(sat: str, year: int, month: int) -> bool: + """POSITIVE coverage claim: does ``sat`` cover EVERY day of the month? + + Used only to decide whether the East-for-West fallback may be clipped, + so both bounds lean CONSERVATIVE (a false "covered" silently loses real + data; a false "not covered" only over-fetches a few duplicate days into + separate per-satellite partitions — the stated-safe direction): + - FULL month: first day >= available_since. The child-side fetch + clamp skips days BEFORE first light, so a first-light month (e.g. + goes17 Aug 2018, since 08-27) yields only its 5-day tail — it must + NOT clip the East platform that covers the other 26 days. + - UN-MARGINED end: last day < _GOES_ACTUAL_DATA_END (the real + last-data date), NOT the margined _AVAILABLE_UNTIL — the margin is + fetch-side slack and claiming coverage from it would suppress the + fallback for months with no real native data (goes17 Feb-Dec 2023). + """ + since = _AVAILABLE_SINCE.get(sat) + if since is not None and date(year, month, 1) < since: + return False + data_end = _GOES_ACTUAL_DATA_END.get(sat) + return not (data_end is not None and _last_day_of_month(year, month) >= data_end) + for sat in satellites: for product in products: for info in station_infos: + fallback_native: list[str] = [] + if sat in goes_sats: + native = _sources.goes_satellites_for_longitude(info.longitude) + if sat not in native: + # Cross-side platform — PRODUCT-aware (Codex P2): + # West PACUS/CONUS-sector products truly cannot see an + # East station (out-of-grid only AFTER a full + # download), so those are ALWAYS excluded. FULL-DISK + # products (DSRF) see most of the Americas from either + # slot, and East platforms genuinely cover West + # stations — both fall through to the month-granular + # fallback rule below (Codex P2 re-stamp: a + # request-level redundancy check dropped goes18 DSRF + # for KNYC 2026 just because era-clamped goes16 was in + # the request): kept only for months no requested + # native FULLY covers, honored in full when no native + # is requested. + if sat in _sources._GOES_WEST_SATELLITES and not ( + _is_full_disk_goes_product(product) + ): + stations_for_sat = excluded.setdefault(sat, []) + if info.icao not in stations_for_sat: + stations_for_sat.append(info.icao) + continue + fallback_native = [s for s in satellites if s in native] + until = _AVAILABLE_UNTIL.get(sat) if sat in goes_sats else None for year in range(year_start, year_end + 1): for month in range(1, 13): + # Operational-era END clamp: a retired GOES platform is + # not enumerated past its margined until-date. NO + # progress marker is written for clamped slices, so + # raising/removing the date later backfills them. + if until is not None and date(year, month, 1) >= until: + clamped[sat] = clamped.get(sat, 0) + 1 + continue + if fallback_native and any( + _native_fully_covers(s, year, month) for s in fallback_native + ): + fallback_clipped[sat] = fallback_clipped.get(sat, 0) + 1 + continue out.append((sat, product, info, year, month)) + for sat, n_months in clamped.items(): + log.warning( + "GOES operational-era clamp: NOT enumerating %d post-retirement " + "month-slice(s) for %s (until %s, margined — see _AVAILABLE_UNTIL). " + "Raise/remove the date to backfill them (no progress marker is " + "written for clamped slices).", + n_months, + sat, + _AVAILABLE_UNTIL[sat].isoformat(), + ) + for sat, icaos in excluded.items(): + log.warning( + "GOES coverage filter: NOT enumerating West platform %s for " + "East-side station(s) %s — its PACUS/CONUS-sector products cannot " + "see them (out-of-grid only AFTER a full download). GOES-East " + "covers them. (Full-disk products are handled separately.)", + sat, + ",".join(icaos), + ) + for sat, n_months in fallback_clipped.items(): + log.warning( + "GOES cross-side fallback: clipped %d month-slice(s) of %s — a " + "requested NATIVE platform FULLY covers those months " + "(first-light/partial/era-clamped months are NOT clipped). " + "Request the cross-side platform WITHOUT native platforms to " + "fetch its oblique view explicitly.", + n_months, + sat, + ) return out -# --------------------------------------------------------------------------- -# Resume layer — durable progress (fsync) + .bak fallback + validation. # --------------------------------------------------------------------------- def _progress_key(satellite: str, product: str, station: str, year: int, month: int) -> str: """Return the resume key encoding the FULL slice identity (P1-2). diff --git a/packages/weather/src/mostlyright/weather/satellite/_sources.py b/packages/weather/src/mostlyright/weather/satellite/_sources.py index ef4940e..985a50d 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_sources.py +++ b/packages/weather/src/mostlyright/weather/satellite/_sources.py @@ -416,6 +416,34 @@ def _goes_route(longitude: float) -> RouteResult: return RouteResult(source="noaa_goes", satellite=satellite, product=spec.default_product) +#: GOES era pairs by side of the East/West split: every platform that EVER +#: served that side. East = goes16 (operational 2017 -> standby 2025, when +#: goes19 took over as GOES-East) + goes19; West = goes17 (operational +#: 2019 -> storage 2023, goes18 took over) + goes18. Backfills request the +#: whole side pair for era-complete coverage — the per-slice +#: ``available_since`` clamp skips pre-first-light slices with no I/O, so the +#: extra platform is nearly free — while the hemisphere filter +#: (``_backfill._enumerate_slices``) keys on +#: :func:`goes_satellites_for_longitude` so a station NEVER enumerates the +#: other side's archive (whole-month downloads that fail out-of-grid only +#: AFTER the bytes are fetched). +_GOES_EAST_SATELLITES: frozenset[str] = frozenset({"goes16", "goes19"}) +_GOES_WEST_SATELLITES: frozenset[str] = frozenset({"goes17", "goes18"}) + + +def goes_satellites_for_longitude(longitude: float) -> frozenset[str]: + """Return every GOES platform that ever covered ``longitude``'s side. + + Uses the SAME ``_GOES_EAST_WEST_SPLIT_LON_DEG`` split as + :func:`_goes_route` (single source of truth): ``longitude < split`` is + GOES-West territory, else GOES-East — so the live router and the backfill + visibility filter can never disagree on a station's side. + """ + if longitude < _GOES_EAST_WEST_SPLIT_LON_DEG: + return _GOES_WEST_SATELLITES + return _GOES_EAST_SATELLITES + + def _default_route(source: str) -> RouteResult: """Build the spec-default route for a source (no per-platform override).""" spec = _SOURCES[source] diff --git a/packages/weather/tests/satellite/test_cli_roster.py b/packages/weather/tests/satellite/test_cli_roster.py index 15104a3..996633e 100644 --- a/packages/weather/tests/satellite/test_cli_roster.py +++ b/packages/weather/tests/satellite/test_cli_roster.py @@ -323,8 +323,18 @@ def __init__(self, gcs_uri, *, fs=None): assert len(captured) == 1 # A durable GCS progress store was wired into bulk_backfill's kwargs... assert isinstance(captured[0]["progress_store"], _FakeStore) - # ...at a shard-disjoint marker URI. - assert built == ["gs://marker-bkt/progress/shard-00003/progress.json"] + # ...at a SCOPE-keyed marker URI (28-29): the resolved (stations, year + # window), NOT the shard index — shard-index keys collided across + # concurrent station-mode jobs (both wrote progress/shard-00000/). + year_start, year_end, _ = _roster_year_window() + stations, y_s, y_e = shard_roster( + resolve_roster("kalshi,polymarket"), 3, 65, year_start=year_start, year_end=year_end + ) + # Scope carries the satellites+products axes too (sorted digest) so + # concurrent same-station jobs with different products never clobber. + sp = _sp_digest(["goes16"], list(cli._DEFAULT_ROSTER_PRODUCTS)) + expected = f"gs://marker-bkt/progress/{'-'.join(stations)}_{y_s}-{y_e}_{sp}/progress.json" + assert built == [expected] def test_explicit_mode_unchanged(captured, tmp_path): @@ -465,3 +475,194 @@ def test_explicit_multi_station_shard(captured): assert kw["stations"] == ["KDEN"] assert kw["year_start"] == 2017 assert kw["year_end"] == 2017 + + +def test_progress_scope_keys_disjoint_across_stations_and_years(captured, monkeypatch): + """Two station-mode jobs for DIFFERENT stations (or year shards) must write + DISJOINT progress objects — the old shard-index key collided (both jobs' + task 0 wrote progress/shard-00000/, a concurrent read-modify-write clobber).""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + + def _dispatch(station: str, index: str) -> None: + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--satellites", + "goes16", + "--products", + "ABI-L2-ACMC", + "--stations", + station, + "--year-start", + "2017", + "--year-end", + "2026", + "--out", + "/tmp/derived", + "--progress-bucket", + "marker-bkt", + "--shard-index", + index, + "--shard-count", + "10", + ] + ) + assert rc == 0 + + _dispatch("KNYC", "0") # KNYC 2017 (year shard 0) + _dispatch("RJTT", "0") # RJTT 2017 — same shard index, different station + _dispatch("KNYC", "1") # KNYC 2018 — same station, different year shard + sp = _sp_digest(["goes16"], ["ABI-L2-ACMC"]) + assert built == [ + f"gs://marker-bkt/progress/KNYC_2017-2017_{sp}/progress.json", + f"gs://marker-bkt/progress/RJTT_2017-2017_{sp}/progress.json", + f"gs://marker-bkt/progress/KNYC_2018-2018_{sp}/progress.json", + ] + assert len(set(built)) == 3 # fully disjoint + + +def test_progress_scope_key_multi_station_uses_digest(captured, monkeypatch): + """A whole-roster scope (>3 stations) digests the station tuple so the + object name stays bounded but deterministic.""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--roster", + "kalshi,polymarket", + "--satellites", + "goes16", + "--progress-bucket", + "marker-bkt", + ] + ) + assert rc == 0 + assert len(built) == 1 + assert built[0].startswith("gs://marker-bkt/progress/multi65-") + assert built[0].endswith("/progress.json") + + +def _sp_digest(satellites: list[str], products: list[str]) -> str: + """Mirror of the CLI's scope digest: sorted satellites + sorted products.""" + import hashlib + + return hashlib.sha1( + (",".join(sorted(satellites)) + "#" + ",".join(sorted(products))).encode() + ).hexdigest()[:8] + + +def test_progress_scope_disjoint_across_products_and_satellites(captured, monkeypatch): + """Exit-panel P2 regression: an ACMC job and a DSRF job for the SAME + station+window are separate concurrent work-scopes — their progress objects + must differ (GcsProgressStore full-map PUTs are last-write-wins on a shared + object). Satellite-set changes must also re-scope; ordering must NOT.""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + + def _dispatch(products: str, satellites: str) -> None: + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--satellites", + satellites, + "--products", + products, + "--stations", + "KNYC", + "--year-start", + "2017", + "--year-end", + "2017", + "--out", + "/tmp/derived", + "--progress-bucket", + "marker-bkt", + ] + ) + assert rc == 0 + + _dispatch("ABI-L2-ACMC", "goes16") + _dispatch("ABI-L2-DSRF", "goes16") # product axis -> new scope + _dispatch("ABI-L2-ACMC", "goes16,goes19") # satellite axis -> new scope + _dispatch("ABI-L2-ACMC", "goes19,goes16") # ORDER only -> same scope as #3 + assert len(built) == 4 + assert len({built[0], built[1], built[2]}) == 3 # three distinct scopes + assert built[2] == built[3] # sorted digest: order-insensitive + + +def test_progress_scope_uses_canonical_station_identity(captured, monkeypatch): + """Codex P2 (round 8): an NWS alias submission (NYC) and the canonical + ICAO submission (KNYC) are the SAME work scope — they must share one + progress object (alias-keyed scopes broke resume rehydration and allowed + concurrent alias/canonical double-work). Order of a multi-station list + must not matter either.""" + built: list[str] = [] + + class _FakeStore: + def __init__(self, gcs_uri, *, fs=None): + built.append(gcs_uri) + + from mostlyright.weather.satellite import _progress + + monkeypatch.setattr(_progress, "GcsProgressStore", _FakeStore) + + def _dispatch(stations: str) -> None: + rc = cli.main( + [ + "backfill", + "--mirror", + "gcp", + "--satellites", + "goes16", + "--products", + "ABI-L2-ACMC", + "--stations", + stations, + "--year-start", + "2017", + "--year-end", + "2017", + "--out", + "/tmp/derived", + "--progress-bucket", + "marker-bkt", + ] + ) + assert rc == 0 + + _dispatch("KNYC") + _dispatch("NYC") # NWS alias of KNYC + _dispatch("KSEA,KNYC") + _dispatch("KNYC,KSEA") # order-insensitive + assert built[0] == built[1], "alias and canonical must share one scope" + assert "KNYC" in built[0] + assert built[2] == built[3], "station order must not change the scope" diff --git a/packages/weather/tests/test_satellite_backfill.py b/packages/weather/tests/test_satellite_backfill.py index 9198bbb..56d3994 100644 --- a/packages/weather/tests/test_satellite_backfill.py +++ b/packages/weather/tests/test_satellite_backfill.py @@ -1402,9 +1402,12 @@ def _slice(**kw): # Future months legitimately fetch + write nothing (no data yet). return _stub_slice(**kw) + # goes19 (OPERATIONAL, no _AVAILABLE_UNTIL clamp): a 2099 goes16 window + # would now be era-clamped at enumeration and this test would pass + # vacuously without exercising the resume semantics. with mock.patch.object(_backfill, "backfill_goes_satellite", _slice): _backfill.bulk_backfill( - satellites=["goes16"], + satellites=["goes19"], products=["ABI-L2-ACMC"], stations=["KNYC"], year_start=2099, @@ -1415,7 +1418,7 @@ def _slice(**kw): progress = _backfill._load_progress(progress_path) # No 2099 month may be marked completed — they are all in the future. future_keys = [ - _backfill._progress_key("goes16", "ABI-L2-ACMC", "KNYC", 2099, m) for m in range(1, 13) + _backfill._progress_key("goes19", "ABI-L2-ACMC", "KNYC", 2099, m) for m in range(1, 13) ] assert all(progress.get(k) != _backfill._PROGRESS_COMPLETED for k in future_keys) @@ -1432,9 +1435,11 @@ def _spy(**kw): ran.append(kw["month"]) return _slice(**kw) + # goes19: operational (no era clamp) so the 2099 future window + # actually enumerates — see the note in the sibling test above. with mock.patch.object(_backfill, "backfill_goes_satellite", _spy): _backfill.bulk_backfill( - satellites=["goes16"], + satellites=["goes19"], products=["ABI-L2-ACMC"], stations=["KNYC"], year_start=2099, @@ -1521,3 +1526,281 @@ def test_is_month_fully_elapsed_predicate(self) -> None: assert _backfill._is_month_fully_elapsed(now.year, now.month) is False # A far-future month is NOT fully elapsed. assert _backfill._is_month_fully_elapsed(now.year + 5, 1) is False + + +# --------------------------------------------------------------------------- +# GOES hemisphere visibility filter (goes19-era coverage fix): _enumerate_slices +# must intersect GOES satellites with the station's side of the East/West split +# so a KNYC backfill never enumerates goes17/goes18 slices (whole-archive +# downloads that fail per-file out-of-grid AFTER the bytes are fetched) and a +# KSEA backfill never enumerates goes16/goes19. Non-GOES satellites are not +# filtered here (their sources own their coverage). +# --------------------------------------------------------------------------- +class TestGoesHemisphereFilter: + def test_east_station_only_east_satellites(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert sats == {"goes16", "goes19"} + + def test_west_station_only_west_satellites(self) -> None: + # Probe year 2022: BOTH West platforms in-era (goes17 until-clamps at + # 2024 — that axis is TestGoesOperationalUntilClamp's, not this one's). + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2022, 2022 + ) + sats = {s[0] for s in slices} + assert sats == {"goes17", "goes18"} + + def test_mixed_stations_get_their_own_pairs(self) -> None: + # Probe year 2022 — inside every platform's era (see note above). + infos = _backfill._resolve_station_infos(["KNYC", "KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2022, 2022 + ) + by_station: dict[str, set[str]] = {} + for sat, _product, info, _y, _m in slices: + by_station.setdefault(info.icao, set()).add(sat) + assert by_station["KNYC"] == {"goes16", "goes19"} + assert by_station["KSEA"] == {"goes17", "goes18"} + + def test_single_requested_satellite_matching_hemisphere_unchanged(self) -> None: + # The pre-fix pilot shape (--satellites goes16 for KNYC) is untouched: + # 12 months x 1 product x 1 satellite. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2024, 2024) + assert len(slices) == 12 + + +# --------------------------------------------------------------------------- +# GOES operational-era END clamp (Codex P2 on the era-coverage fix): a retired +# platform must not be enumerated past its conservative until-date, or every +# future fleet year burns empty per-hour listing calls on it (goes17 after its +# 2023 storage, goes16 after the 2025 goes19 handover). The until-dates carry +# ~1 year of safety margin and skips are LOUD — a wrong date would silently +# lose real settlement-adjacent data, so margin + logging are load-bearing. +# --------------------------------------------------------------------------- +class TestGoesOperationalUntilClamp: + def test_goes16_not_enumerated_after_retirement_margin(self) -> None: + # 2026: measured ALL-EMPTY live (2026-07-05 station run) — past the + # margined clamp, so the slices must not even enumerate. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes19"], ["ABI-L2-ACMC"], infos, 2026, 2026 + ) + sats = {s[0] for s in slices} + assert sats == {"goes19"} + + def test_goes16_still_enumerated_inside_margin(self) -> None: + # 2025: real data through the ~April handover — inside margin, keep. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2025, 2025) + assert len(slices) == 12 + + def test_goes17_not_enumerated_after_retirement_margin(self) -> None: + # goes17 went to storage Jan 2023; clamp (with margin) cuts 2024+. + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes17", "goes18"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert sats == {"goes18"} + + def test_goes17_era_still_enumerated(self) -> None: + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices(["goes17"], ["ABI-L2-ACMC"], infos, 2022, 2022) + assert len(slices) == 12 + + def test_operational_platforms_have_no_until(self) -> None: + # goes18/goes19 are operational: never clamped, any current/future year. + infos = _backfill._resolve_station_infos(["KNYC", "KSEA"]) + slices = _backfill._enumerate_slices( + ["goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2026, 2026 + ) + sats = {s[0] for s in slices} + assert sats == {"goes18", "goes19"} + + def test_partial_year_clamps_only_post_until_months(self) -> None: + # If the until-date falls mid-window the clamp is month-granular, not + # year-granular: a 2025-2026 goes16 request keeps all 12 months of 2025 + # and drops all of 2026 (until 2026-01-01). + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2025, 2026) + months = sorted((y, m) for _s, _p, _i, y, m in slices) + assert months == [(2025, m) for m in range(1, 13)] + + +# --------------------------------------------------------------------------- +# GOES East-for-West era FALLBACK (Codex P1 on the hemisphere filter): the +# East/West split is a ROUTING preference, not a coverage boundary. GOES-East +# CONUS/full-disk genuinely covers the whole Americas (oblique view), and for +# 2017 -> 2018-08-27 (goes17 first light) goes16 was the ONLY ABI platform — +# a pure hemisphere filter silently loses the West side's 2017-2018 era. +# Direction-correct rules: +# - West platforms (goes17/goes18 PACUS) for EAST stations: still excluded +# (true out-of-grid). +# - East platforms (goes16/goes19) for WEST stations: kept ONLY for months +# no REQUESTED native West platform covers (the pre-native fallback era) — +# unless NO native platform was requested at all (explicit oblique study), +# in which case the request is honored fully. +# --------------------------------------------------------------------------- +class TestGoesFullDiskCrossSide: + """Codex P2 (final stamp): the West-for-East exclusion must be + PRODUCT-aware. PACUS/CONUS-sector products truly cannot see East stations, + but FULL-DISK products (ABI-L2-DSRF, 5424x5424) see most of the Americas + from either slot — an explicit goes18+DSRF+KNYC request is geometrically + valid (oblique) and must not be silently dropped. Default requests still + exclude the cross-side platform when natives cover it (cost control).""" + + def test_explicit_west_fulldisk_for_east_station_honored(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes18"], ["ABI-L2-DSRF"], infos, 2023, 2023) + assert len(slices) == 12 + + def test_explicit_west_conus_for_east_station_still_excluded(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices(["goes18"], ["ABI-L2-ACMC"], infos, 2023, 2023) + assert slices == [] + + def test_default_request_fulldisk_still_excludes_cross_side(self) -> None: + # Natives (goes16/goes19) cover KNYC DSRF — no cross-side redundancy. + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-DSRF"], infos, 2023, 2023 + ) + sats = {s[0] for s in slices} + assert "goes17" not in sats and "goes18" not in sats + + def test_cross_side_fulldisk_kept_when_requested_native_is_era_clamped(self) -> None: + """Codex P2 (re-stamp): the cross-side redundancy clip must be + MONTH-granular, not request-level. goes16,goes18 + DSRF + KNYC in + 2026: goes16 (the only requested native) has no 2026 data — goes18's + full-disk view is the ONLY valid coverage and must be kept.""" + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes18"], ["ABI-L2-DSRF"], infos, 2026, 2026 + ) + by_sat = {} + for sat, _p, _i, _y, m in slices: + by_sat.setdefault(sat, []).append(m) + assert sorted(by_sat.get("goes18", [])) == list(range(1, 13)) + assert "goes16" not in by_sat # era-clamped + + def test_cross_side_fulldisk_still_clipped_when_native_covers(self) -> None: + """Same request in 2023: goes16 fully covers -> goes18 clipped.""" + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes18"], ["ABI-L2-DSRF"], infos, 2023, 2023 + ) + sats = {s[0] for s in slices} + assert sats == {"goes16"} + + def test_explicit_west_mixed_products_keeps_only_fulldisk(self) -> None: + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes18"], ["ABI-L2-ACMC", "ABI-L2-DSRF"], infos, 2023, 2023 + ) + prods = {s[1] for s in slices} + assert prods == {"ABI-L2-DSRF"} + assert len(slices) == 12 + + +class TestGoesEastForWestFallback: + def test_west_station_2017_keeps_goes16(self) -> None: + """KSEA 2017: no West platform existed -> goes16 fallback must stay.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2017, 2017 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 13)) + + def test_west_station_2019_drops_goes16(self) -> None: + """KSEA 2019: goes17 (native) covers every month -> no East fallback.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2019, 2019 + ) + sats = {s[0] for s in slices} + assert "goes16" not in sats + assert {"goes17", "goes18"} <= sats + + def test_west_station_2018_fallback_keeps_partial_first_light_month(self) -> None: + """2018: goes17 first light 2018-08-27 -> goes16 must cover Jan-AUG. + + Exit-panel P1 regression: the first cut treated Aug 2018 as + goes17-covered because the month's LAST day >= first light — but the + child-side fetch clamp only pulls Aug 27-31, so clipping goes16 left + Aug 1-26 fetched by NO platform for every West station. A partial + native month must NOT clip the East fallback (the ~5 overlap days land + in separate per-satellite partitions — the safe direction).""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2018, 2018 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 9)) # Jan..Aug (Aug = partial goes17) + + def test_west_station_explicit_east_only_request_honored(self) -> None: + """--satellites goes16 --stations KSEA (no native platform requested): + an explicit oblique-view request runs FULLY — never silently narrowed.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices(["goes16"], ["ABI-L2-ACMC"], infos, 2024, 2024) + assert len(slices) == 12 + + def test_east_station_still_never_gets_west_platforms(self) -> None: + """KNYC x goes17/goes18 stays excluded — PACUS truly cannot see the + East coast (out-of-grid after full download; the original pathology).""" + infos = _backfill._resolve_station_infos(["KNYC"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2017, 2017 + ) + sats = {s[0] for s in slices} + assert sats == {"goes16", "goes19"} + + def test_legacy_pair_request_keeps_goes16_for_partial_jul_2022(self) -> None: + """Exit-panel P1 sibling: goes16,goes18 request (the OLD default pair) + for a West station in 2022 — goes18 first light 2022-07-28 makes July a + partial month, so goes16 must stay for Jan..Jul (not Jan..Jun).""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes18"], ["ABI-L2-ACMC"], infos, 2022, 2022 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 8)) # Jan..Jul + + def test_margined_until_is_not_a_coverage_claim(self) -> None: + """Exit-panel P2 regression: goes17's margined _AVAILABLE_UNTIL + (2024-01-01) must NOT count as coverage for 2023 — its ACTUAL data ends + 2023-01-10. A goes16,goes17 request for KSEA 2023 keeps the goes16 + fallback for ALL 12 months (Jan is partial: data ends mid-month).""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17"], ["ABI-L2-ACMC"], infos, 2023, 2023 + ) + goes16_months = sorted(m for s, _p, _i, _y, m in slices if s == "goes16") + assert goes16_months == list(range(1, 13)) + + def test_default_request_2023_still_clips_via_goes18(self) -> None: + """Default all-four request for KSEA 2023: goes18 (operational, full + coverage) legitimately clips the goes16 fallback for every month.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2023, 2023 + ) + sats = {s[0] for s in slices} + assert "goes16" not in sats + + def test_west_station_2024_native_gap_falls_back_to_east(self) -> None: + """2024: goes17 is until-clamped (2024-01-01) but goes18 (native, since + 2022-07-28) covers every month -> STILL no East fallback.""" + infos = _backfill._resolve_station_infos(["KSEA"]) + slices = _backfill._enumerate_slices( + ["goes16", "goes17", "goes18", "goes19"], ["ABI-L2-ACMC"], infos, 2024, 2024 + ) + sats = {s[0] for s in slices} + assert "goes16" not in sats + assert sats == {"goes18"} diff --git a/packages/weather/tests/test_satellite_s3.py b/packages/weather/tests/test_satellite_s3.py index 087a14d..16900de 100644 --- a/packages/weather/tests/test_satellite_s3.py +++ b/packages/weather/tests/test_satellite_s3.py @@ -286,6 +286,21 @@ def test_open_and_extract_reads_bytesio_not_lazy_file(knyc: StationInfo) -> None fs.open.assert_not_called() +def test_size_cap_accepts_modern_acmc_file_sizes() -> None: + """Live-fleet regression (2026-07-05, KNYC station run): modern ACMC files + are ~4.3-4.6 MB — NOAA added ACM + Cloud_Probabilities to the product + (~2021+), roughly doubling 2017-era 2-variable file sizes. The old 2 MiB + default cap rejected EVERY modern ACMC file "before read", so ACMC 2021+ + produced zero rows fleet-wide. The cap must comfortably admit the observed + modern sizes while still rejecting pathological objects.""" + observed_modern_acmc = 4_614_061 # largest size seen live (2022/002) + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-ACMC"] > observed_modern_acmc * 2 + # Still a real guard: pathological objects stay rejected. + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-ACMC"] < 64 * 1024 * 1024 + # DSRF full-disk keeps its own larger cap. + assert _goes_s3._SIZE_CAP_BYTES["ABI-L2-DSRF"] == 64 * 1024 * 1024 + + def test_size_cap_rejects_before_read(knyc: StationInfo) -> None: """An over-cap object is rejected before any object read on either mirror.""" fs = mock.MagicMock() diff --git a/packages/weather/tests/test_satellite_sources.py b/packages/weather/tests/test_satellite_sources.py index 8effda6..2789cce 100644 --- a/packages/weather/tests/test_satellite_sources.py +++ b/packages/weather/tests/test_satellite_sources.py @@ -130,3 +130,29 @@ def test_validate_satellite_and_product_rejects_cross_source_product() -> None: # An ABI product on a Himawari satellite is a category error. with pytest.raises(ValueError): _sources.validate_satellite_and_product("himawari9", "ABI-L2-ACMC") + + +# --------------------------------------------------------------------------- +# GOES era-set helper: which GOES platforms EVER covered a longitude. The +# backfill visibility filter (_backfill._enumerate_slices) keys on this so an +# East-CONUS station never enumerates (and downloads!) the West archive and +# vice versa, while era completeness (goes16->goes19 East handover 2025, +# goes17->goes18 West handover 2022/23) comes from requesting the full pair +# and letting the available_since clamp skip pre-first-light slices free. +# --------------------------------------------------------------------------- +def test_goes_satellites_for_longitude_east() -> None: + # KNYC (-73.97): GOES-East era pair — goes16 (2017..2025 standby) + goes19. + assert _sources.goes_satellites_for_longitude(-73.97) == frozenset({"goes16", "goes19"}) + + +def test_goes_satellites_for_longitude_west() -> None: + # KSEA (-122.31): GOES-West era pair — goes17 (archive) + goes18. + assert _sources.goes_satellites_for_longitude(-122.31) == frozenset({"goes17", "goes18"}) + + +def test_goes_satellites_for_longitude_split_boundary_matches_router() -> None: + # Exactly at the split: _goes_route sends lon < SPLIT to goes18 (West), so + # the era-set helper must agree — lon == SPLIT is EAST on both. + split = _sources._GOES_EAST_WEST_SPLIT_LON_DEG + assert _sources.goes_satellites_for_longitude(split) == frozenset({"goes16", "goes19"}) + assert _sources.goes_satellites_for_longitude(split - 0.01) == frozenset({"goes17", "goes18"})