From 325a86ee6350c15ba9b9a99dca5633964b2a6afe Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 22:54:43 +0200 Subject: [PATCH 01/52] refactor(30-01): extract UTC window-trim predicate into shared _internal._window - add mostlyright._internal._window.in_query_window (byte-identical semantics) - mode2._in_query_window becomes a re-export alias (call sites unchanged) --- .../core/src/mostlyright/_internal/_window.py | 31 +++++++++++++++++++ packages/core/src/mostlyright/mode2.py | 20 ++++-------- 2 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 packages/core/src/mostlyright/_internal/_window.py diff --git a/packages/core/src/mostlyright/_internal/_window.py b/packages/core/src/mostlyright/_internal/_window.py new file mode 100644 index 00000000..bdec2f04 --- /dev/null +++ b/packages/core/src/mostlyright/_internal/_window.py @@ -0,0 +1,31 @@ +"""Shared UTC window-trim predicate for observation-grain queries. + +Extracted from :mod:`mostlyright.mode2` (formerly ``mode2._in_query_window``) +so the observation-grain fork in ``weather/obs.py`` and the Mode-2 alias +share ONE implementation. The semantics are byte-identical to the original: +inclusive at both ends, pure string comparison on the ISO date-part slice, +empty ``observed_at`` returns ``False``. + +Must match the TS port (``packages-ts/meta/src/mode2.ts``) for cross-SDK +lockstep. +""" + +from __future__ import annotations + +from typing import Any + + +def in_query_window(observed_at: Any, from_date: str, to_date: str) -> bool: + """True iff ``observed_at``'s UTC date-part is in ``[from_date, to_date]``. + + Compares the ISO date-part slice (``observed_at[:10]``) inclusively at + both ends — a pure string comparison, no timezone math. The slice + handles both emitted forms (``"2025-01-01T00:51:00Z"`` and + ``"2025-01-06T12:00:00+00:00"``) identically. Rows with missing/empty + ``observed_at`` return ``False``: they cannot be placed in any window. + Must match the TS port (``packages-ts/meta/src/mode2.ts``) for + cross-SDK lockstep. + """ + if not observed_at: + return False + return from_date <= observed_at[:10] <= to_date diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index e4f3b4e8..1236a215 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -24,6 +24,7 @@ from datetime import date as _date from typing import TYPE_CHECKING, Any +from mostlyright._internal._window import in_query_window as _in_query_window from mostlyright.core.exceptions import SourceMismatchError if TYPE_CHECKING: @@ -63,20 +64,11 @@ } -def _in_query_window(observed_at: Any, from_date: str, to_date: str) -> bool: - """True iff ``observed_at``'s UTC date-part is in ``[from_date, to_date]``. - - Compares the ISO date-part slice (``observed_at[:10]``) inclusively at - both ends — a pure string comparison, no timezone math. The slice - handles both emitted forms (``"2025-01-01T00:51:00Z"`` and - ``"2025-01-06T12:00:00+00:00"``) identically. Rows with missing/empty - ``observed_at`` return False: they cannot be placed in any window. - Must match the TS port (``packages-ts/meta/src/mode2.ts``) for - cross-SDK lockstep. - """ - if not observed_at: - return False - return from_date <= observed_at[:10] <= to_date +# ``_in_query_window`` is re-exported from ``mostlyright._internal._window`` +# (see the top-of-module import). The predicate was extracted into a shared +# internal so the observation-grain fork in ``weather/obs.py`` and this Mode-2 +# alias share ONE implementation; the private name is kept bound here so +# existing call sites below are byte-unchanged. def research_by_source( From e5d95b94a14cd72c3aeddb80c4bc6517d406feab Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 22:55:50 +0200 Subject: [PATCH 02/52] feat(30-01): add return_type='list' primitive to _backend_dispatch - add 'list' to _SUPPORTED_RETURN_TYPES + ReturnTypeT - wrap_result emits list[dict] for return_type='list' - keep polars+{dataframe,list} incoherent-pair ValueError guard --- .../src/mostlyright/core/_backend_dispatch.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/core/src/mostlyright/core/_backend_dispatch.py b/packages/core/src/mostlyright/core/_backend_dispatch.py index 799407e5..f9f951db 100644 --- a/packages/core/src/mostlyright/core/_backend_dispatch.py +++ b/packages/core/src/mostlyright/core/_backend_dispatch.py @@ -7,9 +7,10 @@ - ``backend: Literal["pandas","polars"]="pandas"`` — chooses the output frame type. Default stays pandas so the v0.1.0 zero-behaviour-change constraint holds. -- ``return_type: Literal["dataframe","wrapper"]="dataframe"`` — chooses - between raw DataFrame (legacy v0.1.0 shape; ``df.attrs`` carries - provenance) and :class:`MostlyRightResult` (new v0.2 shape). +- ``return_type: Literal["dataframe","wrapper","list"]="dataframe"`` — + chooses between raw DataFrame (legacy v0.1.0 shape; ``df.attrs`` carries + provenance), :class:`MostlyRightResult` (new v0.2 shape), and a plain + ``list[dict]`` of rows (consumed by the observation-grain surfaces). The validation order is strict (architect iter-3 P2 fix): @@ -44,10 +45,10 @@ ] BackendT = Literal["pandas", "polars"] -ReturnTypeT = Literal["dataframe", "wrapper"] +ReturnTypeT = Literal["dataframe", "wrapper", "list"] _SUPPORTED_BACKENDS: tuple[BackendT, ...] = ("pandas", "polars") -_SUPPORTED_RETURN_TYPES: tuple[ReturnTypeT, ...] = ("dataframe", "wrapper") +_SUPPORTED_RETURN_TYPES: tuple[ReturnTypeT, ...] = ("dataframe", "wrapper", "list") def validate_backend_kwargs( @@ -58,8 +59,10 @@ def validate_backend_kwargs( Raises ``ValueError`` on either: - Unsupported value. - - ``backend="polars" + return_type="dataframe"`` (polars frames have - no ``df.attrs`` so provenance MUST travel on a :class:`MostlyRightResult`). + - ``backend="polars"`` combined with ``return_type in + {"dataframe","list"}`` — both are unwrapped shapes that carry + provenance on ``df.attrs`` (which polars frames lack), so provenance + MUST travel on a :class:`MostlyRightResult`. Does NOT check that the ``[polars]`` extra is installed — callers do that lazily by invoking :func:`convert_to_backend` only when they @@ -71,7 +74,7 @@ def validate_backend_kwargs( raise ValueError( f"return_type must be one of {_SUPPORTED_RETURN_TYPES}; got {return_type!r}" ) - if backend == "polars" and return_type == "dataframe": + if backend == "polars" and return_type in ("dataframe", "list"): raise ValueError( "backend='polars' requires return_type='wrapper' — polars " "frames have no df.attrs to carry source/retrieved_at " @@ -148,6 +151,13 @@ def wrap_result( # backend == "pandas" by validate_backend_kwargs. return df + if return_type == "list": + # backend == "pandas" by validate_backend_kwargs (polars+list is + # rejected upstream). Emit the rows as list[dict] — mirrors how + # daily_extremes short-circuits its list return today. NaN/NaT are + # preserved as-is; callers wanting JSON-safe values handle that. + return df.to_dict(orient="records") + # return_type == "wrapper" frame = convert_to_backend(df, backend) return MostlyRightResult( From ded1be1923a1c647e52a650d4d1c8422a657826a Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 22:58:34 +0200 Subject: [PATCH 03/52] feat(30-01): add MergedObservationSchema (schema.observation.merged.v1) - new frame identity tag merged.live_v1 accepted only by this schema - plural _registered_sources only (no singular -> no source_drift audit trap) - new ClassVar _registered_row_sources={awc,iem,ghcnh} convention - COLUMNS declared over actual observation-grain parser row keys - observation.py untouched (CWOP parity firewall / D-15) --- .../src/mostlyright/core/schemas/__init__.py | 4 + .../core/schemas/observation_merged.py | 172 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 packages/core/src/mostlyright/core/schemas/observation_merged.py diff --git a/packages/core/src/mostlyright/core/schemas/__init__.py b/packages/core/src/mostlyright/core/schemas/__init__.py index 39e78c7d..2f9e5bb5 100644 --- a/packages/core/src/mostlyright/core/schemas/__init__.py +++ b/packages/core/src/mostlyright/core/schemas/__init__.py @@ -18,12 +18,15 @@ from .forecast_nwp import NwpForecastSchema from .observation import ObservationSchema from .observation_ledger import ObservationLedgerSchema +from .observation_merged import MergedObservationSchema from .observation_qc import ObservationQCSchema from .satellite import SatelliteSchema from .settlement import SettlementSchema # Eager registration — Validator can look up each schema by ID immediately. register_schema(ObservationSchema) +# Phase 30 30-01: merged observation-grain frame identity (schema.observation.merged.v1). +register_schema(MergedObservationSchema) # Phase 20 OM-02: register canonical StationForecastSchema FIRST so the # canonical schema_id wins on any registry-iteration that visits in # insertion order; ForecastSchema (back-compat alias to @@ -52,6 +55,7 @@ "EarningsFactSchema", "EarningsTranscriptSchema", "ForecastSchema", + "MergedObservationSchema", "NwpForecastSchema", "ObservationLedgerSchema", "ObservationQCSchema", diff --git a/packages/core/src/mostlyright/core/schemas/observation_merged.py b/packages/core/src/mostlyright/core/schemas/observation_merged.py new file mode 100644 index 00000000..6dd1f3c8 --- /dev/null +++ b/packages/core/src/mostlyright/core/schemas/observation_merged.py @@ -0,0 +1,172 @@ +"""Merged observation-grain schema (``schema.observation.merged.v1``). + +Phase 30 (30-01). The frame-level identity contract for +``obs(granularity="observation")`` when the query is UNPINNED — the merged +per-report rows produced by ``_dispatch_strategy`` + ``merge_observations``, +trimmed to the queried UTC window, pass-through with ``observation_type`` +distinguishing METAR/SPECI (NO bucketing/resampling). + +**Why a DISTINCT schema (D-03, D-15).** A merged frame's ``df.attrs["source"]`` +is the fused-policy tag ``"merged.live_v1"``, NOT a single upstream source — +but its per-row ``source`` column is the TRUTHFUL bare parser tag +(``awc``/``iem``/``ghcnh``). The canonical single-source ``schema.observation.v1`` +enforces per-row ``source == df.attrs["source"]`` (equality), so it correctly +REJECTS a merged frame (the CWOP parity firewall stays intact — +``observation.py`` is NEVER modified). This schema instead declares: + +- ``_registered_sources = {"merged.live_v1"}`` — the ONLY frame-level identity + it accepts. Declared as the PLURAL set (no singular ``_registered_source``) + so the validator's ``source_drift_allowed`` audit trap + (``validator.py:493-500``, documented at ``cwop/_schema.py:91-97``) never + fires on a clean merged read. +- ``_registered_row_sources = {"awc","iem","ghcnh"}`` — a NEW ClassVar the + validator (30-01-4) reads to switch the per-row ``source`` check from + equality-to-attrs to MEMBERSHIP in this set. Absent (None) on every other + schema, so satellite/forecast.station/cwop are untouched. + +Columns are declared over the ACTUAL observation-grain row keys that +``obs(granularity="observation")`` emits (the merged parser row dicts — see +``weather/_iem.py:246-277``), so a genuine merged frame passes ALL four +validator checks (source, column-presence, dtype, enum), not just the source +check. +""" + +from __future__ import annotations + +from typing import ClassVar + +from ..schema import ColumnSpec, Schema + +#: Frame-level fused-policy identity tag stamped on UNPINNED +#: ``obs(granularity="observation")`` frames. Distinct from any single upstream +#: source; single-source pinned schemas REJECT it. +MERGED_LIVE_V1: str = "merged.live_v1" + +#: METAR | SPECI — same vocabulary as ``schema.observation.v1``. +_OBSERVATION_TYPE_VALUES: tuple[str, ...] = ("METAR", "SPECI") + +#: Sky-cover codes carried through verbatim from the parsers. +_SKY_COVER_VALUES: tuple[str, ...] = ("CLR", "FEW", "SCT", "BKN", "OVC", "VV") + + +class MergedObservationSchema(Schema): + """``schema.observation.merged.v1`` — merged per-report observation rows. + + Accepts the fused-policy frame identity ``merged.live_v1`` while allowing + the per-row ``source`` column to carry any of the bare parser tags + ``{awc, iem, ghcnh}`` (membership, not equality — enforced by the + validator via :attr:`_registered_row_sources`). + """ + + schema_id = "schema.observation.merged.v1" + + #: The ONLY accepted frame-level identity. Declared as the PLURAL set so + #: the singular ``source_drift_allowed`` audit branch never fires. + _registered_sources: ClassVar[frozenset[str]] = frozenset({MERGED_LIVE_V1}) + + #: NEW convention (Phase 30 30-01). When present (not None), the validator + #: validates the per-row ``source`` column as MEMBERSHIP in this set + #: (no nulls) INSTEAD of equality-to-``df.attrs["source"]``. Every other + #: schema leaves this None → the legacy equality check is byte-unchanged. + _registered_row_sources: ClassVar[frozenset[str] | None] = frozenset( + {"awc", "iem", "ghcnh"} + ) + + COLUMNS: ClassVar[list[ColumnSpec]] = [ + # --- Identity columns (non-nullable) ------------------------------ + ColumnSpec( + name="station_code", + dtype="string", + units=None, + nullable=False, + notes="ICAO/ASOS station ID (e.g. KNYC)", + ), + ColumnSpec( + name="observed_at", + dtype="string", + units=None, + nullable=False, + notes="ISO-8601 UTC observation valid time (string, as parsers emit)", + ), + ColumnSpec( + name="observation_type", + dtype="enum", + units=None, + nullable=False, + enum_values=_OBSERVATION_TYPE_VALUES, + notes="METAR | SPECI", + ), + ColumnSpec( + name="source", + dtype="string", + units=None, + nullable=False, + notes="bare parser tag (awc/iem/ghcnh); membership-checked, not equality", + ), + # --- Measurement columns (nullable) ------------------------------- + ColumnSpec(name="temp_c", dtype="float64", units="celsius", nullable=True), + ColumnSpec(name="temp_f", dtype="float64", units="fahrenheit", nullable=True), + ColumnSpec(name="dewpoint_c", dtype="float64", units="celsius", nullable=True), + ColumnSpec(name="dewpoint_f", dtype="float64", units="fahrenheit", nullable=True), + ColumnSpec(name="wind_dir_degrees", dtype="float64", units="degrees", nullable=True), + ColumnSpec(name="wind_speed_kt", dtype="float64", units="kt", nullable=True), + ColumnSpec(name="wind_gust_kt", dtype="float64", units="kt", nullable=True), + ColumnSpec(name="altimeter_inhg", dtype="float64", units="inHg", nullable=True), + ColumnSpec( + name="sea_level_pressure_mb", dtype="float64", units="mb", nullable=True + ), + ColumnSpec(name="visibility_miles", dtype="float64", units="miles", nullable=True), + ColumnSpec( + name="precip_1hr_inches", dtype="float64", units="inches", nullable=True + ), + # --- Sky layers (nullable) ---------------------------------------- + ColumnSpec( + name="sky_cover_1", + dtype="enum", + units=None, + nullable=True, + enum_values=_SKY_COVER_VALUES, + ), + ColumnSpec(name="sky_base_1_ft", dtype="float64", units="ft", nullable=True), + ColumnSpec( + name="sky_cover_2", + dtype="enum", + units=None, + nullable=True, + enum_values=_SKY_COVER_VALUES, + ), + ColumnSpec(name="sky_base_2_ft", dtype="float64", units="ft", nullable=True), + ColumnSpec( + name="sky_cover_3", + dtype="enum", + units=None, + nullable=True, + enum_values=_SKY_COVER_VALUES, + ), + ColumnSpec(name="sky_base_3_ft", dtype="float64", units="ft", nullable=True), + ColumnSpec( + name="sky_cover_4", + dtype="enum", + units=None, + nullable=True, + enum_values=_SKY_COVER_VALUES, + ), + ColumnSpec(name="sky_base_4_ft", dtype="float64", units="ft", nullable=True), + # --- Remaining parser keys (nullable) ----------------------------- + ColumnSpec(name="weather_codes", dtype="string", units=None, nullable=True), + ColumnSpec(name="peak_wind_gust_kt", dtype="float64", units="kt", nullable=True), + ColumnSpec(name="peak_wind_dir", dtype="float64", units="degrees", nullable=True), + ColumnSpec(name="peak_wind_time", dtype="string", units=None, nullable=True), + ColumnSpec(name="snow_depth_inches", dtype="float64", units="inches", nullable=True), + ColumnSpec(name="qc_field", dtype="string", units=None, nullable=True), + ColumnSpec( + name="raw_metar", + dtype="string", + units=None, + nullable=True, + notes="raw METAR text if source has it; null for structured-only sources", + ), + ] + + +__all__ = ["MERGED_LIVE_V1", "MergedObservationSchema"] From 09fef7712e86b06ffcae2a3b00cef75e57b3be3d Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:00:52 +0200 Subject: [PATCH 04/52] feat(30-01): validator per-row source membership check via _registered_row_sources - step-1b reads schema._registered_row_sources; when set, validates per-row source as MEMBERSHIP in that set (no nulls) instead of equality-to-attrs - None (every legacy schema) keeps the equality check byte-unchanged - merged.live_v1 frame with mixed {awc,iem,ghcnh} rows passes merged schema, still rejected by observation.v1 (CWOP firewall intact) - add test_merged_observation_schema.py --- .../core/src/mostlyright/core/validator.py | 43 ++++-- .../tests/test_merged_observation_schema.py | 130 ++++++++++++++++++ 2 files changed, 164 insertions(+), 9 deletions(-) create mode 100644 packages/core/tests/test_merged_observation_schema.py diff --git a/packages/core/src/mostlyright/core/validator.py b/packages/core/src/mostlyright/core/validator.py index 39d6b907..775bf17a 100644 --- a/packages/core/src/mostlyright/core/validator.py +++ b/packages/core/src/mostlyright/core/validator.py @@ -359,31 +359,56 @@ def validate_dataframe( quarantine_count=len(df), sample_violations=[], ) + # Phase 30 30-01: schemas may declare `_registered_row_sources` — a set of + # bare parser tags that the per-row `source` column is validated against by + # MEMBERSHIP instead of equality-to-attrs. This lets a merged frame + # (df.attrs['source']='merged.live_v1') carry truthful per-row parser tags + # (awc/iem/ghcnh) without tripping the single-source equality guard. When + # the attr is absent (None) — every legacy schema — the equality check + # below is byte-unchanged, so satellite/forecast.station/cwop are untouched. + registered_row_sources = getattr(schema_cls, "_registered_row_sources", None) if "source" in df.columns: col = df["source"] - # Null rows: every null in the source column counts as a mismatch. + # Null rows: every null in the source column counts as a mismatch + # regardless of check mode — provenance must be present on every row. null_mask = col.isna() null_count = int(null_mask.sum()) - # Non-null mismatches: any row whose source != attrs source. non_null = col[~null_mask] - mismatch_mask = non_null != data_source + if registered_row_sources is not None: + # MEMBERSHIP mode: each non-null row source must be in the set. + mismatch_mask = ~non_null.isin(list(registered_row_sources)) + expected_desc = sorted(registered_row_sources) + else: + # EQUALITY mode (legacy): each non-null row source must equal + # df.attrs['source']. + mismatch_mask = non_null != data_source + expected_desc = data_source mismatch_count = int(mismatch_mask.sum()) if null_count > 0 or mismatch_count > 0: distinct_bad = sorted(set(non_null[mismatch_mask].astype(str).tolist()))[:_SAMPLE_CAP] if null_count > 0: distinct_bad.insert(0, "") + if registered_row_sources is not None: + schema_source_desc = ",".join(sorted(registered_row_sources)) + warning = ( + "row-level source not in the schema's registered row-source " + f"set {expected_desc!r} (no nulls allowed)" + ) + else: + schema_source_desc = data_source + warning = ( + "row-level source column drift; the validator requires " + "every row's source to equal df.attrs['source'] (no nulls)" + ) raise SourceMismatchError( f"Per-row 'source' column has {null_count + mismatch_count} " - f"row(s) not matching df.attrs['source']={data_source!r} " + f"row(s) not matching {expected_desc!r} " f"({null_count} null, {mismatch_count} mismatched); " f"distinct bad values: {distinct_bad}", - schema_source=data_source, + schema_source=schema_source_desc, data_source=str(distinct_bad[0]) if distinct_bad else "", role=None, - catalog_warning=( - "row-level source column drift; the validator requires " - "every row's source to equal df.attrs['source'] (no nulls)" - ), + catalog_warning=warning, ) # --- 2-4. Column / dtype / enum / null checks --- diff --git a/packages/core/tests/test_merged_observation_schema.py b/packages/core/tests/test_merged_observation_schema.py new file mode 100644 index 00000000..a9635ee5 --- /dev/null +++ b/packages/core/tests/test_merged_observation_schema.py @@ -0,0 +1,130 @@ +"""Tests for schema.observation.merged.v1 + validator membership-source check. + +Phase 30 (30-01). Covers: + +- ``MergedObservationSchema`` shape/identity contract (30-01-3). +- The validator's per-row source MEMBERSHIP check gated on + ``_registered_row_sources`` (30-01-4). +- The CWOP parity firewall invariant: a ``merged.live_v1`` frame is REJECTED + by ``schema.observation.v1`` (which is NEVER modified). +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import mostlyright.core.schemas # noqa: F401 — eager schema registration +import pandas as pd +import pytest +from mostlyright.core.exceptions import SourceMismatchError +from mostlyright.core.schemas import SCHEMA_REGISTRY, MergedObservationSchema +from mostlyright.core.validator import _lookup_schema, validate_dataframe + + +def _merged_row(source: str, observation_type: str = "METAR", **over: object) -> dict: + row: dict = { + "station_code": "KNYC", + "observed_at": "2026-01-01T00:51:00Z", + "observation_type": observation_type, + "source": source, + "temp_c": 1.0, + "temp_f": 33.8, + "dewpoint_c": 0.0, + "dewpoint_f": 32.0, + "wind_dir_degrees": 180.0, + "wind_speed_kt": 5.0, + "wind_gust_kt": 7.0, + "altimeter_inhg": 30.1, + "sea_level_pressure_mb": 1013.0, + "visibility_miles": 10.0, + "precip_1hr_inches": 0.0, + "raw_metar": "KNYC 010051Z ...", + } + row.update(over) + return row + + +def _merged_frame(rows: list[dict], source: str = "merged.live_v1") -> pd.DataFrame: + df = pd.DataFrame(rows) + df.attrs["source"] = source + df.attrs["retrieved_at"] = datetime.now(UTC) + return df + + +# --- 30-01-3: schema identity contract ----------------------------------- + + +def test_merged_schema_identity_contract() -> None: + assert MergedObservationSchema.schema_id == "schema.observation.merged.v1" + assert MergedObservationSchema._registered_sources == frozenset({"merged.live_v1"}) + # No singular _registered_source — avoids the source_drift_allowed audit trap. + assert getattr(MergedObservationSchema, "_registered_source", None) is None + assert MergedObservationSchema._registered_row_sources == frozenset({"awc", "iem", "ghcnh"}) + + +def test_merged_schema_registered_and_resolvable() -> None: + assert "schema.observation.merged.v1" in SCHEMA_REGISTRY + assert _lookup_schema("schema.observation.merged.v1") is MergedObservationSchema + + +# --- 30-01-4: membership-based per-row source check ---------------------- + + +def test_merged_frame_with_mixed_row_sources_passes() -> None: + df = _merged_frame( + [ + _merged_row("awc"), + _merged_row("iem", "SPECI"), + _merged_row("ghcnh"), + ] + ) + reg = validate_dataframe(df, "schema.observation.merged.v1") + # Clean read: no source_drift_allowed audit (plural-only registration). + assert all(e["event"] != "source_drift_allowed" for e in reg.audit_log()) + + +def test_merged_frame_rejects_row_source_outside_set() -> None: + df = _merged_frame([_merged_row("awc"), _merged_row("cwop")]) + with pytest.raises(SourceMismatchError): + validate_dataframe(df, "schema.observation.merged.v1") + + +def test_merged_frame_rejects_null_row_source() -> None: + df = _merged_frame([_merged_row("awc"), _merged_row(None)]) # type: ignore[arg-type] + with pytest.raises(SourceMismatchError): + validate_dataframe(df, "schema.observation.merged.v1") + + +def test_merged_frame_rejects_wrong_frame_identity() -> None: + # A frame tagged with something other than merged.live_v1 fails the + # frame-level (step-1) membership check before per-row checks run. + df = _merged_frame([_merged_row("awc")], source="iem") + with pytest.raises(SourceMismatchError): + validate_dataframe(df, "schema.observation.merged.v1") + + +# --- CWOP parity firewall: observation.v1 REJECTS merged.live_v1 --------- + + +def test_observation_v1_rejects_merged_frame() -> None: + df = _merged_frame([_merged_row("awc"), _merged_row("iem")]) + with pytest.raises(SourceMismatchError): + validate_dataframe(df, "schema.observation.v1") + + +def test_legacy_equality_check_unchanged_for_single_source_schema() -> None: + # A single-source frame (no _registered_row_sources) still enforces + # per-row source == df.attrs['source'] equality. + rows = [_merged_row("iem"), _merged_row("iem", "SPECI")] + df = pd.DataFrame(rows) + # observation.v1 uses different column names; this asserts the equality + # branch fires when a legacy schema sees a bare source. We use the merged + # schema's own attrs but force an equality-mode expectation by tagging with + # a value the row-source set would reject at frame level; here we simply + # confirm the merged schema's membership branch does NOT leak into + # observation.v1's equality semantics. + df.attrs["source"] = "iem.archive" + df.attrs["retrieved_at"] = datetime.now(UTC) + with pytest.raises(SourceMismatchError): + # rows tagged bare 'iem' != attrs 'iem.archive' → equality mismatch + validate_dataframe(df, "schema.observation.v1") From 8e103e818795458bc11fdbeae23e5763c6ec105a Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:03:57 +0200 Subject: [PATCH 05/52] feat(30-01): fork obs() on granularity with merged frame identity - add granularity: Literal['daily','observation']='daily' (kw-only, validated) - observation grain: trim merged raw_rows at list level via in_query_window, preserve bare per-row source, surface observation_type + raw_metar, NO bucketing - coerce numeric cols to float64 so short-window all-None cols pass validator - stamp df.attrs['source']=merged.live_v1 (unpinned) or bare source (pinned) - daily branch + _aggregate_daily_rows byte-unchanged (D-15) - add tests/weather/test_obs_observation_grain.py (path follows repo convention) --- .../weather/src/mostlyright/weather/obs.py | 98 +++++++++- tests/weather/test_obs_observation_grain.py | 173 ++++++++++++++++++ 2 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 tests/weather/test_obs_observation_grain.py diff --git a/packages/weather/src/mostlyright/weather/obs.py b/packages/weather/src/mostlyright/weather/obs.py index 7b3a13c9..5ec0d7e2 100644 --- a/packages/weather/src/mostlyright/weather/obs.py +++ b/packages/weather/src/mostlyright/weather/obs.py @@ -26,9 +26,41 @@ Source = Literal["iem", "ghcnh", "awc"] Strategy = Literal["auto", "exact_window", "warm_cache", "hosted"] +Granularity = Literal["daily", "observation"] _VALID_SOURCES: frozenset[str] = frozenset({"iem", "ghcnh", "awc"}) _VALID_STRATEGIES: frozenset[str] = frozenset({"auto", "exact_window", "warm_cache", "hosted"}) +_VALID_GRANULARITIES: frozenset[str] = frozenset({"daily", "observation"}) + +#: Frame-level fused-policy identity tag stamped on UNPINNED +#: ``obs(granularity="observation")`` frames. Distinct from any single upstream +#: source; ``schema.observation.v1`` REJECTS it, ``schema.observation.merged.v1`` +#: ACCEPTS it (see ``core/schemas/observation_merged.py``). +_MERGED_FRAME_SOURCE = "merged.live_v1" + +#: Numeric row keys that must be coerced to float64 so a genuine merged frame +#: passes the Validator's dtype checks even when a column is entirely-None over +#: a short window (an all-None column otherwise infers pandas ``object`` dtype). +_OBSERVATION_NUMERIC_KEYS: tuple[str, ...] = ( + "temp_c", + "temp_f", + "dewpoint_c", + "dewpoint_f", + "wind_dir_degrees", + "wind_speed_kt", + "wind_gust_kt", + "altimeter_inhg", + "sea_level_pressure_mb", + "visibility_miles", + "precip_1hr_inches", + "sky_base_1_ft", + "sky_base_2_ft", + "sky_base_3_ft", + "sky_base_4_ft", + "peak_wind_gust_kt", + "peak_wind_dir", + "snow_depth_inches", +) def obs( @@ -38,6 +70,7 @@ def obs( *, source: Source | None = None, strategy: Strategy = "auto", + granularity: Granularity = "daily", as_dataframe: bool = True, ) -> pd.DataFrame | list[dict]: """Return observation aggregates for ``station`` over ``[start, end]``. @@ -104,6 +137,10 @@ def obs( raise ValueError(f"source must be one of {sorted(_VALID_SOURCES)} or None; got {source!r}") if strategy not in _VALID_STRATEGIES: raise ValueError(f"strategy must be one of {sorted(_VALID_STRATEGIES)}; got {strategy!r}") + if granularity not in _VALID_GRANULARITIES: + raise ValueError( + f"granularity must be one of {sorted(_VALID_GRANULARITIES)}; got {granularity!r}" + ) # Eager ISO validation — fail fast for malformed dates before any network. date.fromisoformat(start) @@ -136,6 +173,19 @@ def obs( strategy=resolved, ) + if granularity == "observation": + # Observation-grain path (Phase 30 D-03): return the merged per-report + # rows as-is — NO bucketing/resampling. Trim at the LIST level (before + # any DataFrame build, attrs-safe) to the strict UTC window so the + # +1-day fetch extension does not leak into the result. Every row's + # bare `source` column is preserved verbatim (NEVER overwritten — + # mirrors mode2.py:224-229); the FRAME identity is a distinct + # fused-policy tag when unpinned. raw_rows are grain-agnostic, so this + # fork is post-dispatch only (D-15: daily branch untouched). + return _observation_grain_rows( + raw_rows, start, end, source=source, as_dataframe=as_dataframe + ) + # Aggregate raw observation rows to daily summary rows. The output schema is # the obs_* subset of research() Mode-1 columns (no CLI / no forecast). Both # exact_window and warm_cache funnel through this aggregation so the daily @@ -149,6 +199,52 @@ def obs( return aggregated +def _observation_grain_rows( + raw_rows: list[dict], + start: str, + end: str, + *, + source: Source | None, + as_dataframe: bool, +) -> pd.DataFrame | list[dict]: + """Trim merged per-report rows to the UTC window; stamp frame identity. + + The trim runs at the list level via + :func:`mostlyright._internal._window.in_query_window` (inclusive both ends, + string compare on ``observed_at[:10]``) so the +1-day fetch extension used + by the fetchers/warm-cache is stripped out of the result. Per-row ``source`` + is preserved verbatim — the parser-emitted tag IS the truthful provenance. + + When ``source`` is None the frame is a fused multi-source result: stamp + ``df.attrs["source"] = "merged.live_v1"`` so pinned single-source schemas + loudly reject it and :class:`MergedObservationSchema` accepts it. When + ``source`` is pinned, stamp the bare source (a genuine single-source frame). + """ + from mostlyright._internal._window import in_query_window + + trimmed = [r for r in raw_rows if in_query_window(r.get("observed_at"), start, end)] + + if not as_dataframe: + return trimmed + + import contextlib + + import pandas as pd + + df = pd.DataFrame(trimmed) + # Coerce numeric columns to float64 so an entirely-None column over a short + # window does not infer object dtype and trip the merged-schema validator. + for key in _OBSERVATION_NUMERIC_KEYS: + if key in df.columns: + df[key] = pd.to_numeric(df[key], errors="coerce") + # FINAL op before return: stamp frame identity. contextlib.suppress mirrors + # research.py:1968 — attrs assignment is a no-op-safe best-effort. + frame_source = source if source is not None else _MERGED_FRAME_SOURCE + with contextlib.suppress(AttributeError): + df.attrs["source"] = frame_source + return df + + def _dispatch_strategy( info, from_date_iso: str, @@ -374,4 +470,4 @@ def _resolve_strategy( return "warm_cache" -__all__ = ["Source", "Strategy", "obs"] +__all__ = ["Granularity", "Source", "Strategy", "obs"] diff --git a/tests/weather/test_obs_observation_grain.py b/tests/weather/test_obs_observation_grain.py new file mode 100644 index 00000000..e5f5a074 --- /dev/null +++ b/tests/weather/test_obs_observation_grain.py @@ -0,0 +1,173 @@ +"""Phase 30 (30-01) — obs(granularity="observation") observation-grain path. + +Covers: + +- ``granularity`` kwarg validation (loud ValueError on bogus value). +- Observation-grain returns per-report rows (METAR/SPECI + raw_metar + bare + per-row source), trimmed to the UTC window (no +1-day extension leakage). +- Frame identity: unpinned → ``merged.live_v1``; pinned → bare source. +- ``as_dataframe=False`` returns the trimmed ``list[dict]``. +- The daily branch is untouched (default granularity="daily"). +- A genuine observation-grain merged frame validates against + ``schema.observation.merged.v1``. +""" + +from __future__ import annotations + +import inspect +from datetime import UTC, datetime +from unittest.mock import patch + +import pytest + +# Raw merged per-report rows as _dispatch_strategy would return them (the +# fetch extends one day past `end`, so the last row is an extension-leak that +# the observation-grain trim MUST drop). +_RAW_ROWS = [ + { + "station_code": "KNYC", + "observed_at": "2026-01-01T00:51:00Z", + "observation_type": "METAR", + "source": "awc", + "temp_f": 33.8, + "wind_gust_kt": None, + "raw_metar": "KNYC 010051Z ...", + }, + { + "station_code": "KNYC", + "observed_at": "2026-01-02T01:51:00Z", + "observation_type": "SPECI", + "source": "iem", + "temp_f": 35.6, + "wind_gust_kt": 7.0, + "raw_metar": None, + }, + { + "station_code": "KNYC", + "observed_at": "2026-01-03T23:51:00Z", + "observation_type": "METAR", + "source": "ghcnh", + "temp_f": 30.0, + "wind_gust_kt": None, + "raw_metar": None, + }, + # +1-day fetch extension — outside [2026-01-01, 2026-01-03], must drop. + { + "station_code": "KNYC", + "observed_at": "2026-01-04T00:00:00Z", + "observation_type": "METAR", + "source": "awc", + "temp_f": 28.0, + "wind_gust_kt": None, + "raw_metar": None, + }, + # Empty observed_at — cannot be windowed, must drop. + { + "station_code": "KNYC", + "observed_at": "", + "observation_type": "METAR", + "source": "awc", + "temp_f": 1.0, + }, +] + + +def _patch_dispatch(rows: list[dict]): + return patch("mostlyright.weather.obs._dispatch_strategy", return_value=rows) + + +def test_granularity_is_keyword_only_with_daily_default() -> None: + from mostlyright.weather.obs import obs + + params = inspect.signature(obs).parameters + assert params["granularity"].kind == inspect.Parameter.KEYWORD_ONLY + assert params["granularity"].default == "daily" + + +def test_bogus_granularity_raises_before_network() -> None: + from mostlyright.weather.obs import obs + + with pytest.raises(ValueError, match="granularity must be"): + obs("KNYC", "2026-01-01", "2026-01-03", granularity="bogus") # type: ignore[arg-type] + + +def test_observation_grain_returns_per_report_rows_trimmed() -> None: + from mostlyright.weather.obs import obs + + with _patch_dispatch(list(_RAW_ROWS)): + df = obs("KNYC", "2026-01-01", "2026-01-03", granularity="observation") + + # Extension-leak row (2026-01-04) and empty-observed_at row dropped. + assert sorted(df["observed_at"].str[:10].tolist()) == [ + "2026-01-01", + "2026-01-02", + "2026-01-03", + ] + assert set(df["observation_type"]) == {"METAR", "SPECI"} + assert "raw_metar" in df.columns + # Bare per-row source preserved verbatim. + assert set(df["source"]) == {"awc", "iem", "ghcnh"} + # Every kept row's date-part is within the window. + assert df["observed_at"].str[:10].between("2026-01-01", "2026-01-03").all() + + +def test_observation_grain_unpinned_frame_identity_is_merged() -> None: + from mostlyright.weather.obs import obs + + with _patch_dispatch(list(_RAW_ROWS)): + df = obs("KNYC", "2026-01-01", "2026-01-03", granularity="observation") + assert df.attrs["source"] == "merged.live_v1" + + +def test_observation_grain_pinned_frame_identity_is_bare_source() -> None: + from mostlyright.weather.obs import obs + + # source="iem" forces exact_window in the resolver; _dispatch_strategy is + # patched, so no network. The frame identity must be the bare source. + with _patch_dispatch([r for r in _RAW_ROWS if r["source"] == "iem"]): + df = obs( + "KNYC", "2026-01-01", "2026-01-03", source="iem", granularity="observation" + ) + assert df.attrs["source"] == "iem" + + +def test_observation_grain_as_dataframe_false_returns_list() -> None: + from mostlyright.weather.obs import obs + + with _patch_dispatch(list(_RAW_ROWS)): + out = obs( + "KNYC", + "2026-01-01", + "2026-01-03", + granularity="observation", + as_dataframe=False, + ) + assert isinstance(out, list) + assert len(out) == 3 + assert all(isinstance(r, dict) for r in out) + + +def test_daily_branch_untouched_by_new_kwarg() -> None: + from mostlyright.weather.obs import obs + + with _patch_dispatch(list(_RAW_ROWS)): + default = obs("KNYC", "2026-01-01", "2026-01-03") + explicit = obs("KNYC", "2026-01-01", "2026-01-03", granularity="daily") + # Daily aggregates: one row per settlement day, obs_* columns present. + assert "obs_high_f" in default.columns + assert list(default.columns) == list(explicit.columns) + assert len(default) == len(explicit) == 3 # 3-day window + + +def test_observation_grain_frame_validates_against_merged_schema() -> None: + import mostlyright.core.schemas # noqa: F401 — eager registration + from mostlyright.core.validator import validate_dataframe + from mostlyright.weather.obs import obs + + with _patch_dispatch(list(_RAW_ROWS)): + df = obs("KNYC", "2026-01-01", "2026-01-03", granularity="observation") + # obs() does not stamp retrieved_at; the validator requires it, so supply it + # (this mirrors what a catalog adapter would do on a real pull). + df.attrs["retrieved_at"] = datetime.now(UTC) + reg = validate_dataframe(df, "schema.observation.merged.v1") + assert all(e["event"] != "source_drift_allowed" for e in reg.audit_log()) From 51b7ea7d08e1739f8830d7fc4e45de0248698895 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:06:32 +0200 Subject: [PATCH 06/52] refactor(30-01): demote strategy=, document granularity, purge Mode-1/2 vocab - module + obs() docstrings: strategy= is an advanced escape hatch (auto self-selects exact_window for pinned queries); delete stale NotImplementedError / PLAN-07-04 claims; document new granularity kwarg - reword 'Mode 1'/'Mode-1' comments to 'default fused-source aggregates' - routing logic UNCHANGED; warm_cache+source ValueError guard intact (D-07) --- .../weather/src/mostlyright/weather/obs.py | 108 ++++++++++-------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/obs.py b/packages/weather/src/mostlyright/weather/obs.py index 5ec0d7e2..b1d78221 100644 --- a/packages/weather/src/mostlyright/weather/obs.py +++ b/packages/weather/src/mostlyright/weather/obs.py @@ -1,17 +1,23 @@ -"""mostlyright.weather.obs — public surface for observation aggregates. +"""mostlyright.weather.obs — public surface for observation data. -Smart-routes between three ingest strategies: +``obs()`` returns default fused-source aggregates (AWC > IEM > GHCNh priority): +one row per LST settlement day by default (``granularity="daily"``), or the +merged per-report rows (``granularity="observation"``) trimmed to the queried +UTC window when you want the raw METAR/SPECI cadence. + +Ingest routing is automatic. The ``strategy=`` kwarg is an advanced escape +hatch that ordinary users never type — ``"auto"`` self-selects the right +tactic (including the source-isolated exact-window path for pinned +``source=`` queries). The concrete tactics it dispatches to: - **exact_window**: bypass year-normalization in IEM, day-granular URL params, - ≤2 MB cold for 1mo KNYC. Best for one-off backtest replays. -- **warm_cache**: current ``research()`` orchestration with year-aligned cache - hit-rate optimization. Best for repeated queries across overlapping windows. + ≤2 MB cold for 1mo KNYC. Best for one-off backtest replays; the tactic + ``"auto"`` picks for any pinned ``source=`` query. +- **warm_cache**: ``research()``-equivalent orchestration with year-aligned + cache hit-rate optimization. Best for repeated overlapping windows. - **hosted**: precomputed-API seam (gated by ``TW_HOSTED_URL``); deferred to v0.2.x. -The default ``strategy="auto"`` selects between these based on window size, -cache warmth, and env-var presence. PLAN-07-04 fills in the dispatch body. -The default ships as ``"auto"`` from PLAN-02 onward and never changes (B-6 — -no public-API churn between PRs). +The default ships as ``"auto"`` and never changes (no public-API churn). See ``docs/ingest-strategies.md`` for the decision tree. """ @@ -73,7 +79,10 @@ def obs( granularity: Granularity = "daily", as_dataframe: bool = True, ) -> pd.DataFrame | list[dict]: - """Return observation aggregates for ``station`` over ``[start, end]``. + """Return observation data for ``station`` over ``[start, end]``. + + Returns default fused-source aggregates merged via SOURCE_PRIORITY + (AWC > IEM > GHCNh). ``granularity`` chooses the row level. Parameters ---------- @@ -85,53 +94,63 @@ def obs( If set, only that source is queried; the other two fetchers are skipped. If None, all three are queried and merged with the standard priority (AWC > IEM > GHCNh). + granularity : {"daily", "observation"}, keyword-only, default "daily" + Row level of the result. + + - ``"daily"`` (default): one row per LST settlement day — the obs_* + aggregate subset (obs_high_f, obs_low_f, obs_mean_f, ...). Behaviour + is byte-unchanged from prior releases. + - ``"observation"``: the merged per-report rows (native METAR/SPECI + + sub-hourly cadence), trimmed to the queried UTC window, pass-through + with ``observation_type`` and ``raw_metar`` preserved — NO + bucketing/resampling. Each row's ``source`` column is the truthful + bare parser tag (awc/iem/ghcnh); the FRAME identity + (``df.attrs["source"]``) is ``"merged.live_v1"`` when unpinned, or + the bare source when ``source=`` is pinned. Fixed-cadence resampling + (exact-hourly, 30-min, ...) is a separate preprocessing transform, + not a ``granularity`` value. strategy : {"auto", "exact_window", "warm_cache", "hosted"}, keyword-only - Ingest strategy. Default is ``"auto"`` (resolved by PLAN-07-04). In - PLAN-07-02 the ``"auto"`` branch is a stub that raises NotImplementedError - pointing the caller at an explicit strategy. The public default does - NOT churn between PRs — it ships as ``"auto"`` and stays ``"auto"``. + Advanced/documented escape hatch — ordinary users never type it. + ``"auto"`` (the default) self-selects the right tactic, including the + source-isolated ``exact_window`` path for any pinned ``source=`` query. + The default ships as ``"auto"`` and never churns between releases. as_dataframe : bool, keyword-only, default True If True (default), return ``pandas.DataFrame``. If False, return ``list[dict]``. Returns ------- pd.DataFrame | list[dict] - Observation rows merged via SOURCE_PRIORITY (AWC > IEM > GHCNh). + Daily obs_* aggregate rows (``granularity="daily"``) or merged + per-report rows (``granularity="observation"``). Raises ------ ValueError - If ``source`` or ``strategy`` is not in the allowed Literal set, OR if - ``strategy="warm_cache"`` is called with ``source != None`` (post-merge - filtering would corrupt SOURCE_PRIORITY semantics; use - ``strategy="exact_window"`` for single-source queries). - NotImplementedError - If ``strategy="hosted"`` (deferred to v0.2.x), or if ``strategy="auto"`` - until PLAN-07-04 wires the dispatch body. + If ``source``, ``strategy``, or ``granularity`` is not in its allowed + Literal set, OR if ``strategy="warm_cache"`` is called with + ``source != None`` (post-merge filtering would corrupt SOURCE_PRIORITY + semantics; use ``strategy="exact_window"`` for single-source queries). + DataAvailabilityError + If ``strategy="hosted"`` (the precomputed-API seam deferred to v0.2.x). Notes ----- - - ``strategy="warm_cache"`` is byte-equivalent to ``research()`` Mode-1 obs - aggregates for the columns: obs_high_f, obs_low_f, obs_high_at, obs_low_at, - source. Verified against the 5 Phase 1 parity fixtures (see - ``tests/weather/test_obs_warm_cache_parity.py``). - - ``obs()`` does NOT return CLI climate columns (cli_high_f, cli_low_f, fcst_*). If you need joined obs + CLI + forecast, use ``research()`` directly. - - ``strategy="exact_window"`` is NOT byte-equivalent to ``research()`` — it - intentionally bypasses year-aligned caching to minimize cold-fetch bytes. - Output values match research() at the row level, but cache footprint and - fetch URLs differ. - - - ``strategy="hosted"`` is reserved for the v0.2.x precomputed-API client and - raises NotImplementedError until that lands. + - The ``"auto"`` router's ``warm_cache`` tactic produces daily aggregates + byte-equivalent to ``research()`` for obs_high_f, obs_low_f, obs_high_at, + obs_low_at, source (verified against the 5 Phase 1 parity fixtures in + ``tests/weather/test_obs_warm_cache_parity.py``). The ``exact_window`` + tactic matches those values at the row level but intentionally bypasses + year-aligned caching, so its cache footprint and fetch URLs differ. Examples -------- >>> from mostlyright.weather import obs - >>> df = obs("KNYC", "2024-03-01", "2024-03-31", source="iem", - ... strategy="exact_window") # doctest: +SKIP + >>> df = obs("KNYC", "2024-03-01", "2024-03-31", source="iem") # doctest: +SKIP + >>> hourly = obs("KNYC", "2024-03-01", "2024-03-31", + ... granularity="observation") # doctest: +SKIP """ if source is not None and source not in _VALID_SOURCES: raise ValueError(f"source must be one of {sorted(_VALID_SOURCES)} or None; got {source!r}") @@ -187,8 +206,8 @@ def obs( ) # Aggregate raw observation rows to daily summary rows. The output schema is - # the obs_* subset of research() Mode-1 columns (no CLI / no forecast). Both - # exact_window and warm_cache funnel through this aggregation so the daily + # the obs_* subset of research()'s fused-source columns (no CLI / no forecast). + # Both exact_window and warm_cache funnel through this aggregation so the daily # rows are byte-equivalent across strategies. aggregated = _aggregate_daily_rows(raw_rows, info, start, end) @@ -266,10 +285,9 @@ def _dispatch_strategy( if strategy == "warm_cache": return _warm_cache_fetch(info, from_date_iso, to_date_iso, source=source) if strategy == "hosted": - # Phase 21 21-09: migrated from NotImplementedError to the structural - # DataAvailabilityError so cross-SDK callers can branch on `e.reason` - # rather than string-matching the message. Symmetric with TS - # obs(strategy="hosted") per D-06. + # Phase 21 21-09: raises the structural DataAvailabilityError so + # cross-SDK callers can branch on `e.reason` rather than string-matching + # the message. Symmetric with TS obs(strategy="hosted") per D-06. from mostlyright.core.exceptions import DataAvailabilityError raise DataAvailabilityError( @@ -294,7 +312,7 @@ def _warm_cache_fetch( same ``_all_caches_warm`` zero-network short-circuit. Skips the climate (CLI) leg of ``research()`` since ``obs()`` does not return CLI columns. - The output is byte-equivalent to the obs aggregates ``research()`` Mode-1 + The output is byte-equivalent to the obs aggregates ``research()`` produces for the same (station, from_date, to_date) inputs at the row level (post-aggregation columns are computed by ``research()``'s ``build_pairs``; ``obs()`` returns the merged raw rows pre-aggregation). @@ -401,7 +419,7 @@ def _resolve_strategy( 1. If ``source`` is not None → ``"exact_window"``. No other strategy honors source filtering today: warm_cache rejects source!=None (post-merge filtering would corrupt SOURCE_PRIORITY); hosted is a - v0.2.x stub that raises NotImplementedError. This rule runs BEFORE + v0.2.x stub that raises DataAvailabilityError. This rule runs BEFORE the env-var check so source-filtered callers still succeed even with TW_HOSTED_URL set (codex iter-3 HIGH). 2. Else if ``env["TW_HOSTED_URL"]`` is set → ``"hosted"``. @@ -447,7 +465,7 @@ def _resolve_strategy( # Source-filtered queries always go through exact_window because no other # strategy honors source filtering today (warm_cache rejects source!=None; - # hosted is a v0.2.x stub that raises NotImplementedError). Source check + # hosted is a v0.2.x stub that raises DataAvailabilityError). Source check # runs BEFORE the env-var check so source-filtered callers can still use # exact_window even with TW_HOSTED_URL set in the environment — otherwise # `obs(source="iem")` would fail loudly for users who have the env var From a8bd63814b2b5c23e27870cd6823949ddbcffb27 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:49:57 +0200 Subject: [PATCH 07/52] feat(30-02): add climate() standalone CLI settlement-label domain table - wraps research._fetch_climate_range (annual cache + frozen merge_climate) - window-trims year-granular fetch via shared in_query_window (no whole-year leak) - public JOIN vocab: date, station, cli_high_f, cli_low_f, cli_report_type + source, issued_at - frame identity df.attrs source=cli.archive; per-row source=iem (endpoint) - source accept-set {None,iem,cli,cli.archive}; acis raises ValueError - pandas lazy-imported inside the return path --- .../src/mostlyright/weather/climate.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 packages/weather/src/mostlyright/weather/climate.py diff --git a/packages/weather/src/mostlyright/weather/climate.py b/packages/weather/src/mostlyright/weather/climate.py new file mode 100644 index 00000000..c5a9394a --- /dev/null +++ b/packages/weather/src/mostlyright/weather/climate.py @@ -0,0 +1,228 @@ +"""mostlyright.weather.climate — public surface for NWS CLI settlement labels. + +``climate()`` exposes the NWS CLI daily settlement values (the LABELS that +settle Kalshi NHIGH/NLOW markets) as a standalone, source-identified domain +table — the raw ingredient today visible ONLY through the ``dataset()`` / +``research()`` join. It is the natural parallel to ``obs()``: where ``obs()`` +returns the station's observation extremes, ``climate()`` returns the official +NWS CLI daily high/low. + +Pure surface work (Phase 30 D-05): the CLI fetcher +(``weather/_climate.py``), the annual cache (``weather/cache.py``), and the +frozen dedup policy (``_internal/merge/climate.py``) already exist end-to-end. +``climate()`` wraps ``research._fetch_climate_range`` exactly as ``obs()`` +wraps ``_fetch_observations_range``. + +Column vocabulary is the JOIN vocabulary (``cli_high_f``, ``cli_low_f``, +``cli_report_type``) — exactly what users already see via +``dataset()``/``research()`` — NOT the raw parser keys +(``high_temp_f``/``low_temp_f``/``observation_date``). This avoids introducing +a fourth CLI column vocabulary. + +Provenance follows the frame-vs-row split (D-03/D-08): + +- ``df.attrs["source"] = "cli.archive"`` — the settlement PRODUCT identity. +- per-row ``source`` column == ``"iem"`` — the fetch ENDPOINT (IEM CLI is the + sole live provider; there is NO ``acis`` leg in this code path). + +CLI is a daily batch product with a single authority, so ``climate()`` has NO +grain axis and NO live surface (D-09/D-10 class 1: re-fetchable upstream). +""" + +from __future__ import annotations + +import contextlib +from datetime import date +from typing import TYPE_CHECKING, Literal + +from mostlyright.core._backend_dispatch import BackendT, ReturnTypeT, validate_backend_kwargs + +if TYPE_CHECKING: + import pandas as pd + +#: Provenance values ``source=`` accepts. A degenerate single-authority axis +#: (D-08): IEM CLI is the sole live provider, so ``None``, ``"iem"``, ``"cli"`` +#: and the frame-product tag ``"cli.archive"`` all resolve to the same fetch. +#: ``"acis"`` (CLAUDE.md's aspirational climate-LIVE_V1 spec) raises loudly — +#: there is no ACIS leg in this code path. +_VALID_SOURCES: frozenset[str] = frozenset({"iem", "cli", "cli.archive"}) + +#: Frame-level settlement-product identity stamped on ``climate()`` frames. +#: Distinct from the per-row provider tag ``"iem"`` (the fetch endpoint). This +#: mirrors D-03's frame-vs-row split — the frame is the settlement PRODUCT, +#: the rows carry the ENDPOINT that served them. +_CLIMATE_FRAME_SOURCE = "cli.archive" + +#: Public output column order. The JOIN vocabulary (``cli_*``) plus provenance. +_PUBLIC_COLUMNS: tuple[str, ...] = ( + "date", + "station", + "cli_high_f", + "cli_low_f", + "cli_report_type", + "source", + "issued_at", +) + + +def climate( + station: str, + start: str, + end: str, + *, + source: str | None = None, + backend: BackendT = "pandas", + return_type: ReturnTypeT = "dataframe", +) -> pd.DataFrame | list[dict]: + """Return NWS CLI daily settlement labels for ``station`` over ``[start, end]``. + + The settlement source for Kalshi NHIGH/NLOW markets: the official NWS CLI + daily high/low, merged via the frozen dedup policy (highest + ``report_type_priority`` with STRICT ``>`` — overnight final wins). + + Parameters + ---------- + station : str + ICAO code (e.g. ``"KNYC"``) or 3-letter NWS code. + start, end : str + ISO date strings (YYYY-MM-DD), inclusive bounds. + source : {None, "iem", "cli", "cli.archive"}, keyword-only, default None + Provenance selector. A degenerate single-authority axis (D-08): IEM + CLI is the sole live provider, so every accepted value resolves to the + same fetch. ``None`` means best-available (== ``"iem"``). ``"acis"`` + raises ``ValueError`` — there is no ACIS leg in this code path. + backend : {"pandas", "polars"}, keyword-only, default "pandas" + Output frame type. ``"polars"`` requires ``return_type="wrapper"`` + (polars frames carry no ``df.attrs`` for provenance). + return_type : {"dataframe", "wrapper", "list"}, keyword-only, default "dataframe" + - ``"dataframe"`` (default): a ``pandas.DataFrame`` with provenance on + ``df.attrs``. + - ``"list"``: a plain ``list[dict]`` of rows. + - ``"wrapper"``: a :class:`~mostlyright.core.result.MostlyRightResult`. + + Returns + ------- + pd.DataFrame | list[dict] + One row per CLI settlement day with the public column contract: + ``date, station, cli_high_f, cli_low_f, cli_report_type`` plus + provenance ``source`` (per-row endpoint tag ``"iem"``) and + ``issued_at``. Frame identity ``df.attrs["source"] == "cli.archive"``. + + Raises + ------ + ValueError + If ``source`` is not in ``{None, "iem", "cli", "cli.archive"}``, if + ``start``/``end`` are not ISO dates, or if the + ``backend``/``return_type`` pair is incoherent (e.g. + ``backend="polars", return_type="dataframe"``). + + Notes + ----- + - CLI is a daily batch product — ``climate()`` has NO grain axis and NO + live surface (D-09/D-10 class 1: re-fetchable upstream). + - This wraps the LOCAL CLI fetcher only; it makes NO hosted call. + + Examples + -------- + >>> from mostlyright.weather import climate + >>> df = climate("KNYC", "2026-01-01", "2026-01-03") # doctest: +SKIP + >>> df.attrs["source"] # doctest: +SKIP + 'cli.archive' + """ + if source is not None and source not in _VALID_SOURCES: + raise ValueError( + f"source must be one of {sorted(_VALID_SOURCES)} or None; got {source!r}. " + "IEM CLI is the sole live provider — there is no ACIS leg in this code path." + ) + + # Coherence of backend/return_type BEFORE any network (mirrors research()). + validate_backend_kwargs(backend, return_type) + + # Eager ISO validation — fail fast for malformed dates before any network. + date.fromisoformat(start) + date.fromisoformat(end) + + # Resolve station via the existing research.py helper (single source of truth). + from mostlyright.research import _fetch_climate_range, _resolve_station + + info = _resolve_station(station) + + # Delegate the fetch to the shared range fetcher (annual cache + IEM CLI + # download + frozen merge_climate dedup). The fetch is YEAR-granular + # (research.py year-loop) and does NOT trim to the query window — without + # the window-trim below a 3-day query would leak the whole year (the exact + # bug commit 40ef469 fixed for research_by_source). + raw_rows = _fetch_climate_range(info, start, end) + + return _project_climate_rows( + raw_rows, + start, + end, + info_code=info.code, + backend=backend, + return_type=return_type, + ) + + +def _project_climate_rows( + raw_rows: list[dict], + start: str, + end: str, + *, + info_code: str, + backend: BackendT, + return_type: ReturnTypeT, +) -> pd.DataFrame | list[dict]: + """Window-trim, project to the public JOIN vocabulary, stamp frame identity. + + The year-granular fetch is trimmed to ``observation_date in [start, end]`` + via :func:`mostlyright._internal._window.in_query_window` (inclusive both + ends, string compare) so a short query never leaks the whole cached year. + Raw parser keys are projected onto the public columns + (``high_temp_f -> cli_high_f``, ``low_temp_f -> cli_low_f``, + ``report_type -> cli_report_type``), rows sorted by date, and the settlement + -product frame tag ``"cli.archive"`` stamped as the FINAL op before return. + """ + from mostlyright._internal._window import in_query_window + + trimmed = [ + r for r in raw_rows if in_query_window(r.get("observation_date"), start, end) + ] + + projected: list[dict] = [ + { + "date": r.get("observation_date"), + "station": info_code, + "cli_high_f": r.get("high_temp_f"), + "cli_low_f": r.get("low_temp_f"), + "cli_report_type": r.get("report_type"), + # per-row provenance = the fetch ENDPOINT tag ("iem"), NOT the + # settlement-product frame tag. Preserve it verbatim. + "source": r.get("source"), + "issued_at": r.get("issued_at"), + } + for r in trimmed + ] + projected.sort(key=lambda row: row["date"] or "") + + # Lazy pandas import INSIDE the return path (never at module top): the fast + # suite runs without extras and `import mostlyright.weather` stays pandas-free. + import pandas as pd + + from mostlyright.core._backend_dispatch import wrap_result + + df = pd.DataFrame(projected, columns=list(_PUBLIC_COLUMNS)) + # FINAL op before return: stamp the settlement-product frame identity. + # contextlib.suppress mirrors research.py — attrs assignment is best-effort. + with contextlib.suppress(AttributeError): + df.attrs["source"] = _CLIMATE_FRAME_SOURCE + + return wrap_result( + df, + backend=backend, + return_type=return_type, + source=_CLIMATE_FRAME_SOURCE, + ) + + +__all__ = ["climate"] From ad7737a750f1e7d2fe4bc539da57ede02a6d41c4 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:50:23 +0200 Subject: [PATCH 08/52] feat(30-02): re-export climate() from mostlyright.weather - add climate import + "climate" to __all__ alongside obs --- packages/weather/src/mostlyright/weather/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/weather/src/mostlyright/weather/__init__.py b/packages/weather/src/mostlyright/weather/__init__.py index b92dd4d9..6bc3fcb4 100644 --- a/packages/weather/src/mostlyright/weather/__init__.py +++ b/packages/weather/src/mostlyright/weather/__init__.py @@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Any from mostlyright.weather._fetchers._open_meteo import fetch_open_meteo +from mostlyright.weather.climate import climate as climate # Phase 30 (30-02) surface from mostlyright.weather.obs import obs as obs # re-export Phase 7 public surface from mostlyright.weather.satellite import satellite as satellite # Phase 25 surface @@ -45,7 +46,7 @@ __version__ = _dist_version("mostlyrightmd-weather") except PackageNotFoundError: # editable/source tree without installed dist metadata __version__ = "0.0.0+unknown" -__all__ = ["__version__", "cwop", "fetch_open_meteo", "obs", "satellite"] +__all__ = ["__version__", "climate", "cwop", "fetch_open_meteo", "obs", "satellite"] def __getattr__(name: str) -> Any: From 6c3490dd13ae070dedc356c06bcbe175d0e92ce6 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:51:30 +0200 Subject: [PATCH 09/52] feat(30-03): rename research()->dataset() with research alias sharing one body - dataset() is now the canonical def in research.py (single ~450-line body) - research = dataset plain forwarding binding, NO DeprecationWarning this release (D-02) - export both dataset and research from mostlyright.__init__ __all__ - positional signature + default-path aggregation unchanged (parity, D-15) --- packages/core/src/mostlyright/__init__.py | 6 ++++-- packages/core/src/mostlyright/research.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/core/src/mostlyright/__init__.py b/packages/core/src/mostlyright/__init__.py index 569db045..fe2d57dd 100644 --- a/packages/core/src/mostlyright/__init__.py +++ b/packages/core/src/mostlyright/__init__.py @@ -1,8 +1,9 @@ """mostlyright — local-first SDK for prediction-market weather settlement research. Sprint 0 v0.1.0 ships: -- ``mostlyright.research(station, from_date, to_date, ...)`` — the v0.14.1 ``pairs()`` join, +- ``mostlyright.dataset(station, from_date, to_date, ...)`` — the v0.14.1 ``pairs()`` join, lifted from monorepo-v0.14.1, calling AWC + IEM + GHCNh + NWS CLI directly. + ``mostlyright.research`` remains a fully working alias of ``dataset`` (Phase 30 D-02). - ``mostlyright.snapshot`` — settlement-window math (LST, market_close_utc). Adjacent surfaces: @@ -29,7 +30,7 @@ __version__ = "0.0.0+unknown" from mostlyright.discover import discover -from mostlyright.research import research +from mostlyright.research import dataset, research from mostlyright.stations import CATALOG, Station, StationCatalog __all__ = [ @@ -37,6 +38,7 @@ "Station", "StationCatalog", "__version__", + "dataset", "discover", "live", "research", diff --git a/packages/core/src/mostlyright/research.py b/packages/core/src/mostlyright/research.py index c66acd62..fee8f8ae 100644 --- a/packages/core/src/mostlyright/research.py +++ b/packages/core/src/mostlyright/research.py @@ -1556,7 +1556,7 @@ def _normalize_forecast_source( return items -def research( +def dataset( station: str | None = None, from_date: str | None = None, to_date: str | None = None, @@ -1999,9 +1999,18 @@ def research( return_type=return_type, # type: ignore[arg-type] source=wrap_source, retrieved_at=result.attrs.get("retrieved_at"), - schema_id=None, # research() returns heterogeneous pairs, not a single schema + schema_id=None, # dataset() returns heterogeneous pairs, not a single schema qc=qc_summary, ) -__all__ = ["research"] +# Phase 30 D-02: ``dataset()`` is the canonical name; ``research()`` is a plain +# forwarding binding that shares the single ~450-line body above (no duplication). +# NO ``DeprecationWarning`` on ``research()`` in this release — the warning lands +# one minor release later, then removal at 2.0 (see 30-CONTEXT.md D-02/D-12). Both +# names are exported so existing `mostlyright.research(...)` call sites keep working +# byte-for-byte while `dataset()` becomes the name docs/README/quickstarts teach. +research = dataset + + +__all__ = ["dataset", "research"] From 70cae4b174afaeb18ff68b7671416bb3f870e1fb Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:52:04 +0200 Subject: [PATCH 10/52] fix(30-02): make climate.py pandas-free at module scope - mostlyright.core eagerly loads the pandas-backed validator, so importing _backend_dispatch at module top transitively pulled pandas - move validate_backend_kwargs to a lazy in-function import; keep BackendT/ ReturnTypeT under TYPE_CHECKING (annotations are strings via __future__) - satisfies acceptance: 'from mostlyright.weather import climate' needs no pandas at module scope (pandas only imported when called) --- packages/weather/src/mostlyright/weather/climate.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/climate.py b/packages/weather/src/mostlyright/weather/climate.py index c5a9394a..bf771aad 100644 --- a/packages/weather/src/mostlyright/weather/climate.py +++ b/packages/weather/src/mostlyright/weather/climate.py @@ -33,13 +33,13 @@ import contextlib from datetime import date -from typing import TYPE_CHECKING, Literal - -from mostlyright.core._backend_dispatch import BackendT, ReturnTypeT, validate_backend_kwargs +from typing import TYPE_CHECKING if TYPE_CHECKING: import pandas as pd + from mostlyright.core._backend_dispatch import BackendT, ReturnTypeT + #: Provenance values ``source=`` accepts. A degenerate single-authority axis #: (D-08): IEM CLI is the sole live provider, so ``None``, ``"iem"``, ``"cli"`` #: and the frame-product tag ``"cli.archive"`` all resolve to the same fetch. @@ -136,6 +136,10 @@ def climate( ) # Coherence of backend/return_type BEFORE any network (mirrors research()). + # Lazy import: `mostlyright.core` eagerly loads the pandas-backed validator, + # so keeping this out of module scope preserves the pandas-free import. + from mostlyright.core._backend_dispatch import validate_backend_kwargs + validate_backend_kwargs(backend, return_type) # Eager ISO validation — fail fast for malformed dates before any network. From aa5db6a626d599d3b57a0454a0a3f933a34e34d1 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:53:22 +0200 Subject: [PATCH 11/52] feat(30-03): relocate assert_source_identity into mostlyright.core and re-export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - new core/source_identity.py is the canonical home (D-04) - re-exported from mostlyright.core.__init__ __all__ - mode2 back-imports it (core defines, mode2 imports — no cycle) - scrub 'Mode 2 dispatch requested' -> source-pinned neutral wording (D-11) - add role= kwarg (default 'observations', backward-compatible) --- .../core/src/mostlyright/core/__init__.py | 2 + .../src/mostlyright/core/source_identity.py | 64 +++++++++++++++++++ packages/core/src/mostlyright/mode2.py | 33 ++++------ 3 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 packages/core/src/mostlyright/core/source_identity.py diff --git a/packages/core/src/mostlyright/core/__init__.py b/packages/core/src/mostlyright/core/__init__.py index c0f2d85a..2711e50e 100644 --- a/packages/core/src/mostlyright/core/__init__.py +++ b/packages/core/src/mostlyright/core/__init__.py @@ -34,6 +34,7 @@ ) from mostlyright.core.result import MostlyRightResult from mostlyright.core.schema import ColumnSpec, Schema, SchemaRegistration +from mostlyright.core.source_identity import assert_source_identity from mostlyright.core.temporal import ( KnowledgeView, LeakageDetector, @@ -58,5 +59,6 @@ "TemporalDriftError", "TimePoint", "assert_no_leakage", + "assert_source_identity", "validate_dataframe", ] diff --git a/packages/core/src/mostlyright/core/source_identity.py b/packages/core/src/mostlyright/core/source_identity.py new file mode 100644 index 00000000..1de94461 --- /dev/null +++ b/packages/core/src/mostlyright/core/source_identity.py @@ -0,0 +1,64 @@ +"""Canonical home for the ``assert_source_identity`` source-pinning gate. + +``assert_source_identity`` is a source-identity VALIDATOR utility, not a "mode". +Phase 30 (D-04) relocates its canonical implementation here from the deprecated +``mostlyright.mode2`` module and re-exports it from :mod:`mostlyright.core`, so +callers reach it as ``from mostlyright.core import assert_source_identity`` without +touching the deprecated module path. + +Import direction is one-way: ``core`` DEFINES the function; ``mode2`` back-imports +it for backward compatibility. The reverse (``core`` importing ``mode2``) would be +a partial-initialization cycle — ``mode2`` already imports +``mostlyright.core.exceptions`` at module load. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from mostlyright.core.exceptions import SourceMismatchError + +if TYPE_CHECKING: + import pandas as pd + + +__all__ = ["assert_source_identity"] + + +def assert_source_identity( + df: pd.DataFrame, + expected_source: str, + *, + role: str = "observations", +) -> None: + """Raise :class:`SourceMismatchError` if any row's source != expected. + + A defense-in-depth per-row gate: mirrors the per-row check in + :func:`mostlyright.core.validator.validate_dataframe` but at the + source-pinned dispatch layer so callers get a role-flavored error + message when a pinned single-source frame carries a foreign row. + + Args: + df: The DataFrame to check. If it has no ``source`` column the + function is a silent no-op (the Validator handles that case). + expected_source: The source every row must carry. + role: The role label surfaced on the raised error (default + ``"observations"``). + + Raises: + SourceMismatchError: when one or more rows carry a ``source`` + other than ``expected_source``. + """ + if "source" not in df.columns: + return + bad = df[df["source"] != expected_source] + if not bad.empty: + distinct = sorted(set(bad["source"].dropna().astype(str).tolist())) + raise SourceMismatchError( + f"source-pinned dispatch requested {expected_source!r} but DataFrame " + f"contains {len(bad)} row(s) with other sources: {distinct}", + schema_source=expected_source, + data_source=distinct[0] if distinct else "", + role=role, + catalog_warning=None, + ) diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index 1236a215..af6b6c6a 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -25,7 +25,14 @@ from typing import TYPE_CHECKING, Any from mostlyright._internal._window import in_query_window as _in_query_window -from mostlyright.core.exceptions import SourceMismatchError + +# Phase 30 D-04: ``assert_source_identity`` now lives canonically in +# ``mostlyright.core.source_identity``. Re-import it here so the historical +# ``from mostlyright.mode2 import assert_source_identity`` path keeps working +# (back-compat binding). Direction is core -> (defines), mode2 -> (imports); +# the reverse would be a partial-init cycle since mode2 also imports +# ``mostlyright.core.exceptions`` at load. +from mostlyright.core.source_identity import assert_source_identity if TYPE_CHECKING: import pandas as pd @@ -242,24 +249,6 @@ def research_by_source( ) -def assert_source_identity(df: pd.DataFrame, expected_source: str) -> None: - """Raise :class:`SourceMismatchError` if any row's source != expected. - - Mirrors the per-row check in - :func:`mostlyright.core.validator.validate_dataframe` but at the - Mode 2 dispatch layer so callers get a Mode-2-flavored error - message naming the role. - """ - if "source" not in df.columns: - return - bad = df[df["source"] != expected_source] - if not bad.empty: - distinct = sorted(set(bad["source"].dropna().astype(str).tolist())) - raise SourceMismatchError( - f"Mode 2 dispatch requested {expected_source!r} but DataFrame " - f"contains {len(bad)} row(s) with other sources: {distinct}", - schema_source=expected_source, - data_source=distinct[0] if distinct else "", - role="observations", - catalog_warning=None, - ) +# ``assert_source_identity`` is imported at the top of this module from +# ``mostlyright.core.source_identity`` (its canonical home as of Phase 30 D-04). +# It remains listed in ``__all__`` so the back-compat import path resolves. From ed1cb95dea4207aeec58c57b6b18bf76d5a23730 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:53:39 +0200 Subject: [PATCH 12/52] test(30-02): climate() surface, window-trim, provenance, byte-equivalence - pytest.importorskip('pandas') guard for no-extras collection - window-trim drops out-of-window year rows (whole-year-leak guard, 40ef469) - frame/row source split: attrs cli.archive, per-row iem - source=None == source=iem; acis raises ValueError - return_type=list -> list[dict]; polars+dataframe raises ValueError - byte-equivalence vs the build_pairs_row cli_* projection oracle --- packages/weather/tests/test_climate.py | 300 +++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 packages/weather/tests/test_climate.py diff --git a/packages/weather/tests/test_climate.py b/packages/weather/tests/test_climate.py new file mode 100644 index 00000000..f6222832 --- /dev/null +++ b/packages/weather/tests/test_climate.py @@ -0,0 +1,300 @@ +"""Phase 30 (30-02) — climate() standalone CLI settlement-label domain table. + +Covers: + +- Surface contract: signature, source accept-set, exact public columns. +- Window-trim correctness (a 3-day query in a warm-cached YEAR of rows returns + exactly the 3 in-window daily rows — no whole-year leak; the fix mirrors the + bug commit 40ef469 patched for research_by_source). +- Provenance frame/row split: df.attrs["source"] == "cli.archive" (settlement + PRODUCT), per-row source == "iem" (fetch ENDPOINT). +- source=None == source="iem" row-for-row equivalence. +- return_type="list" returns list[dict]; backend="polars"+return_type="dataframe" + raises ValueError (incoherent output pair). +- Byte-equivalence: climate()'s cli_high_f/cli_low_f for a window equal the + cli_* columns dataset()/research() project for the same raw CLI rows. + +CI runs WITHOUT extras — the unguarded pandas import below is guarded via +`pytest.importorskip` so collection never crashes when pandas is absent. +Uses local/patched fixtures only (no network in the fast suite); does NOT +import boto3/xarray/s3fs/polars. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +pd = pytest.importorskip("pandas") + + +# Merged CLI rows exactly as research._fetch_climate_range returns them: whole +# parser/merge dicts keyed on observation_date. The fetch is YEAR-granular, so +# a 3-day query must window-trim these down. Rows span the whole year 2026 to +# prove the trim drops out-of-window rows (whole-year-leak guard). +def _year_of_cli_rows() -> list[dict]: + return [ + { + "station_code": "NYC", + "observation_date": "2026-01-01", + "high_temp_f": 40, + "low_temp_f": 31, + "report_type": "final", + "report_type_priority": 3.0, + "source": "iem", + "product_id": "CLINYC", + "issued_at": "2026-01-02T06:15:00Z", + }, + { + "station_code": "NYC", + "observation_date": "2026-01-02", + "high_temp_f": 38, + "low_temp_f": 29, + "report_type": "final", + "report_type_priority": 3.0, + "source": "iem", + "product_id": "CLINYC", + "issued_at": "2026-01-03T06:15:00Z", + }, + { + "station_code": "NYC", + "observation_date": "2026-01-03", + "high_temp_f": 45, + "low_temp_f": 33, + "report_type": "final", + "report_type_priority": 3.0, + "source": "iem", + "product_id": "CLINYC", + "issued_at": "2026-01-04T06:15:00Z", + }, + # Out-of-window rows from later in the same fetched year — MUST be + # trimmed away by climate()'s window-trim. Without the trim, a 3-day + # query would leak these (the whole-year-leak bug 40ef469). + { + "station_code": "NYC", + "observation_date": "2026-06-15", + "high_temp_f": 88, + "low_temp_f": 70, + "report_type": "final", + "report_type_priority": 3.0, + "source": "iem", + "product_id": "CLINYC", + "issued_at": "2026-06-16T06:15:00Z", + }, + { + "station_code": "NYC", + "observation_date": "2026-12-31", + "high_temp_f": 34, + "low_temp_f": 20, + "report_type": "final", + "report_type_priority": 3.0, + "source": "iem", + "product_id": "CLINYC", + "issued_at": "2027-01-01T06:15:00Z", + }, + ] + + +def _patch_fetch(rows: list[dict]): + # climate() imports _fetch_climate_range from mostlyright.research inside its + # body; patch it at the source module so the imported name resolves to the mock. + return patch("mostlyright.research._fetch_climate_range", return_value=rows) + + +# --------------------------------------------------------------------------- # +# Surface / signature +# --------------------------------------------------------------------------- # +def test_climate_importable_from_weather_package() -> None: + import mostlyright.weather as w + + assert "climate" in w.__all__ + assert callable(w.climate) + + +def test_climate_signature_kwonly_defaults() -> None: + import inspect + + from mostlyright.weather.climate import climate + + params = inspect.signature(climate).parameters + assert list(params)[:3] == ["station", "start", "end"] + for name in ("source", "backend", "return_type"): + assert params[name].kind == inspect.Parameter.KEYWORD_ONLY + assert params["source"].default is None + assert params["backend"].default == "pandas" + assert params["return_type"].default == "dataframe" + + +def test_climate_bogus_dates_raise_before_network() -> None: + from mostlyright.weather.climate import climate + + with pytest.raises(ValueError): + climate("KNYC", "not-a-date", "2026-01-03") + + +# --------------------------------------------------------------------------- # +# source accept-set +# --------------------------------------------------------------------------- # +def test_climate_acis_source_raises_value_error() -> None: + from mostlyright.weather.climate import climate + + with pytest.raises(ValueError, match="source must be"): + climate("KNYC", "2026-01-01", "2026-01-03", source="acis") + + +@pytest.mark.parametrize("src", [None, "iem", "cli", "cli.archive"]) +def test_climate_accepts_valid_sources(src) -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + df = climate("KNYC", "2026-01-01", "2026-01-03", source=src) + assert len(df) == 3 + + +def test_climate_source_none_equals_iem_row_for_row() -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + df_none = climate("KNYC", "2026-01-01", "2026-01-03", source=None) + with _patch_fetch(_year_of_cli_rows()): + df_iem = climate("KNYC", "2026-01-01", "2026-01-03", source="iem") + pd.testing.assert_frame_equal(df_none, df_iem) + + +# --------------------------------------------------------------------------- # +# Window-trim (whole-year-leak guard) +# --------------------------------------------------------------------------- # +def test_climate_window_trim_drops_out_of_window_year_rows() -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + df = climate("KNYC", "2026-01-01", "2026-01-03") + + # Exactly the 3 in-window daily rows; June + Dec year rows trimmed away. + assert len(df) == 3 + assert df["date"].tolist() == ["2026-01-01", "2026-01-02", "2026-01-03"] + assert df["date"].between("2026-01-01", "2026-01-03").all() + + +def test_climate_rows_sorted_by_date() -> None: + from mostlyright.weather.climate import climate + + shuffled = list(reversed(_year_of_cli_rows()[:3])) + with _patch_fetch(shuffled): + df = climate("KNYC", "2026-01-01", "2026-01-03") + assert df["date"].tolist() == sorted(df["date"].tolist()) + + +# --------------------------------------------------------------------------- # +# Column contract + frame/row provenance split +# --------------------------------------------------------------------------- # +def test_climate_exact_public_columns() -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + df = climate("KNYC", "2026-01-01", "2026-01-03") + + assert list(df.columns) == [ + "date", + "station", + "cli_high_f", + "cli_low_f", + "cli_report_type", + "source", + "issued_at", + ] + + +def test_climate_frame_and_row_source_split() -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + df = climate("KNYC", "2026-01-01", "2026-01-03") + + # Frame identity = settlement PRODUCT. + assert df.attrs["source"] == "cli.archive" + # Per-row provenance = the fetch ENDPOINT (IEM), preserved verbatim. + assert set(df["source"]) == {"iem"} + + +def test_climate_projects_join_vocabulary_values() -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + df = climate("KNYC", "2026-01-01", "2026-01-03") + + row0 = df.iloc[0] + assert row0["date"] == "2026-01-01" + assert row0["cli_high_f"] == 40 + assert row0["cli_low_f"] == 31 + assert row0["cli_report_type"] == "final" + assert row0["issued_at"] == "2026-01-02T06:15:00Z" + + +# --------------------------------------------------------------------------- # +# Output-knob unification (backend + return_type) +# --------------------------------------------------------------------------- # +def test_climate_return_type_list_returns_list_of_dicts() -> None: + from mostlyright.weather.climate import climate + + with _patch_fetch(_year_of_cli_rows()): + out = climate("KNYC", "2026-01-01", "2026-01-03", return_type="list") + + assert isinstance(out, list) + assert len(out) == 3 + assert all(isinstance(r, dict) for r in out) + assert set(out[0]) == { + "date", + "station", + "cli_high_f", + "cli_low_f", + "cli_report_type", + "source", + "issued_at", + } + + +def test_climate_polars_dataframe_pair_raises_value_error() -> None: + from mostlyright.weather.climate import climate + + # Incoherent output pair rejected BEFORE the [polars] extra is probed. + with pytest.raises(ValueError, match="polars"): + climate( + "KNYC", + "2026-01-01", + "2026-01-03", + backend="polars", + return_type="dataframe", + ) + + +# --------------------------------------------------------------------------- # +# Byte-equivalence vs the dataset()/research() cli_* projection oracle +# --------------------------------------------------------------------------- # +def test_climate_labels_byte_equal_to_join_projection() -> None: + """climate()'s cli_* for a window equal what research()/dataset() project. + + The join (``_pairs.build_pairs_row``) reads exactly + ``climate_dict.get("high_temp_f")`` / ``.get("low_temp_f")`` / + ``.get("report_type")`` from the SAME merged CLI dicts climate() consumes. + This test pins that climate() introduces ZERO transformation drift versus + the settlement-join label projection. + """ + from mostlyright.weather.climate import climate + + rows = _year_of_cli_rows() + with _patch_fetch(rows): + df = climate("KNYC", "2026-01-01", "2026-01-03") + + # Oracle: the exact projection build_pairs_row applies per settlement day. + in_window = { + r["observation_date"]: r + for r in rows + if "2026-01-01" <= r["observation_date"] <= "2026-01-03" + } + for _, row in df.iterrows(): + cli = in_window[row["date"]] + assert row["cli_high_f"] == cli.get("high_temp_f") + assert row["cli_low_f"] == cli.get("low_temp_f") + assert row["cli_report_type"] == cli.get("report_type") From 9a2641ad33a726a7248be613fa142758dcc03eaa Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:54:04 +0200 Subject: [PATCH 13/52] style(30-02): ruff import-order + format on climate module and tests --- packages/weather/src/mostlyright/weather/climate.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/climate.py b/packages/weather/src/mostlyright/weather/climate.py index bf771aad..dc69676b 100644 --- a/packages/weather/src/mostlyright/weather/climate.py +++ b/packages/weather/src/mostlyright/weather/climate.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: import pandas as pd - from mostlyright.core._backend_dispatch import BackendT, ReturnTypeT #: Provenance values ``source=`` accepts. A degenerate single-authority axis @@ -189,9 +188,7 @@ def _project_climate_rows( """ from mostlyright._internal._window import in_query_window - trimmed = [ - r for r in raw_rows if in_query_window(r.get("observation_date"), start, end) - ] + trimmed = [r for r in raw_rows if in_query_window(r.get("observation_date"), start, end)] projected: list[dict] = [ { @@ -212,7 +209,6 @@ def _project_climate_rows( # Lazy pandas import INSIDE the return path (never at module top): the fast # suite runs without extras and `import mostlyright.weather` stays pandas-free. import pandas as pd - from mostlyright.core._backend_dispatch import wrap_result df = pd.DataFrame(projected, columns=list(_PUBLIC_COLUMNS)) From 4b6177ff9905172a5493f3977139bf591a2e1d32 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Wed, 8 Jul 2026 23:58:10 +0200 Subject: [PATCH 14/52] feat(30-03): deprecate research_by_source() + mostlyright.mode2 import path - call-time DeprecationWarning on research_by_source (every call, stacklevel=2) - module __getattr__ once-per-session import-path warning (set guard) - both warnings name obs(source=..., granularity="observation") + retrain note (D-04/D-12) - KEEP post-merge body byte-unchanged (zero silent data change) - purge Mode 1/Mode 2 vocab from mode2 docstrings/comments (D-11) - runtime "Mode 2 source must be one of" string deferred to 30-03-5 --- packages/core/src/mostlyright/mode2.py | 151 ++++++++++++++++++------- 1 file changed, 109 insertions(+), 42 deletions(-) diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index af6b6c6a..35bed5a3 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -1,56 +1,94 @@ -"""Phase 3 — Mode 2 source-explicit dispatch for ``research()``. - -Mode 1 (the v0.14.1 parity baseline shipped in Phase 1) uses an -internal AWC > IEM > GHCNh priority. Mode 2 lets the caller pin -observations to a single named source — the workflow Vojtech wanted for -strategies that need source-identified training pairs that backtest the -same way they trade. +"""Source-pinned observation dispatch (DEPRECATED module — Phase 30 D-04). + +.. deprecated:: + This module and :func:`research_by_source` are deprecated. Pin observations + to a single named source with ``obs(source=..., granularity="observation")`` + from :mod:`mostlyright.weather` instead, and reach + :func:`assert_source_identity` from :mod:`mostlyright.core`. The + ``mostlyright.mode2`` import path is retained for backward compatibility and + scheduled for removal at least two minor releases out. + +The default source-blind join (:func:`mostlyright.dataset` / +:func:`mostlyright.research`) uses an internal AWC > IEM > GHCNh priority. +Source-pinned dispatch lets the caller pin observations to a single named +source — the workflow Vojtech wanted for strategies that need +source-identified training pairs that backtest the same way they trade. Surface: - :func:`research_by_source(station, source, from_date, to_date)` — - Mode 2 entry point. Returns a DataFrame where every row's - ``source`` is the supplied ``source`` ID. + source-pinned entry point (deprecated). Returns a DataFrame where every + row's ``source`` is the supplied ``source`` ID. - :func:`assert_source_identity(df, expected_source)` — raise - :class:`SourceMismatchError` if any row's source disagrees. + :class:`SourceMismatchError` if any row's source disagrees (canonical home + is now :mod:`mostlyright.core.source_identity`). -See ``docs/design.md`` §R for the source-identity invariant Mode 2 +See ``docs/design.md`` §R for the source-identity invariant this surface enforces. """ from __future__ import annotations +import warnings from datetime import UTC, datetime, timedelta from datetime import date as _date -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final from mostlyright._internal._window import in_query_window as _in_query_window # Phase 30 D-04: ``assert_source_identity`` now lives canonically in -# ``mostlyright.core.source_identity``. Re-import it here so the historical -# ``from mostlyright.mode2 import assert_source_identity`` path keeps working -# (back-compat binding). Direction is core -> (defines), mode2 -> (imports); -# the reverse would be a partial-init cycle since mode2 also imports -# ``mostlyright.core.exceptions`` at load. -from mostlyright.core.source_identity import assert_source_identity +# ``mostlyright.core.source_identity``. Re-import it here (under a private alias) +# so the historical ``from mostlyright.mode2 import assert_source_identity`` path +# keeps working via the module ``__getattr__`` below — which also fires the +# once-per-session import-path DeprecationWarning. Direction is +# core -> (defines), mode2 -> (imports); the reverse would be a partial-init +# cycle since mode2 also imports ``mostlyright.core.exceptions`` at load. +from mostlyright.core.source_identity import ( + assert_source_identity as _assert_source_identity, +) if TYPE_CHECKING: import pandas as pd -__all__ = ["assert_source_identity", "research_by_source"] +# Both names are the deprecated public surface of this module; they are resolved +# lazily through ``__getattr__`` (which fires the import-path DeprecationWarning), +# so they are not present as module globals. ``noqa: F822`` because ruff cannot +# see through the ``__getattr__`` indirection — the names ARE reachable. +__all__ = ["assert_source_identity", "research_by_source"] # noqa: F822 + +# Phase 30 D-04: deprecation copy. ``research_by_source`` warns at call time; +# the ``mostlyright.mode2`` import path warns once per session via __getattr__. +_RESEARCH_BY_SOURCE_DEPRECATION: Final[str] = ( + "research_by_source() is deprecated; use " + 'obs(source=..., granularity="observation") from mostlyright.weather. ' + "Note: the new pinned path returns pre-merge FULL single-source coverage, a " + "DIFFERENT row composition than research_by_source()'s post-merge output — " + "migrating a pinned feature is a RETRAIN event, not a find-replace. " + "Removal target: >=2 minor releases out." +) +_MODE2_IMPORT_DEPRECATION: Final[str] = ( + "The mostlyright.mode2 import path is deprecated; import " + 'obs(source=..., granularity="observation") from mostlyright.weather and ' + "assert_source_identity from mostlyright.core instead. " + "Removal target: >=2 minor releases out." +) -#: Mode 2 supported observation sources for v0.1.0. +#: Names whose import-path deprecation warning has already fired this session. +_MODE2_IMPORT_WARNED: set[str] = set() + + +#: Supported source-pinned observation sources for v0.1.0. #: #: Production parsers emit BARE source tags (`iem`, `awc`, `ghcnh`) per #: ``_iem.py:230``, ``_awc.py:320``, ``_ghcnh.py:306``. The dotted #: forms (`iem.archive`, `iem.live`, `awc.live`, `ghcnh.archive`) are #: mostlyright' canonical source-identity vocabulary documented in #: ``docs/design.md`` §R and used by every schema's ``_registered_source``. -#: Mode 2 accepts BOTH at the input boundary and uses +#: The pinned path accepts BOTH at the input boundary and uses #: :data:`_SOURCE_ALIASES` to map each request to the parser tags that -#: satisfy it — without this, a Mode 2 request for "iem.archive" would +#: satisfy it — without this, a request for "iem.archive" would #: find zero rows in production (parser emits bare "iem"). The #: per-row source overlay column is the parser-emitted tag, NOT a #: rewrite to the requested form, so downstream Validator schemas see @@ -73,12 +111,12 @@ # ``_in_query_window`` is re-exported from ``mostlyright._internal._window`` # (see the top-of-module import). The predicate was extracted into a shared -# internal so the observation-grain fork in ``weather/obs.py`` and this Mode-2 -# alias share ONE implementation; the private name is kept bound here so -# existing call sites below are byte-unchanged. +# internal so the observation-grain fork in ``weather/obs.py`` and this +# source-pinned alias share ONE implementation; the private name is kept bound +# here so existing call sites below are byte-unchanged. -def research_by_source( +def _research_by_source( station: str, source: str, from_date: str, @@ -88,24 +126,34 @@ def research_by_source( backend: str = "pandas", return_type: str = "dataframe", ) -> pd.DataFrame | list[dict[str, Any]]: - """Return source-explicit Mode 2 raw observations. - - Calls the existing observation fetcher (the same one Mode 1 - aggregates into pairs) and filters rows to the requested source. - Result carries ``df.attrs["source"] = source`` + - ``df.attrs["retrieved_at"]`` + a per-row ``source`` overlay column — - the canonical validator contract used by all of mostlyright' Mode 2 + """Return source-pinned raw observations (DEPRECATED — Phase 30 D-04). + + .. deprecated:: + Use ``obs(source=..., granularity="observation")`` from + :mod:`mostlyright.weather`. The new pinned path returns pre-merge FULL + single-source coverage — a DIFFERENT row composition than this + function's post-merge output — so migrating a pinned feature is a + RETRAIN event, not a find-replace. Removal target: >=2 minor releases + out. This function KEEPS its existing post-merge code path unchanged so + current callers see zero silent data change (D-04/D-12). + + Calls the existing observation fetcher (the same one the source-blind + :func:`mostlyright.dataset` join aggregates into pairs) and filters rows to + the requested source. Result carries ``df.attrs["source"] = source`` + + ``df.attrs["retrieved_at"]`` + a per-row ``source`` overlay column — the + canonical validator contract used by all of mostlyright' source-pinned surfaces. **v0.1.0 limitation (codex iter-1 P1):** ``_fetch_observations_range`` - applies the Mode 1 merge policy (AWC > IEM > GHCNh on key collision) - BEFORE returning rows. A Mode 2 caller asking for ``iem.archive`` + applies the source-blind merge policy (AWC > IEM > GHCNh on key collision) + BEFORE returning rows. A caller asking for ``iem.archive`` over a window where AWC also has data therefore sees only the IEM rows AWC did NOT supersede — not the full IEM coverage of the - upstream feed. v0.2 will add a pre-merge source-isolated path; v0.1 - callers who need that today should call the per-source fetchers - in ``mostlyright.weather._fetchers`` directly and skip the catalog - layer. + upstream feed. The new ``obs(source=..., granularity="observation")`` path + fixes this via a pre-merge source-isolated fetch; v0.1 + callers who need that through this legacy surface should call the + per-source fetchers in ``mostlyright.weather._fetchers`` directly and skip + the catalog layer. Args: station: 3- or 4-letter station code (e.g. ``"NYC"``, ``"KNYC"``). @@ -130,6 +178,12 @@ def research_by_source( SourceMismatchError: filtered rows came back tagged with a different source than requested (defense-in-depth). """ + # Phase 30 D-04: call-time deprecation. Emitted on EVERY call (not + # once-per-session) so a caller running under ``-W error::DeprecationWarning`` + # surfaces it deterministically. stacklevel=2 points at the caller's call + # site (mirrors _cache_dir.py:49-53). + warnings.warn(_RESEARCH_BY_SOURCE_DEPRECATION, DeprecationWarning, stacklevel=2) + if source not in _VALID_OBSERVATION_SOURCES: raise ValueError( f"Mode 2 source must be one of {sorted(_VALID_OBSERVATION_SOURCES)}; got {source!r}" @@ -249,6 +303,19 @@ def research_by_source( ) -# ``assert_source_identity`` is imported at the top of this module from -# ``mostlyright.core.source_identity`` (its canonical home as of Phase 30 D-04). -# It remains listed in ``__all__`` so the back-compat import path resolves. +# Phase 30 D-04: the whole ``mostlyright.mode2`` module is deprecated. Its two +# public names (``research_by_source``, ``assert_source_identity``) are resolved +# through ``__getattr__`` — kept OUT of module globals as ``_research_by_source`` +# / ``_assert_source_identity`` — so that ANY access via the ``mostlyright.mode2`` +# import path fires a once-per-session import-path DeprecationWarning (mirrors +# ``core/exceptions.py:1073-1086``, reusing the module-level ``set`` guard). The +# PRIVATE names (``_SOURCE_ALIASES``, ``_VALID_OBSERVATION_SOURCES``, +# ``_in_query_window``) that ``research.py`` imports function-locally stay in +# globals and are therefore NOT routed here — internal reuse never warns. +def __getattr__(name: str) -> Any: + if name in ("research_by_source", "assert_source_identity"): + if name not in _MODE2_IMPORT_WARNED: + warnings.warn(_MODE2_IMPORT_DEPRECATION, DeprecationWarning, stacklevel=2) + _MODE2_IMPORT_WARNED.add(name) + return _research_by_source if name == "research_by_source" else _assert_source_identity + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From b910edea968d463f5abd9293aea34d76574417e9 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:08:48 +0200 Subject: [PATCH 15/52] feat(30-03): unify output kwargs on backend/return_type, deprecate as_dataframe - new shared resolve_return_type + AS_DATAFRAME_DEPRECATION in _backend_dispatch (D-06) - add backend/return_type to obs(); route via validate_backend_kwargs - dataset()/research(), research_by_source(), obs(): warn on explicit as_dataframe - map as_dataframe=False->return_type=list, True->dataframe; explicit return_type wins - as_dataframe default -> None sentinel; no-arg call byte-stable per function - daily_extremes docstring documents backend/return_type (already on the pair) - update test_obs_surface signature assertion for sentinel + new kwargs (lockstep) --- .../src/mostlyright/core/_backend_dispatch.py | 59 +++++++++++- .../core/src/mostlyright/international.py | 13 +++ packages/core/src/mostlyright/mode2.py | 23 ++++- packages/core/src/mostlyright/research.py | 57 +++++++---- .../weather/src/mostlyright/weather/obs.py | 96 +++++++++++++++---- tests/weather/test_obs_surface.py | 10 +- 6 files changed, 217 insertions(+), 41 deletions(-) diff --git a/packages/core/src/mostlyright/core/_backend_dispatch.py b/packages/core/src/mostlyright/core/_backend_dispatch.py index f9f951db..e5756b3d 100644 --- a/packages/core/src/mostlyright/core/_backend_dispatch.py +++ b/packages/core/src/mostlyright/core/_backend_dispatch.py @@ -27,8 +27,9 @@ from __future__ import annotations +import warnings from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Final, Literal from mostlyright.core.result import MostlyRightResult @@ -37,13 +38,69 @@ __all__ = [ + "AS_DATAFRAME_DEPRECATION", "BackendT", "ReturnTypeT", "convert_to_backend", + "resolve_return_type", "validate_backend_kwargs", "wrap_result", ] +#: Phase 30 D-06: single shared deprecation message. ``backend`` + ``return_type`` +#: is the WINNING output convention (dominant, centralized, validated, and +#: strictly more expressive). The legacy boolean ``as_dataframe`` is deprecated +#: across every public surface that carried it. +AS_DATAFRAME_DEPRECATION: Final[str] = ( + "as_dataframe= is deprecated; use return_type= instead " + '(as_dataframe=True -> return_type="dataframe", ' + 'as_dataframe=False -> return_type="list"). When both are supplied an ' + "explicit return_type= wins. Removal target: >=2 minor releases out." +) + + +def resolve_return_type( + as_dataframe: bool | None, + return_type: str, + *, + default_return_type: str, + stacklevel: int = 3, +) -> str: + """Fold the deprecated ``as_dataframe`` bool into the ``return_type`` knob. + + Phase 30 D-06 precedence rules: + + - ``as_dataframe`` passed EXPLICITLY (not ``None``) -> emit a + :class:`DeprecationWarning`. + - An explicit ``return_type`` (any value != ``default_return_type``) WINS + over ``as_dataframe`` — the caller opted into the new knob. + - Otherwise, ``as_dataframe=True`` -> ``"dataframe"``, + ``as_dataframe=False`` -> ``"list"``. + - When ``as_dataframe`` is ``None`` (not passed), ``return_type`` is + returned verbatim so the per-function DEFAULT is byte-stable. + + Args: + as_dataframe: The legacy bool. ``None`` means the caller did not pass + it (sentinel default). + return_type: The value of the ``return_type`` kwarg as received. + default_return_type: The function's own default ``return_type`` value, + used to detect whether the caller passed an explicit ``return_type``. + stacklevel: Passed to :func:`warnings.warn` so the warning points at the + public caller's call site (default 3: warn -> resolver -> public fn). + + Returns: + The effective ``return_type`` string. + """ + if as_dataframe is None: + return return_type + + warnings.warn(AS_DATAFRAME_DEPRECATION, DeprecationWarning, stacklevel=stacklevel) + + # Explicit return_type wins over as_dataframe. + if return_type != default_return_type: + return return_type + return "dataframe" if as_dataframe else "list" + BackendT = Literal["pandas", "polars"] ReturnTypeT = Literal["dataframe", "wrapper", "list"] diff --git a/packages/core/src/mostlyright/international.py b/packages/core/src/mostlyright/international.py index b9a477c2..7c680f00 100644 --- a/packages/core/src/mostlyright/international.py +++ b/packages/core/src/mostlyright/international.py @@ -231,6 +231,19 @@ def daily_extremes( to_date: Inclusive local-calendar-day end. merge: Merge policy. Only ``"live_v1"`` is supported in v0.1.0; anything else raises ``ValueError``. + backend: Output frame backend, ``{"pandas", "polars"}`` (default + ``"pandas"``). ``"polars"`` requires ``return_type="wrapper"`` + (polars frames carry no ``df.attrs`` provenance). Part of the + unified output convention (Phase 30 D-06); validated via the shared + ``validate_backend_kwargs`` gate. + return_type: Output shape, ``{"list", "dataframe", "wrapper"}``. + **Defaults to ``"list"``** for this function — a no-arg call returns + ``list[dict]`` (byte-stable, D-12/D-15), unlike the DataFrame-default + surfaces. ``"dataframe"`` returns a ``pandas.DataFrame``; + ``"wrapper"`` returns a + :class:`~mostlyright.core.result.MostlyRightResult`. ``daily_extremes`` + has no ``as_dataframe`` knob — it was born on the winning + ``backend``/``return_type`` convention. Returns: ``list[dict]`` with one entry per local calendar day in diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index 35bed5a3..be371a7b 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -122,7 +122,7 @@ def _research_by_source( from_date: str, to_date: str, *, - as_dataframe: bool = True, + as_dataframe: bool | None = None, backend: str = "pandas", return_type: str = "dataframe", ) -> pd.DataFrame | list[dict[str, Any]]: @@ -160,8 +160,11 @@ def _research_by_source( source: One of :data:`_VALID_OBSERVATION_SOURCES`. from_date: ``YYYY-MM-DD`` inclusive. to_date: ``YYYY-MM-DD`` inclusive. - as_dataframe: When True (default) return a pandas DataFrame; - else return the raw ``list[dict]`` rows. + as_dataframe: **DEPRECATED (Phase 30 D-06)** — use ``return_type`` + (``True`` → ``"dataframe"``, ``False`` → ``"list"``). Passing it + explicitly emits a ``DeprecationWarning``; an explicit + ``return_type`` wins. Unset (default), ``return_type`` governs and a + no-arg call still returns a pandas DataFrame. Returns: DataFrame (or list[dict]) with one row per observation matching @@ -192,7 +195,17 @@ def _research_by_source( # Codex iter-4 P2 fix: validate backend/return_type BEFORE any # network fetch or cache write so a typo doesn't trigger live # API calls + cache mutations before raising. - from mostlyright.core._backend_dispatch import validate_backend_kwargs + from mostlyright.core._backend_dispatch import ( + resolve_return_type, + validate_backend_kwargs, + ) + + # Phase 30 D-06: fold the deprecated `as_dataframe` bool into `return_type`. + # Effective "list" preserves the historic `as_dataframe=False` raw-rows path. + return_type = resolve_return_type( + as_dataframe, return_type, default_return_type="dataframe" + ) + _as_dataframe = return_type != "list" validate_backend_kwargs(backend, return_type) # type: ignore[arg-type] @@ -246,7 +259,7 @@ def _research_by_source( and _in_query_window(r.get("observed_at"), from_date, to_date) ] - if not as_dataframe: + if not _as_dataframe: return filtered import pandas as pd diff --git a/packages/core/src/mostlyright/research.py b/packages/core/src/mostlyright/research.py index fee8f8ae..6e8c0120 100644 --- a/packages/core/src/mostlyright/research.py +++ b/packages/core/src/mostlyright/research.py @@ -1572,7 +1572,7 @@ def dataset( forecast_model: str | None = None, forecast_models: list[str] | None = None, forecast_source: str | list[str] | tuple[str, ...] = "iem_mos", - as_dataframe: bool = True, + as_dataframe: bool | None = None, tz_override: str | None = None, qc: bool = False, backend: str = "pandas", @@ -1606,9 +1606,15 @@ def dataset( run selection. Passed through to :func:`mostlyright._internal._pairs.build_pairs`. Ignored when ``include_forecast=False``. - as_dataframe: When ``True`` (default) return a Pandas DataFrame - indexed by ``date``. When ``False`` return the raw - ``list[dict]`` rows produced by ``build_pairs``. + as_dataframe: **DEPRECATED (Phase 30 D-06)** — use ``return_type`` + instead (``as_dataframe=True`` → ``return_type="dataframe"``, + ``as_dataframe=False`` → ``return_type="list"``). Passing it + explicitly emits a ``DeprecationWarning``; when both are supplied an + explicit ``return_type`` wins. Left unset (default), the + ``return_type`` knob governs and a no-arg call still returns a + Pandas DataFrame indexed by ``date`` (byte-stable). ``return_type= + "list"`` returns the raw ``list[dict]`` rows produced by + ``build_pairs``. tz_override: IANA timezone name override for stations not yet in :data:`mostlyright.snapshot._STATION_TZ`. Passed through to settlement-date math; rarely needed for the US registry @@ -1685,13 +1691,31 @@ def dataset( include_forecast=include_forecast, ) + # Phase 30 D-06: fold the deprecated `as_dataframe` bool into the winning + # `return_type` knob. `as_dataframe=None` (default) means the caller did not + # pass it → `return_type` governs unchanged (byte-stable default). Passing + # `as_dataframe` explicitly emits a DeprecationWarning; an explicit + # `return_type` wins over it. Effective "list" preserves the historic + # `as_dataframe=False` raw-rows behavior below. + from mostlyright.core._backend_dispatch import ( + resolve_return_type, + validate_backend_kwargs, + ) + + return_type = resolve_return_type( + as_dataframe, return_type, default_return_type="dataframe" + ) + # Phase 6 codex iter-2 P2 fix: validate backend / return_type kwargs # BEFORE any network fetch or cache write. A typo in the new kwargs # otherwise hits live APIs + mutates the parquet cache before raising. - from mostlyright.core._backend_dispatch import validate_backend_kwargs - validate_backend_kwargs(backend, return_type) # type: ignore[arg-type] + # Internal daily/list fork. Effective "list" return_type == the historic + # `as_dataframe=False` path (raw build_pairs rows), so byte-stable for the + # existing callers who passed `as_dataframe=False` (D-12/D-15). + _as_dataframe = return_type != "list" + # Phase 10 selector validation. Backwards-compat: the original # `station, from_date, to_date` positional signature still works — # detected when `station` is provided AND no new selector is. @@ -1953,25 +1977,26 @@ def dataset( # caller opted into a non-default backend or return_type. from mostlyright.core._backend_dispatch import wrap_result - result = pairs_to_dataframe(rows) if as_dataframe else rows + result = pairs_to_dataframe(rows) if _as_dataframe else rows # Phase 3.4: surface qc summary on df.attrs when the qc=True opt-in - # ran. Mode 1 parity rows themselves are unchanged (only attrs). - if qc_summary is not None and as_dataframe: + # ran. The default-path parity rows themselves are unchanged (only attrs). + if qc_summary is not None and _as_dataframe: with contextlib.suppress(AttributeError): result.attrs["qc"] = qc_summary - # Mode 2 provenance contract: stamp the requested source on the - # returned DataFrame when pinned (mirrors mode2.research_by_source's + # Source-pin provenance contract: stamp the requested source on the + # returned DataFrame when pinned (mirrors research_by_source's # df.attrs["source"]). No attrs change when source is None — the # parity firewall keeps the default path untouched. - if source is not None and as_dataframe: + if source is not None and _as_dataframe: with contextlib.suppress(AttributeError): result.attrs["source"] = source - # Phase 6 W3-T2: when as_dataframe=False the caller wants raw list[dict] — - # backend/return_type kwargs do not apply. Same when backend kwarg is the - # default (pandas + dataframe) — return unchanged for zero-overhead. - if not as_dataframe or (backend == "pandas" and return_type == "dataframe"): + # Effective return_type "list" (or historic as_dataframe=False) → the caller + # wants raw list[dict]; backend/return_type wrapping does not apply. Same + # when backend is the default (pandas + dataframe) — return unchanged for + # zero-overhead byte-stability. + if not _as_dataframe or (backend == "pandas" and return_type == "dataframe"): return result # Wrapper / polars conversion path. result.attrs carries source + diff --git a/packages/weather/src/mostlyright/weather/obs.py b/packages/weather/src/mostlyright/weather/obs.py index b1d78221..6f7d55ae 100644 --- a/packages/weather/src/mostlyright/weather/obs.py +++ b/packages/weather/src/mostlyright/weather/obs.py @@ -77,7 +77,9 @@ def obs( source: Source | None = None, strategy: Strategy = "auto", granularity: Granularity = "daily", - as_dataframe: bool = True, + as_dataframe: bool | None = None, + backend: str = "pandas", + return_type: str = "dataframe", ) -> pd.DataFrame | list[dict]: """Return observation data for ``station`` over ``[start, end]``. @@ -114,14 +116,27 @@ def obs( ``"auto"`` (the default) self-selects the right tactic, including the source-isolated ``exact_window`` path for any pinned ``source=`` query. The default ships as ``"auto"`` and never churns between releases. - as_dataframe : bool, keyword-only, default True - If True (default), return ``pandas.DataFrame``. If False, return ``list[dict]``. + as_dataframe : bool | None, keyword-only, default None + **DEPRECATED (Phase 30 D-06)** — use ``return_type`` instead + (``True`` → ``"dataframe"``, ``False`` → ``"list"``). Passing it + explicitly emits a ``DeprecationWarning``; when both are supplied an + explicit ``return_type`` wins. Left unset (default), ``return_type`` + governs and a no-arg call still returns a ``pandas.DataFrame``. + backend : {"pandas", "polars"}, keyword-only, default "pandas" + Output frame backend. ``"polars"`` requires ``return_type="wrapper"`` + (polars frames carry no ``df.attrs`` provenance). Validated via the + shared ``validate_backend_kwargs`` gate. + return_type : {"dataframe", "list", "wrapper"}, keyword-only, default "dataframe" + Output shape: raw ``pandas.DataFrame`` (default), plain ``list[dict]`` + of rows, or a :class:`~mostlyright.core.result.MostlyRightResult` + wrapper carrying provenance. Returns ------- pd.DataFrame | list[dict] Daily obs_* aggregate rows (``granularity="daily"``) or merged - per-report rows (``granularity="observation"``). + per-report rows (``granularity="observation"``). Shape follows + ``return_type``. Raises ------ @@ -161,6 +176,20 @@ def obs( f"granularity must be one of {sorted(_VALID_GRANULARITIES)}; got {granularity!r}" ) + # Phase 30 D-06: fold the deprecated `as_dataframe` bool into the winning + # `backend`/`return_type` pair, then validate the pair BEFORE any network + # fetch or cache write. Effective "list" == the historic `as_dataframe=False` + # raw-rows path; the default (as_dataframe unset) is byte-stable. + from mostlyright.core._backend_dispatch import ( + resolve_return_type, + validate_backend_kwargs, + ) + + return_type = resolve_return_type( + as_dataframe, return_type, default_return_type="dataframe" + ) + validate_backend_kwargs(backend, return_type) # type: ignore[arg-type] + # Eager ISO validation — fail fast for malformed dates before any network. date.fromisoformat(start) date.fromisoformat(end) @@ -198,11 +227,11 @@ def obs( # any DataFrame build, attrs-safe) to the strict UTC window so the # +1-day fetch extension does not leak into the result. Every row's # bare `source` column is preserved verbatim (NEVER overwritten — - # mirrors mode2.py:224-229); the FRAME identity is a distinct - # fused-policy tag when unpinned. raw_rows are grain-agnostic, so this - # fork is post-dispatch only (D-15: daily branch untouched). + # mirrors mode2.py); the FRAME identity is a distinct fused-policy tag + # when unpinned. raw_rows are grain-agnostic, so this fork is + # post-dispatch only (D-15: daily branch untouched). return _observation_grain_rows( - raw_rows, start, end, source=source, as_dataframe=as_dataframe + raw_rows, start, end, source=source, backend=backend, return_type=return_type ) # Aggregate raw observation rows to daily summary rows. The output schema is @@ -211,11 +240,27 @@ def obs( # rows are byte-equivalent across strategies. aggregated = _aggregate_daily_rows(raw_rows, info, start, end) - if as_dataframe: - import pandas as pd + # Effective "list" == the historic `as_dataframe=False` raw-rows path; + # backend/return_type wrapping does not apply. + if return_type == "list": + return aggregated + + import pandas as pd + + df = pd.DataFrame(aggregated) + if backend == "pandas" and return_type == "dataframe": + return df - return pd.DataFrame(aggregated) - return aggregated + # Wrapper / polars conversion path. obs() daily aggregates carry no single + # source identity (they are fused AWC>IEM>GHCNh); surface "merged.live_v1". + from mostlyright.core._backend_dispatch import wrap_result + + return wrap_result( + df, + backend=backend, # type: ignore[arg-type] + return_type=return_type, # type: ignore[arg-type] + source=source if source is not None else _MERGED_FRAME_SOURCE, + ) def _observation_grain_rows( @@ -224,7 +269,8 @@ def _observation_grain_rows( end: str, *, source: Source | None, - as_dataframe: bool, + backend: str, + return_type: str, ) -> pd.DataFrame | list[dict]: """Trim merged per-report rows to the UTC window; stamp frame identity. @@ -238,12 +284,16 @@ def _observation_grain_rows( ``df.attrs["source"] = "merged.live_v1"`` so pinned single-source schemas loudly reject it and :class:`MergedObservationSchema` accepts it. When ``source`` is pinned, stamp the bare source (a genuine single-source frame). + + ``return_type="list"`` returns the trimmed raw rows (historic + ``as_dataframe=False`` behavior); ``"dataframe"`` / ``"wrapper"`` build the + frame and honor ``backend`` via the shared ``wrap_result``. """ from mostlyright._internal._window import in_query_window trimmed = [r for r in raw_rows if in_query_window(r.get("observed_at"), start, end)] - if not as_dataframe: + if return_type == "list": return trimmed import contextlib @@ -256,12 +306,24 @@ def _observation_grain_rows( for key in _OBSERVATION_NUMERIC_KEYS: if key in df.columns: df[key] = pd.to_numeric(df[key], errors="coerce") - # FINAL op before return: stamp frame identity. contextlib.suppress mirrors - # research.py:1968 — attrs assignment is a no-op-safe best-effort. + # Stamp frame identity. contextlib.suppress mirrors research.py — attrs + # assignment is a no-op-safe best-effort. frame_source = source if source is not None else _MERGED_FRAME_SOURCE with contextlib.suppress(AttributeError): df.attrs["source"] = frame_source - return df + + if backend == "pandas" and return_type == "dataframe": + return df + + # Wrapper / polars conversion path. + from mostlyright.core._backend_dispatch import wrap_result + + return wrap_result( + df, + backend=backend, # type: ignore[arg-type] + return_type=return_type, # type: ignore[arg-type] + source=frame_source, + ) def _dispatch_strategy( diff --git a/tests/weather/test_obs_surface.py b/tests/weather/test_obs_surface.py index 8f425662..5444f578 100644 --- a/tests/weather/test_obs_surface.py +++ b/tests/weather/test_obs_surface.py @@ -36,13 +36,19 @@ def test_obs_signature_has_required_kwonly_params(): assert "station" in params assert "start" in params assert "end" in params - for name in ("source", "strategy", "as_dataframe"): + for name in ("source", "strategy", "as_dataframe", "backend", "return_type"): assert params[name].kind == inspect.Parameter.KEYWORD_ONLY, f"{name} must be keyword-only" # B-6: ships as "auto" from PLAN-02 onward; never churns. assert params["source"].default is None assert params["strategy"].default == "auto" - assert params["as_dataframe"].default is True + # Phase 30 D-06: `as_dataframe` is DEPRECATED and defaults to the None + # sentinel (so explicit passing is detectable); the winning + # `backend`/`return_type` pair now governs output shape. A no-arg call still + # returns a DataFrame (byte-stable) — the default is the sentinel, not True. + assert params["as_dataframe"].default is None + assert params["backend"].default == "pandas" + assert params["return_type"].default == "dataframe" def test_obs_invalid_source_raises_value_error(): From 48c01dabb7ba3a98b1dbb2d645f9666fe7d2442b Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:12:44 +0200 Subject: [PATCH 16/52] refactor(30-03): purge Mode 1/Mode 2 vocab in research.py + reconcile runtime strings - research.py Axis-B docstrings: Mode 1->IEM MOS, Mode 2->Per-NWP-model (NWP) - research.py comments/source docstring -> source-blind/source-pinned language - source= docstring now points at obs(source=..., granularity="observation") - mode2.py runtime "Mode 2 source must be one of" -> "source must be one of" - update test_mode2 match string in lockstep - NOTE: stale plan line refs (1718-1720 NotImplementedError w/ Mode vocab) did not exist in live 1.12.1 code; TS mirror at mode2.test.ts:151 is 30-04 scope --- packages/core/src/mostlyright/mode2.py | 2 +- packages/core/src/mostlyright/research.py | 67 ++++++++++++----------- packages/core/tests/test_mode2.py | 2 +- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index be371a7b..b69ac5a5 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -189,7 +189,7 @@ def _research_by_source( if source not in _VALID_OBSERVATION_SOURCES: raise ValueError( - f"Mode 2 source must be one of {sorted(_VALID_OBSERVATION_SOURCES)}; got {source!r}" + f"source must be one of {sorted(_VALID_OBSERVATION_SOURCES)}; got {source!r}" ) # Codex iter-4 P2 fix: validate backend/return_type BEFORE any diff --git a/packages/core/src/mostlyright/research.py b/packages/core/src/mostlyright/research.py index 6e8c0120..f81893af 100644 --- a/packages/core/src/mostlyright/research.py +++ b/packages/core/src/mostlyright/research.py @@ -923,7 +923,7 @@ def _fetch_iem_mos_range( *, model: str = "nbe", ) -> dict[str, list[dict[str, Any]]]: - """Mode 1 — fetch IEM MOS forecasts grouped by settlement date (ISO). + """IEM MOS — fetch IEM MOS forecasts grouped by settlement date (ISO). Wraps ``mostlyright.weather._fetchers._iem_mos.fetch_iem_mos`` and pivots its tabular DataFrame to the ``{date_iso: [forecast_row, ...]}`` shape @@ -1013,7 +1013,7 @@ def _fetch_nwp_models_range( to_date: str, forecast_models: list[str], ) -> dict[str, dict[str, list[dict[str, Any]]]]: - """Mode 2 — fetch per-model NWP forecasts covering the LST settlement window. + """Per-NWP-model (NWP) — fetch per-model NWP forecasts covering the LST settlement window. Iterates each model in ``forecast_models``. For each LST settlement date in ``[from_date, to_date]`` fetches enough ``(cycle, fxx)`` pairs @@ -1205,7 +1205,7 @@ def _run_qc_and_write_sidecar( via an ``error`` key. The QC engine reads observation rows but DOES NOT mutate them — the - parity gate's "Mode 1 row contents are byte-identical to v0.14.1 + parity gate's "default-path row contents are byte-identical to v0.14.1 pairs() output" invariant is preserved. """ summary: dict[str, Any] = { @@ -1295,8 +1295,8 @@ def _run_qc_and_write_sidecar( return summary # Optional: IEM-vs-GHCNh crosscheck. Only runs if both sources are - # present in the raw observations (downstream Mode 1 path doesn't - # actually carry the source column in every row, so we only flag + # present in the raw observations (the downstream source-blind join path + # doesn't actually carry the source column in every row, so we only flag # when partitionable). try: if "source" in obs_df.columns and "event_time" in obs_df.columns: @@ -1619,7 +1619,7 @@ def dataset( :data:`mostlyright.snapshot._STATION_TZ`. Passed through to settlement-date math; rarely needed for the US registry (all entries are covered). - source: Pin observations to a single named source (Mode 2). One of + source: Pin observations to a single named source. One of :data:`mostlyright.mode2._VALID_OBSERVATION_SOURCES` (bare ``"iem"``/``"awc"``/``"ghcnh"`` or the dotted canonical forms); validated early, so an unknown value raises ``ValueError`` @@ -1627,12 +1627,14 @@ def dataset( requested source's alias-accept set before aggregation, and ``df.attrs["source"]`` records the pin. The ``cli_*`` columns are UNAFFECTED — NWS CLI is the settlement source, independent of the - observation-source pin. **v0.1.0 limitation:** the Mode 1 merge - policy (AWC > IEM > GHCNh on key collision) runs BEFORE the pin - filter, so pinning ``"iem"`` over a window where AWC also + observation-source pin. **v0.1.0 limitation:** the source-blind + merge policy (AWC > IEM > GHCNh on key collision) runs BEFORE the + pin filter, so pinning ``"iem"`` over a window where AWC also reported yields only the IEM rows AWC did not supersede — not the - full upstream IEM coverage. ``source=None`` (default) leaves the - path byte-identical to the v0.14.1 parity baseline. + full upstream IEM coverage. For full pre-merge single-source + coverage use ``obs(source=..., granularity="observation")``. + ``source=None`` (default) leaves the path byte-identical to the + v0.14.1 parity baseline. sources: Multi-source subset selector (plural). Validated at the mutual-exclusion boundary but the data-selection wiring lands in v0.3 — passing it raises ``NotImplementedError``. Mutually @@ -1755,8 +1757,8 @@ def dataset( "subset (sources=[...]) is the v0.3 deliverable." ) - # Mode 2 single-source pin. Validate source= EARLY — before any network - # or cache work — against Mode 2's source vocabulary, then compute the + # Single-source pin (source=). Validate source= EARLY — before any network + # or cache work — against the source-pinned vocabulary, then compute the # alias-accept set used to subset the raw observation rows after the # fetch. Reuse mode2's tables rather than duplicating them. The import # is function-local (not module-level): mode2 already imports research @@ -1815,10 +1817,10 @@ def dataset( info = _resolve_station(station) - # Phase 17 PLAN-09: include_forecast=True wires Mode 1 (IEM MOS) + - # optional Mode 2 (per-NWP-model). Mode 1 emits the additive - # ``fcst_*`` columns the v0.14.1 schema reserves; Mode 2 emits - # ``fcst_*_nwp_`` columns on top. include_forecast=False + # Phase 17 PLAN-09: include_forecast=True wires IEM MOS + optional + # per-NWP-model (NWP) forecasts. IEM MOS emits the additive + # ``fcst_*`` columns the v0.14.1 schema reserves; the per-model NWP path + # emits ``fcst_*_nwp_`` columns on top. include_forecast=False # leaves both dicts empty so build_pairs() sees forecasts_by_date=None # AND nwp_forecasts_by_model_date=None — byte-equivalent baseline. iem_mos_by_date: dict[str, list[dict[str, Any]]] = {} @@ -1902,21 +1904,23 @@ def dataset( prefetched_awc_rows=awc_rows, ) - # Mode 2 single-source pin (source=): keep only rows whose parser- - # emitted `source` is in the requested alias-accept set. Applied as a + # Single-source pin (source=): keep only rows whose parser-emitted + # `source` is in the requested alias-accept set. Applied as a # pure post-fetch row subset BEFORE the settlement-date bucketing below, # which already trims the window — so the pinned frame keeps the same - # columns/shape as unpinned research(). PARITY FIREWALL: this runs only + # columns/shape as the unpinned join. PARITY FIREWALL: this runs only # when source is not None (accepted_sources stays None otherwise), so # the default path is byte-identical. cli_* (NWS CLI settlement) columns # are unaffected — CLI is the settlement source, independent of the # observation-source pin. # - # v0.1.0 limitation (inherited from Mode 2): _fetch_observations_range - # applies the Mode 1 merge policy (AWC > IEM > GHCNh on key collision) - # BEFORE this filter, so pinning "iem" over a window where AWC also - # reported yields only the IEM rows AWC did NOT supersede — not the full - # upstream IEM coverage. A pre-merge source-isolated path is v0.2 work. + # v0.1.0 limitation (inherited from the source-pinned surface): + # _fetch_observations_range applies the source-blind merge policy + # (AWC > IEM > GHCNh on key collision) BEFORE this filter, so pinning + # "iem" over a window where AWC also reported yields only the IEM rows + # AWC did NOT supersede — not the full upstream IEM coverage. + # obs(source=..., granularity="observation") is the pre-merge source- + # isolated path. if accepted_sources is not None: raw_obs = [r for r in raw_obs if r.get("source") in accepted_sources] @@ -1924,7 +1928,8 @@ def dataset( # Phase 3.4: opt-in QC. Runs the QCEngine + IEM-vs-GHCNh crosscheck # against raw_obs WITHOUT mutating the rows themselves (parity gate - # invariant: Mode 1 must NEVER alter observation row contents). The + # invariant: the default join path must NEVER alter observation row + # contents). The # QC summary is stashed on the returned DataFrame's df.attrs and the # sidecar parquet is written to ~/.mostlyright/cache/v1/observations_qc/ # for later join. @@ -1959,8 +1964,8 @@ def dataset( # PLAN-09: when include_forecast=False, pass None for both forecast # dicts so build_pairs/build_pairs_row hit the parity-preserving paths # (no fcst_* population beyond the default-None scaffolding). When - # include_forecast=True, hand over the IEM MOS Mode 1 dict and (if - # forecast_models was provided) the Mode 2 per-model NWP dict. + # include_forecast=True, hand over the IEM MOS dict and (if + # forecast_models was provided) the per-model NWP dict. rows = build_pairs( info.code, dates, @@ -2008,11 +2013,11 @@ def dataset( # stamp this (pairs are heterogeneous), so the fallback to now() # is acceptable here — but future pairs-level provenance work # should populate result.attrs["retrieved_at"] for consistency. - # Mode 2 provenance parity: when source= is pinned, thread it explicitly - # into the wrapper the same way mode2.research_by_source does + # Source-pin provenance parity: when source= is pinned, thread it explicitly + # into the wrapper the same way research_by_source does # (source=source), so the wrapper contract doesn't depend on the # df.attrs round-trip above. On this path the round-trip also works - # (as_dataframe=True guarantees a real DataFrame), so this is + # (a real DataFrame is guaranteed), so this is # defense-in-depth, not a behavior fix. When source is None the default # path is byte-identical (unchanged attrs-derived string). wrap_source = ( diff --git a/packages/core/tests/test_mode2.py b/packages/core/tests/test_mode2.py index c12fa1bf..165e9260 100644 --- a/packages/core/tests/test_mode2.py +++ b/packages/core/tests/test_mode2.py @@ -29,7 +29,7 @@ def test_valid_sources_documented(): def test_unknown_source_raises(): - with pytest.raises(ValueError, match="Mode 2 source must be one of"): + with pytest.raises(ValueError, match="source must be one of"): research_by_source("KNYC", "bogus.source", "2025-01-01", "2025-01-31") From 48a03084254cc0a4ab1dd35532baf6ea47a7758f Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:14:58 +0200 Subject: [PATCH 17/52] test(30-03): alias-equivalence + deprecation coverage - test_dataset_alias.py: dataset is research (one body), both exported, research warning-free this release - test_mode2_deprecation.py: research_by_source call-time + mode2 import-path each warn exactly once naming obs(source=..., granularity="observation"); output unchanged; as_dataframe=False maps to list - test_parity.py: dataset()==research() assertion in the live parity case + network-free identity lock in the fast suite --- packages/core/tests/test_dataset_alias.py | 49 ++++++ packages/core/tests/test_mode2_deprecation.py | 159 ++++++++++++++++++ tests/test_parity.py | 17 ++ 3 files changed, 225 insertions(+) create mode 100644 packages/core/tests/test_dataset_alias.py create mode 100644 packages/core/tests/test_mode2_deprecation.py diff --git a/packages/core/tests/test_dataset_alias.py b/packages/core/tests/test_dataset_alias.py new file mode 100644 index 00000000..aa183cef --- /dev/null +++ b/packages/core/tests/test_dataset_alias.py @@ -0,0 +1,49 @@ +"""Phase 30 (30-03) — ``dataset()`` / ``research()`` alias equivalence. + +Locks the D-02 rename contract: + +- ``mostlyright.dataset`` and ``mostlyright.research`` are the SAME callable + (a plain forwarding binding sharing one body) — so their output is + byte-identical by construction, no network call required. +- Both names are exported from ``mostlyright.__all__``. +- ``research()`` emits NO ``DeprecationWarning`` this release (the warning + lands one minor release later — D-02/D-12). +""" + +from __future__ import annotations + +import warnings + +import mostlyright + + +def test_dataset_and_research_are_the_same_callable() -> None: + """The alias shares ONE body — identity guarantees byte-identical output.""" + assert mostlyright.dataset is mostlyright.research + assert callable(mostlyright.dataset) + + +def test_both_names_exported() -> None: + assert "dataset" in mostlyright.__all__ + assert "research" in mostlyright.__all__ + + +def test_research_alias_emits_no_deprecation_warning_this_release() -> None: + """D-02: ``research()`` stays warning-free in 1.13.0. + + Merely resolving / referencing the alias must not warn — the deprecation + train adds the warning one release later. + """ + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + # Reference + re-import the alias; neither may raise. + _ = mostlyright.research + from mostlyright import research as _research # noqa: F401 + from mostlyright.research import dataset as _dataset # noqa: F401 + from mostlyright.research import research as _research2 # noqa: F401 + + +def test_dataset_importable_from_research_module() -> None: + from mostlyright.research import dataset, research + + assert dataset is research is mostlyright.dataset diff --git a/packages/core/tests/test_mode2_deprecation.py b/packages/core/tests/test_mode2_deprecation.py new file mode 100644 index 00000000..49a506c1 --- /dev/null +++ b/packages/core/tests/test_mode2_deprecation.py @@ -0,0 +1,159 @@ +"""Phase 30 (30-03) — ``mostlyright.mode2`` deprecation contract (D-04). + +Locks: + +- ``research_by_source()`` emits exactly one call-time ``DeprecationWarning`` + naming the new ``obs(source=..., granularity="observation")`` surface. +- The ``mostlyright.mode2`` import PATH emits exactly one (per-session) + ``DeprecationWarning`` naming the new surface, via the module ``__getattr__``. +- ``research_by_source()`` output is UNCHANGED — the post-merge code path is + preserved (zero silent data change for current callers). +""" + +from __future__ import annotations + +import importlib +import warnings + +import pytest + +pd = pytest.importorskip("pandas") + +_NEW_SURFACE = 'obs(source=..., granularity="observation")' + + +def _reset_import_warned() -> None: + """Clear the once-per-session import-path guard so a warning can re-fire.""" + import mostlyright.mode2 as _m2 + + _m2._MODE2_IMPORT_WARNED.clear() + + +def test_research_by_source_call_emits_one_deprecation_warning(tmp_path, monkeypatch) -> None: + """Call-time DeprecationWarning fires and names the new surface.""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + research_module = importlib.import_module("mostlyright.research") + monkeypatch.setattr(research_module, "_fetch_observations_range", lambda *a, **kw: []) + monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) + + from mostlyright.mode2 import research_by_source + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + research_by_source("KNYC", "iem", "2025-01-06", "2025-01-12") + + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + call_warnings = [w for w in dep if "research_by_source() is deprecated" in str(w.message)] + assert len(call_warnings) == 1, f"expected exactly one call-time warning, got {len(call_warnings)}" + assert _NEW_SURFACE in str(call_warnings[0].message) + # Migration guardrail (D-12): the message flags the retrain event. + assert "RETRAIN" in str(call_warnings[0].message).upper() + + +def test_mode2_import_path_emits_one_deprecation_warning() -> None: + """Accessing a public name via the mode2 path warns once per session.""" + _reset_import_warned() + import mostlyright.mode2 as m2 + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = m2.research_by_source # first access → warns + _ = m2.research_by_source # second access → guarded, no re-warn + + dep = [ + w + for w in caught + if issubclass(w.category, DeprecationWarning) + and "mostlyright.mode2 import path is deprecated" in str(w.message) + ] + assert len(dep) == 1, f"expected exactly one import-path warning, got {len(dep)}" + assert _NEW_SURFACE in str(dep[0].message) + + +def test_mode2_import_path_warns_for_assert_source_identity() -> None: + _reset_import_warned() + import mostlyright.mode2 as m2 + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _ = m2.assert_source_identity + + dep = [ + w + for w in caught + if issubclass(w.category, DeprecationWarning) + and "mostlyright.mode2 import path is deprecated" in str(w.message) + ] + assert len(dep) == 1 + # Same object as the canonical core home. + from mostlyright.core import assert_source_identity as core_asi + + assert m2.assert_source_identity is core_asi + + +def test_research_by_source_output_unchanged(tmp_path, monkeypatch) -> None: + """Post-merge body preserved: filter + provenance identical to pre-change.""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + research_module = importlib.import_module("mostlyright.research") + synthetic_obs = [ + { + "station_code": "KNYC", + "observed_at": "2025-01-06T12:00:00+00:00", + "source": "iem", # parser emits the BARE form + "temp_c": 5.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-06T13:00:00+00:00", + "source": "awc", + "temp_c": 6.0, + }, + ] + monkeypatch.setattr(research_module, "_fetch_observations_range", lambda *a, **kw: synthetic_obs) + monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) + + from mostlyright.mode2 import research_by_source + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + df = research_by_source("KNYC", "iem.archive", "2025-01-06", "2025-01-12") + + # Unchanged post-merge behaviour: dotted request matches bare parser tag, + # per-row source is the truthful bare tag, attrs record the requested pin. + assert len(df) == 1 + assert df.iloc[0]["source"] == "iem" + assert df.attrs.get("source") == "iem.archive" + assert df.attrs.get("accepted_sources") == ["iem", "iem.archive"] + + +def test_research_by_source_as_dataframe_false_still_returns_list(tmp_path, monkeypatch) -> None: + """Deprecated as_dataframe=False maps to a raw list[dict] (return_type=list).""" + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + research_module = importlib.import_module("mostlyright.research") + monkeypatch.setattr( + research_module, + "_fetch_observations_range", + lambda *a, **kw: [ + { + "station_code": "KNYC", + "observed_at": "2025-01-06T12:00:00+00:00", + "source": "iem", + "temp_c": 5.0, + } + ], + ) + monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) + + from mostlyright.mode2 import research_by_source + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + out = research_by_source( + "KNYC", "iem", "2025-01-06", "2025-01-12", as_dataframe=False + ) + + assert isinstance(out, list) + assert out and out[0]["source"] == "iem" + # Both the call-time deprecation AND the as_dataframe deprecation fire. + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + assert any("as_dataframe= is deprecated" in m for m in msgs) diff --git a/tests/test_parity.py b/tests/test_parity.py index e7cdf3ba..eb2c8e50 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -171,6 +171,11 @@ def test_parity_case(case_num: int, station: str, frm: str, to: str) -> None: expected = pd.read_parquet(FIXTURES / f"case_{case_num}_{station}_{frm}_{to}.parquet") actual = mostlyright.research(station, frm, to) + # Phase 30 (30-03) alias lock: dataset() must equal research() on a parity + # case — the D-02 rename may never drift the byte-equivalent output. + actual_dataset = mostlyright.dataset(station, frm, to) + assert_frame_equal(_canon(actual_dataset), _canon(actual), check_dtype=True) + actual_c = _canon(actual) expected_c = _canon(expected) @@ -210,3 +215,15 @@ def test_dtypes_match_ground_truth() -> None: f"expected_dtypes.json is stale for case_{case_num}; " "re-run scripts/capture_expected_dtypes.py" ) + + +def test_dataset_is_research_alias() -> None: + """Phase 30 (30-03) D-02: ``dataset`` and ``research`` are one callable. + + Network-free lock (runs in the fast suite): identity guarantees the + ``dataset()`` alias produces byte-identical output to ``research()`` on + every parity case without a live fetch. + """ + assert mostlyright.dataset is mostlyright.research + assert "dataset" in mostlyright.__all__ + assert "research" in mostlyright.__all__ From cad10589029441bfaf43093fe08670f00202d2cf Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:15:35 +0200 Subject: [PATCH 18/52] style(30-03): apply ruff format to touched Python files - ruff format normalization of line-wrapping in mode2/research/obs + new test --- packages/core/src/mostlyright/mode2.py | 4 +--- packages/core/src/mostlyright/research.py | 4 +--- packages/core/tests/test_mode2_deprecation.py | 12 +++++++----- packages/weather/src/mostlyright/weather/obs.py | 4 +--- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index b69ac5a5..e067f2eb 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -202,9 +202,7 @@ def _research_by_source( # Phase 30 D-06: fold the deprecated `as_dataframe` bool into `return_type`. # Effective "list" preserves the historic `as_dataframe=False` raw-rows path. - return_type = resolve_return_type( - as_dataframe, return_type, default_return_type="dataframe" - ) + return_type = resolve_return_type(as_dataframe, return_type, default_return_type="dataframe") _as_dataframe = return_type != "list" validate_backend_kwargs(backend, return_type) # type: ignore[arg-type] diff --git a/packages/core/src/mostlyright/research.py b/packages/core/src/mostlyright/research.py index f81893af..b128a47b 100644 --- a/packages/core/src/mostlyright/research.py +++ b/packages/core/src/mostlyright/research.py @@ -1704,9 +1704,7 @@ def dataset( validate_backend_kwargs, ) - return_type = resolve_return_type( - as_dataframe, return_type, default_return_type="dataframe" - ) + return_type = resolve_return_type(as_dataframe, return_type, default_return_type="dataframe") # Phase 6 codex iter-2 P2 fix: validate backend / return_type kwargs # BEFORE any network fetch or cache write. A typo in the new kwargs diff --git a/packages/core/tests/test_mode2_deprecation.py b/packages/core/tests/test_mode2_deprecation.py index 49a506c1..df5b49cf 100644 --- a/packages/core/tests/test_mode2_deprecation.py +++ b/packages/core/tests/test_mode2_deprecation.py @@ -44,7 +44,9 @@ def test_research_by_source_call_emits_one_deprecation_warning(tmp_path, monkeyp dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] call_warnings = [w for w in dep if "research_by_source() is deprecated" in str(w.message)] - assert len(call_warnings) == 1, f"expected exactly one call-time warning, got {len(call_warnings)}" + assert len(call_warnings) == 1, ( + f"expected exactly one call-time warning, got {len(call_warnings)}" + ) assert _NEW_SURFACE in str(call_warnings[0].message) # Migration guardrail (D-12): the message flags the retrain event. assert "RETRAIN" in str(call_warnings[0].message).upper() @@ -109,7 +111,9 @@ def test_research_by_source_output_unchanged(tmp_path, monkeypatch) -> None: "temp_c": 6.0, }, ] - monkeypatch.setattr(research_module, "_fetch_observations_range", lambda *a, **kw: synthetic_obs) + monkeypatch.setattr( + research_module, "_fetch_observations_range", lambda *a, **kw: synthetic_obs + ) monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) from mostlyright.mode2 import research_by_source @@ -148,9 +152,7 @@ def test_research_by_source_as_dataframe_false_still_returns_list(tmp_path, monk with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - out = research_by_source( - "KNYC", "iem", "2025-01-06", "2025-01-12", as_dataframe=False - ) + out = research_by_source("KNYC", "iem", "2025-01-06", "2025-01-12", as_dataframe=False) assert isinstance(out, list) assert out and out[0]["source"] == "iem" diff --git a/packages/weather/src/mostlyright/weather/obs.py b/packages/weather/src/mostlyright/weather/obs.py index 6f7d55ae..17fc5450 100644 --- a/packages/weather/src/mostlyright/weather/obs.py +++ b/packages/weather/src/mostlyright/weather/obs.py @@ -185,9 +185,7 @@ def obs( validate_backend_kwargs, ) - return_type = resolve_return_type( - as_dataframe, return_type, default_return_type="dataframe" - ) + return_type = resolve_return_type(as_dataframe, return_type, default_return_type="dataframe") validate_backend_kwargs(backend, return_type) # type: ignore[arg-type] # Eager ISO validation — fail fast for malformed dates before any network. From b3b384d4ce40917d591d8fbd98e22da44c09a5ac Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:29:42 +0200 Subject: [PATCH 19/52] feat(30-04): export dataset aliasing research in TS meta barrel - dataset is the primary documented name (D-02); research retained as a working alias (same callable, export { research as dataset }) - no DeprecationWarning on research this release, mirroring Python 30-03 - built barrel exports both dataset and research; typecheck green --- packages-ts/meta/src/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages-ts/meta/src/index.ts b/packages-ts/meta/src/index.ts index 279804c7..954fa30b 100644 --- a/packages-ts/meta/src/index.ts +++ b/packages-ts/meta/src/index.ts @@ -20,7 +20,14 @@ export { core, markets, weather }; // ASOS + GHCNh + CLI). Lives here (NOT in @mostlyrightmd/core) so the core // package stays dep-free; the orchestrator pulls in both core + weather. // `PairsRow` is the canonical row shape from @mostlyrightmd/core/internal/pairs. -export { research, type ResearchOptions, type PairsRow } from "./research.js"; +// +// Phase 30 30-04 (D-02): `dataset` is the primary documented name for the +// leakage-safe composed training table; `research` is retained as a fully +// working alias (the SAME callable — `export { research as dataset }` binds +// one implementation to two names). No `DeprecationWarning` on `research` +// this release, mirroring the Python side (30-03 kept `research` warning-free +// this release; the warning arrives one release later, removal at 2.0). +export { research, research as dataset, type ResearchOptions, type PairsRow } from "./research.js"; // TS-W4 Wave 1: Mode 2 source-explicit dispatch (researchBySource + // assertSourceIdentity + Mode2Source const-union). Lives in the meta From 1cdc8c3bf74eb4ee83d90ae33f2e58909dc7ff8c Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:36:43 +0200 Subject: [PATCH 20/52] feat(30-04): add granularity to TS obs() with observation-grain frame identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ObsOptions.granularity?: 'daily'|'observation', default 'observation' (D-03; preserves byte-stable per-report TS behavior — D-12) - granularity='daily' throws documented DataAvailabilityError (not yet ported in TS; CROSS-SDK-SYNC ticket) — never silently returns per-report - obs() return stamps non-enumerable frameSource: 'merged.live_v1' (unpinned) / bare source (pinned), mirroring Python df.attrs['source'] - resolveAutoStrategy(source) routes pinned queries to exact_window (D-07) - weather bundle 25.99 KB (within 26 KB budget) --- packages-ts/meta/src/index.ts | 3 + packages-ts/weather/src/index.ts | 3 + packages-ts/weather/src/obs.ts | 98 ++++++++++++++++++++++++---- packages-ts/weather/src/obs.types.ts | 53 +++++++++++++++ 4 files changed, 146 insertions(+), 11 deletions(-) diff --git a/packages-ts/meta/src/index.ts b/packages-ts/meta/src/index.ts index 954fa30b..115eff54 100644 --- a/packages-ts/meta/src/index.ts +++ b/packages-ts/meta/src/index.ts @@ -111,7 +111,10 @@ export { // surface re-exported via meta. export { obs, + type ObsFrameSource, + type ObsGranularity, type ObsOptions, + type ObsResult, type ObsRow, type ObsSourceFilter, type ObsStrategy, diff --git a/packages-ts/weather/src/index.ts b/packages-ts/weather/src/index.ts index 7ff31976..ea64a3c2 100644 --- a/packages-ts/weather/src/index.ts +++ b/packages-ts/weather/src/index.ts @@ -187,7 +187,10 @@ export type { // strategies; matches Python `tw.weather.obs` signature. export { obs, resolveAutoStrategy } from "./obs.js"; export type { + ObsFrameSource, + ObsGranularity, ObsOptions, + ObsResult, ObsRow, ObsSourceFilter, ObsStrategy, diff --git a/packages-ts/weather/src/obs.ts b/packages-ts/weather/src/obs.ts index c127cdb4..4961c0b7 100644 --- a/packages-ts/weather/src/obs.ts +++ b/packages-ts/weather/src/obs.ts @@ -24,7 +24,7 @@ import { fetchAwcMetars } from "./_fetchers/awc.js"; import { downloadIemAsos } from "./_fetchers/iem-asos.js"; import { awcToObservation } from "./_parsers/awc.js"; import { parseIemCsv } from "./_parsers/iem.js"; -import type { ObsOptions, ObsRow, ObsStrategy } from "./obs.types.js"; +import type { ObsFrameSource, ObsOptions, ObsResult, ObsRow, ObsStrategy } from "./obs.types.js"; /** Day count between two ISO YYYY-MM-DD strings (inclusive). */ function daysBetween(fromDate: string, toDate: string): number { @@ -41,8 +41,28 @@ function daysBetween(fromDate: string, toDate: string): number { return Math.round((to - from) / (24 * 60 * 60 * 1000)) + 1; } -/** Resolve `auto` to the concrete strategy chosen by the smart-router. */ -export function resolveAutoStrategy(fromDate: string, toDate: string): ObsStrategy { +/** + * Resolve `auto` to the concrete strategy chosen by the smart-router. + * + * Phase 30 30-04 (D-07): `"auto"` picks the right tactic itself so ordinary + * callers never type `strategy=`. A PINNED query (`source` set) always routes + * to the source-isolated exact-window path — pinned callers want full + * single-source coverage for the exact window they asked for, never a + * year-padded warm-cache span. Unpinned queries keep the window-size + * heuristic (≤7 days → exact_window, else warm_cache). + * + * @param source when set (pinned), forces `exact_window` regardless of window + * size. `null`/`undefined` (unpinned) uses the size heuristic. + */ +export function resolveAutoStrategy( + fromDate: string, + toDate: string, + source?: ObsOptions["source"], +): ObsStrategy { + // D-07: pinned queries auto-route to the source-isolated exact-window path. + if (source !== null && source !== undefined) { + return "exact_window"; + } // Python heuristic: 7-day or smaller windows route to exact_window // (one-off cold fetch ≤ 2 MB estimated payload); larger windows route // to warm_cache (year-aligned cache layout, reusable across calls). @@ -190,29 +210,72 @@ async function fetchByStrategy( return results.flat(); } +/** + * Attach the frame-level `frameSource` tag to the returned row array as a + * non-enumerable, read-only property. Mirrors Python's `df.attrs["source"]` + * without disturbing the rows: iteration, spread, `JSON.stringify`, and + * `Object.keys` over the elements stay byte-identical to the pre-Phase-30 + * output. Introspect via `(result as ObsResult).frameSource`. + */ +function stampFrameSource(rows: ReadonlyArray, frameSource: ObsFrameSource): ObsResult { + Object.defineProperty(rows, "frameSource", { value: frameSource }); + return rows as ObsResult; +} + /** * Fetch raw observations for a station's window. * * Matches Python `tw.weather.obs(station, start, end, source=None, - * strategy='auto')` signature. The strategy enum selects between the - * smart-router's three concrete fetch paths. + * strategy='auto', granularity='observation')` signature. The strategy enum + * selects between the smart-router's three concrete fetch paths. + * + * Phase 30 30-04 (D-03): the returned array carries a non-enumerable + * `frameSource` tag (`"merged.live_v1"` when unpinned, the bare source when + * pinned) mirroring Python's `df.attrs["source"]`. The rows themselves are + * byte-stable — no existing caller breaks. * * @param station ICAO code (e.g. "KNYC") * @param fromDate ISO date `YYYY-MM-DD` (inclusive) * @param toDate ISO date `YYYY-MM-DD` (inclusive) - * @param opts optional source filter + strategy mode - * @returns flat array of ObsRow (each row is a single METAR observation) + * @param opts optional source filter + strategy mode + grain + * @returns {@link ObsResult}: per-report ObsRow array + `frameSource` * @throws DataAvailabilityError when strategy='hosted' (v0.2.x deferral) - * @throws TypeError when strategy is not in the accepted enum + * @throws DataAvailabilityError when granularity='daily' (not yet ported in TS) + * @throws TypeError when strategy or granularity is not in the accepted enum */ export async function obs( station: string, fromDate: string, toDate: string, opts: ObsOptions = {}, -): Promise> { +): Promise { const strategy = opts.strategy ?? "auto"; const source = opts.source ?? null; + // Phase 30 30-04 (D-03): default grain is "observation" — the per-report + // rows TS `obs()` has always returned. This preserves byte-stability for + // every existing no-option caller (D-12). + const granularity = opts.granularity ?? "observation"; + + if (granularity === "daily") { + // "daily" aggregation is the NEW grain TS currently lacks. Per the + // grain-reconciliation decision (30-04-PLAN): a no-option call NEVER + // reaches here (default is "observation"); an EXPLICIT opt-in throws a + // documented not-yet-ported error rather than silently returning + // per-report rows. The daily aggregation branch ships via a + // CROSS-SDK-SYNC parity ticket. + throw new DataAvailabilityError({ + reason: "model_unavailable", + source: "obs.daily", + hint: + "granularity='daily' not yet ported in TS (CROSS-SDK-SYNC ticket); " + + "use granularity='observation' or dailyExtremes(...).", + }); + } + if (granularity !== "observation") { + throw new TypeError( + `obs: unknown granularity "${String(granularity)}" — expected one of: daily, observation`, + ); + } // Phase 21 21-09 fix-iter-1: reject source values that have no TS fetcher // wiring loudly rather than silently returning [] (codex+ts-architect @@ -245,7 +308,11 @@ export async function obs( let resolved: Exclude; if (strategy === "auto") { - resolved = resolveAutoStrategy(fromDate, toDate) as Exclude; + // D-07: pass `source` so pinned queries auto-route to exact_window. + resolved = resolveAutoStrategy(fromDate, toDate, source) as Exclude< + ObsStrategy, + "auto" | "hosted" + >; } else if (strategy === "exact_window" || strategy === "warm_cache") { resolved = strategy; } else { @@ -254,11 +321,20 @@ export async function obs( ); } - return fetchByStrategy(station, fromDate, toDate, resolved, source); + const rows = await fetchByStrategy(station, fromDate, toDate, resolved, source); + + // Phase 30 30-04 (D-03): stamp frame identity. Unpinned = the fused-policy + // tag "merged.live_v1" (a pinned/single-source schema MUST reject it); + // pinned = the bare source (`awc`/`iem`). `ghcnh` is rejected above. + const frameSource: ObsFrameSource = source === null ? "merged.live_v1" : source; + return stampFrameSource(rows, frameSource); } export type { + ObsFrameSource, + ObsGranularity, ObsOptions, + ObsResult, ObsRow, ObsSourceFilter, ObsStrategy, diff --git a/packages-ts/weather/src/obs.types.ts b/packages-ts/weather/src/obs.types.ts index c6ed230d..3256603f 100644 --- a/packages-ts/weather/src/obs.types.ts +++ b/packages-ts/weather/src/obs.types.ts @@ -16,6 +16,52 @@ /** Strategy enum — matches Python verbatim. */ export type ObsStrategy = "auto" | "exact_window" | "warm_cache" | "hosted"; +/** + * Phase 30 30-04 (D-03) — row-level grain for `obs()`. + * + * LOCKED to `{"daily", "observation"}` — structural row-levels, NOT clock + * cadences (`"hourly"` was ruled out as a misnomer: native cadence is + * per-report incl. SPECI + sub-hourly routines). + * + * **TS default = `"observation"` (DELIBERATE divergence from Python's + * `"daily"` default, D-12).** TS `obs()` today already returns per-report + * rows with no daily aggregation — the OPPOSITE of Python's daily default. + * Mirroring Python's default VALUE would silently break every existing TS + * caller (Rob's browser Kalshi plugin receives per-report rows today). We + * mirror the CONTRACT (the grain axis + value names), not the default value; + * each SDK's default preserves its own prior behavior (documented in + * docs/source-identity.md, 30-05). + * + * - `"observation"` — per-report rows (today's exact behavior), window- + * trimmed to the UTC `[fromDate, toDate]` window. No bucketing/resampling. + * - `"daily"` — daily aggregation. NOT YET PORTED in the TS SDK; selecting + * it explicitly throws a documented `DataAvailabilityError` (see obs.ts). + * A no-option call NEVER reaches this branch (default is `"observation"`). + */ +export type ObsGranularity = "daily" | "observation"; + +/** + * Frame-level source identity carried on the array `obs()` returns. + * + * Python stamps `df.attrs["source"]`; TS has no `.attrs` on plain arrays, so + * the returned array carries a non-enumerable `frameSource` property (see + * {@link ObsResult}). Value mirrors Python: + * - unpinned (`source` omitted / null) → `"merged.live_v1"` (the fused-policy + * frame tag a single-source/pinned schema MUST loudly reject). + * - pinned (`source` set) → the bare pinned source (`"awc"` / `"iem"`). + */ +export type ObsFrameSource = "merged.live_v1" | "awc" | "iem" | "ghcnh"; + +/** + * The array `obs()` returns, augmented with a non-enumerable frame-level + * `frameSource` tag (mirrors Python `df.attrs["source"]`). The rows + * themselves are byte-identical to the pre-Phase-30 output — `frameSource` + * is a non-enumerable property, so iteration, `JSON.stringify`, spread, and + * `Object.keys` over the elements are all unchanged. Introspect via + * `(result as ObsResult).frameSource`. + */ +export type ObsResult = ReadonlyArray & { readonly frameSource: ObsFrameSource }; + /** * Source filter — matches Python `tw.weather.obs(source=...)` `_VALID_SOURCES` * frozenset `{"awc", "iem", "ghcnh"}`. `null` (default) means all sources @@ -34,6 +80,13 @@ export interface ObsOptions { source?: ObsSourceFilter; /** Strategy mode; default `"auto"`. */ strategy?: ObsStrategy; + /** + * Phase 30 30-04 (D-03) — row-level grain. Default `"observation"` (the + * per-report rows TS `obs()` has always returned — byte-stable no-option + * behavior). `"daily"` is NOT YET PORTED in TS and throws when passed + * explicitly; see {@link ObsGranularity}. + */ + granularity?: ObsGranularity; } /** From 323f8ae79e42a35959cd3eb3d3773166fa532b2b Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:39:17 +0200 Subject: [PATCH 21/52] feat(30-04): add composed TS climate() domain table via lean subpath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - new climate(station, start, end, opts?) mirrors Python climate() (30-02): composes downloadCliRange/parseCliResponse/mergeClimate, window-trims to [start,end], projects JOIN vocab (cli_high_f/cli_low_f/cli_report_type + date/station/source/issued_at) - frame identity 'cli.archive' (product) via non-enumerable frameSource; per-row source 'iem' (endpoint) — mirrors 30-02 frame-vs-row split - source accept-set {null,'iem','cli','cli.archive'}; 'acis' throws - surfaced via @mostlyrightmd/weather/climate subpath (NOT root barrel) to stay within the 26 KB TS-BUNDLE-01 budget (barrel would exceed by 274 B); mirrors live/forecasts/hosted lean-subpath pattern - browser-safe fetch+JSON, no Node APIs; weather barrel 25.96 KB --- packages-ts/weather/package.json | 5 + packages-ts/weather/src/climate.ts | 129 +++++++++++++++++++++++ packages-ts/weather/src/climate.types.ts | 75 +++++++++++++ packages-ts/weather/src/index.ts | 18 ++++ packages-ts/weather/tsup.config.ts | 11 +- 5 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 packages-ts/weather/src/climate.ts create mode 100644 packages-ts/weather/src/climate.types.ts diff --git a/packages-ts/weather/package.json b/packages-ts/weather/package.json index f21108da..a7f693a5 100644 --- a/packages-ts/weather/package.json +++ b/packages-ts/weather/package.json @@ -54,6 +54,11 @@ "import": "./dist/forecasts/index.mjs", "require": "./dist/forecasts/index.cjs" }, + "./climate": { + "types": "./dist/climate.d.ts", + "import": "./dist/climate.mjs", + "require": "./dist/climate.cjs" + }, "./hosted": { "types": "./dist/hosted/index.d.ts", "import": "./dist/hosted/index.mjs", diff --git a/packages-ts/weather/src/climate.ts b/packages-ts/weather/src/climate.ts new file mode 100644 index 00000000..ed6c62b0 --- /dev/null +++ b/packages-ts/weather/src/climate.ts @@ -0,0 +1,129 @@ +// Phase 30 30-04 — climate(station, start, end, opts?) public function. +// +// TS parity for the Python `climate()` standalone domain table (30-02): +// exposes the NWS CLI daily settlement labels (the Kalshi NHIGH/NLOW +// settlement values) as a source-identified domain table, composing the +// existing building blocks — `downloadCliRange` (year-granular fetch), +// `parseCliResponse` (raw → ClimateObservation), and `mergeClimate` (the +// frozen `(station, date)` dedup, D-15). Browser-safe: fetch + JSON, no +// Node APIs. +// +// Mirrors Python 30-02: +// - window-trim the year-granular fetch to [start, end] at the list level +// (guards the whole-year-leak class); +// - project raw parser keys onto the JOIN vocab +// (high_temp_f→cli_high_f, low_temp_f→cli_low_f, report_type→ +// cli_report_type, observation_date→date, + station, source, issued_at); +// - stamp frame identity `"cli.archive"` (product) while the per-row +// `source` stays the truthful endpoint tag `"iem"`. + +import { downloadCliRange } from "./_fetchers/iem-cli.js"; +import { mergeClimate, parseCliResponse } from "./_parsers/cli.js"; +import type { + ClimateFrameSource, + ClimateOptions, + ClimateResult, + ClimateRow, +} from "./climate.types.js"; + +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; + +/** Accepted `source` values — mirrors Python `{None,"iem","cli","cli.archive"}`. */ +const VALID_CLIMATE_SOURCES: ReadonlySet = new Set(["iem", "cli", "cli.archive"]); + +function validateDate(label: string, value: string): void { + if (!DATE_RE.test(value)) { + throw new Error(`${label} must be YYYY-MM-DD, got ${JSON.stringify(value)}`); + } +} + +function yearOf(isoDate: string): number { + return Number.parseInt(isoDate.slice(0, 4), 10); +} + +/** + * Attach the frame-level `frameSource` tag (`"cli.archive"`) as a + * non-enumerable property (mirrors Python `df.attrs["source"]`). Rows stay + * byte-stable under iteration / spread / JSON. + */ +function stampFrameSource(rows: ReadonlyArray): ClimateResult { + const frameSource: ClimateFrameSource = "cli.archive"; + Object.defineProperty(rows, "frameSource", { value: frameSource }); + return rows as ClimateResult; +} + +/** + * Fetch the NWS CLI daily settlement labels for a station's window. + * + * Mirrors Python `mostlyright.weather.climate(station, start, end, *, + * source=None, ...)`. Composes the local CLI fetch path (year-granular + * download → parse → frozen `mergeClimate` dedup), window-trims to + * `[start, end]`, and projects onto the public JOIN vocabulary. + * + * @param station ICAO code (e.g. "KNYC") — IEM CLI expects a 4-letter ICAO. + * @param start Inclusive start date, ISO `YYYY-MM-DD`. + * @param end Inclusive end date, ISO `YYYY-MM-DD`. + * @param opts See {@link ClimateOptions}. + * @returns {@link ClimateResult}: projected ClimateRow[] + `frameSource`. + * @throws Error on malformed dates, `start > end`, or `source="acis"` + * (no ACIS leg in this code path). + */ +export async function climate( + station: string, + start: string, + end: string, + opts: ClimateOptions = {}, +): Promise { + validateDate("start", start); + validateDate("end", end); + if (start > end) { + throw new Error(`start (${start}) must be <= end (${end})`); + } + + const source = opts.source ?? null; + if (source !== null && !VALID_CLIMATE_SOURCES.has(source)) { + throw new Error( + `climate(): source must be one of null, "iem", "cli", "cli.archive"; got ${JSON.stringify(source)}`, + ); + } + + const stationCode = station.trim().toUpperCase(); + + // Year-granular fetch (IEM cli.py has no partial-year request); trim later. + const rangeOpts: { politenessMs?: number; signal?: AbortSignal } = {}; + if (opts.politenessMs !== undefined) rangeOpts.politenessMs = opts.politenessMs; + if (opts.signal !== undefined) rangeOpts.signal = opts.signal; + const raw = await downloadCliRange(stationCode, yearOf(start), yearOf(end), rangeOpts); + + const parsed = parseCliResponse(raw, stationCode); + // Frozen `(station, date)` dedup — keeps the settlement `final` (D-15). + const deduped = mergeClimate(parsed); + + const rows: ClimateRow[] = []; + for (const r of deduped) { + // Window-trim at the list level — the year-granular fetch pulls whole + // calendar years; keep only [start, end] inclusive (guards whole-year leak). + if (r.observation_date < start || r.observation_date > end) continue; + rows.push({ + date: r.observation_date, + station: r.station_code, + cli_high_f: r.high_temp_f, + cli_low_f: r.low_temp_f, + cli_report_type: r.report_type, + source: "iem", + issued_at: r.issued_at, + }); + } + // Sort by date for a stable settlement-day ordering (matches Python). + rows.sort((a, b) => (a.date < b.date ? -1 : a.date > b.date ? 1 : 0)); + + return stampFrameSource(rows); +} + +export type { + ClimateFrameSource, + ClimateOptions, + ClimateResult, + ClimateRow, + ClimateSourceFilter, +} from "./climate.types.js"; diff --git a/packages-ts/weather/src/climate.types.ts b/packages-ts/weather/src/climate.types.ts new file mode 100644 index 00000000..02409b94 --- /dev/null +++ b/packages-ts/weather/src/climate.types.ts @@ -0,0 +1,75 @@ +// Phase 30 30-04 — climate(station, start, end, opts?) type contract. +// +// TS parity for the Python `climate()` standalone domain table (30-02): +// the NWS CLI daily settlement labels (the Kalshi NHIGH/NLOW settlement +// values), previously visible ONLY through the research()/dataset() join. +// Mirrors the Python surface so cross-language code reads the same way. +// +// Column vocabulary = the JOIN vocab (`cli_high_f`/`cli_low_f`/ +// `cli_report_type`) — no fourth CLI vocabulary is introduced. Frame +// identity is the settlement PRODUCT tag `"cli.archive"`; the per-row +// `source` is the fetch ENDPOINT tag `"iem"` (mirrors 30-02's frame-vs-row +// split, same shape as obs()'s `frameSource`). + +/** + * Source filter for `climate()`. Mirrors the Python accept-set + * `{None, "iem", "cli", "cli.archive"}`. `null`/`undefined` (default) == + * `"iem"`. `"acis"` is REJECTED (no ACIS leg in this code path) — matching + * Python's `ValueError`. + */ +export type ClimateSourceFilter = "iem" | "cli" | "cli.archive" | null; + +/** + * Frame-level source identity carried on the array `climate()` returns. + * Always the settlement PRODUCT tag `"cli.archive"` — distinct from the + * per-row endpoint tag `"iem"`. Mirrors Python `df.attrs["source"]`. + */ +export type ClimateFrameSource = "cli.archive"; + +export interface ClimateOptions { + /** + * Provenance filter. `null`/`undefined` (default) == `"iem"`. Accepts + * `{"iem", "cli", "cli.archive"}`; `"acis"` throws (no ACIS leg here). + */ + source?: ClimateSourceFilter; + /** + * Polite delay (ms) between IEM CLI yearly requests. Forwarded to + * `downloadCliRange`. Default 1000 (`IEM_CLI_POLITE_DELAY_MS`). + */ + politenessMs?: number; + /** Forward to the underlying fetcher; aborts the range download. */ + signal?: AbortSignal; +} + +/** + * Single climate (CLI daily settlement) row. Projects the raw parser keys + * onto the public JOIN vocabulary. `date`/`station`/`cli_high_f`/ + * `cli_low_f`/`cli_report_type` are the settlement contract; `source` + + * `issued_at` carry provenance. + */ +export interface ClimateRow { + /** Local climate day, YYYY-MM-DD (from the parser's `observation_date`). */ + date: string; + /** Station code (as resolved from the caller's input). */ + station: string; + /** Daily high °F (int) or `null`. JOIN vocab — same as PairsRow.cli_high_f. */ + cli_high_f: number | null; + /** Daily low °F (int) or `null`. JOIN vocab — same as PairsRow.cli_low_f. */ + cli_low_f: number | null; + /** Inferred CLI report type (`final`/`preliminary`/...). */ + cli_report_type: string; + /** Per-row endpoint tag — always `"iem"` for CLI. */ + source: "iem"; + /** ISO 8601 UTC issuance time when present, else `null`. */ + issued_at: string | null; +} + +/** + * The array `climate()` returns, augmented with a non-enumerable + * `frameSource` tag (`"cli.archive"`) mirroring Python `df.attrs["source"]`. + * Non-enumerable, so iteration / spread / `JSON.stringify` over the rows are + * unchanged. Introspect via `(result as ClimateResult).frameSource`. + */ +export type ClimateResult = ReadonlyArray & { + readonly frameSource: ClimateFrameSource; +}; diff --git a/packages-ts/weather/src/index.ts b/packages-ts/weather/src/index.ts index ea64a3c2..4ea242f7 100644 --- a/packages-ts/weather/src/index.ts +++ b/packages-ts/weather/src/index.ts @@ -196,6 +196,24 @@ export type { ObsStrategy, } from "./obs.types.js"; +// Phase 30 30-04 — climate(station, start, end, opts?) standalone domain +// table. TS parity for Python `climate()` (30-02): the NWS CLI daily +// settlement labels as a source-identified table, composing the CLI +// building blocks (downloadCliRange/parseCliResponse/mergeClimate). +// +// Deliberately NOT re-exported from this root barrel — surfaced ONLY via the +// LEAN `@mostlyrightmd/weather/climate` subpath, mirroring the +// live/forecasts/hosted subpath pattern. Adding climate() to the root +// barrel exceeds the 26 KB TS-BUNDLE-01 budget (measured +274 B over). The +// subpath keeps the default browser bundle lean while making climate() fully +// importable: +// +// import { climate } from "@mostlyrightmd/weather/climate" +// +// Cross-SDK note: Python exposes climate() from the top-level +// `mostlyright.weather` namespace; the TS subpath is the browser-budget +// adaptation (documented in docs/source-identity.md, 30-05). + // Phase 28 28-40 — TS hosted-fetch shim (MV3-safe; the extension's opt-in path // to the hosted API). Mirrors the Python delivery="hosted" / WEATHER_HOSTED_URL // / EARNINGS_HOSTED_URL seams: fetch + JSON + EventSource, no Node APIs. Also diff --git a/packages-ts/weather/tsup.config.ts b/packages-ts/weather/tsup.config.ts index 9d9fb1ed..31cd8de2 100644 --- a/packages-ts/weather/tsup.config.ts +++ b/packages-ts/weather/tsup.config.ts @@ -6,7 +6,16 @@ export default defineConfig({ // `exports` map (`./live` → `./dist/live/index.{mjs,cjs,d.ts}`). // Phase 17 PLAN-11: same for `forecasts/` subpath. // Phase 28 28-40: same for the MV3 `hosted/` subpath (lean extension bundle). - entry: ["src/index.ts", "src/live/index.ts", "src/forecasts/index.ts", "src/hosted/index.ts"], + // Phase 30 30-04: `src/climate.ts` is a flat subpath entry (→ dist/climate.*) + // so `import { climate } from "@mostlyrightmd/weather/climate"` resolves via + // the exports map WITHOUT adding climate() to the size-budgeted root barrel. + entry: [ + "src/index.ts", + "src/live/index.ts", + "src/forecasts/index.ts", + "src/hosted/index.ts", + "src/climate.ts", + ], format: ["esm", "cjs", "iife"], globalName: "mostlyrightWeather", dts: true, From 847138b70b4d36e7a9029437d79f3e3d56b40d83 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:43:52 +0200 Subject: [PATCH 22/52] feat(30-04): mirror mode2 deprecation + return_type reconcile + vocab purge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rename-with-alias train (D-11/D-02/D-04): SOURCE_PINNED_SOURCES / SourcePinnedSource / isSourcePinnedSource are the neutral names; MODE2_SOURCES / Mode2Source / isMode2Source kept as deprecated aliases (same values) — no consumer break, removal at 2.0 - return_type accepts 'dataframe' (canonical, D-06) + keeps 'frame' alias + 'wrapper' no-op; both type-check and validate - runtime string purge (D-11): 'Mode 2 source must be one of' -> 'source must be one of'; 'Mode 2 dispatch requested' -> 'source-pinned dispatch requested' (lockstep with Python 30-03-2/30-03-5) - 'Mode 1/Mode 2' vocab scrubbed from mode2.ts docstrings/comments - mode2.test.ts:150 assertion updated in lockstep; barrel exports new names --- packages-ts/meta/src/index.ts | 8 ++ packages-ts/meta/src/mode2.ts | 112 +++++++++++++++++-------- packages-ts/meta/src/research.types.ts | 13 +-- packages-ts/meta/tests/mode2.test.ts | 5 +- 4 files changed, 95 insertions(+), 43 deletions(-) diff --git a/packages-ts/meta/src/index.ts b/packages-ts/meta/src/index.ts index 115eff54..f21308b2 100644 --- a/packages-ts/meta/src/index.ts +++ b/packages-ts/meta/src/index.ts @@ -41,18 +41,26 @@ export { research, research as dataset, type ResearchOptions, type PairsRow } fr // `research()`'s single-source pin validates against — so callers can // introspect the accepted names without a deep import, mirroring the // narrow-dispatch siblings (SOURCE_ALIASES / isMode2Source). +// Phase 30 30-04 (D-11): `MODE2_SOURCES` / `Mode2Source` / `isMode2Source` +// are renamed to `SOURCE_PINNED_SOURCES` / `SourcePinnedSource` / +// `isSourcePinnedSource` as part of the "Mode 1/Mode 2" vocabulary purge. +// The old identifiers stay exported as deprecated aliases so no consumer +// breaks (removal at 2.0). export { MODE2_SOURCES, RESEARCH_OBSERVATION_SOURCES, RESEARCH_SOURCE_ALIASES, SOURCE_ALIASES, + SOURCE_PINNED_SOURCES, assertSourceIdentity, isMode2Source, isResearchObservationSource, + isSourcePinnedSource, researchBySource, type Mode2Source, type ResearchBySourceOptions, type SourceMismatchRole, + type SourcePinnedSource, } from "./mode2.js"; // Phase 10: composable research() dispatcher + discover() ergonomic diff --git a/packages-ts/meta/src/mode2.ts b/packages-ts/meta/src/mode2.ts index c27b5960..39695fa8 100644 --- a/packages-ts/meta/src/mode2.ts +++ b/packages-ts/meta/src/mode2.ts @@ -1,10 +1,16 @@ -// Mode 2 — source-explicit research() variant. +// Source-pinned research() variant (`researchBySource`). // -// Mirrors packages/core/src/mostlyright/mode2.py. Mode 1 (the existing -// `research()`) merges AWC > IEM > GHCNh; Mode 2 lets the caller pin -// observations to a single named source for source-identified -// training pairs (the workflow Vojtech wanted for backtests that -// need source-identity invariants). +// Mirrors packages/core/src/mostlyright/mode2.py. The default `research()` +// merges AWC > IEM > GHCNh source-blind; `researchBySource` lets the caller +// pin observations to a single named source for source-identified training +// pairs (the workflow Vojtech wanted for backtests that need source-identity +// invariants). +// +// Phase 30 30-04 (D-04/D-11): the Python `mostlyright.mode2` module is +// deprecated in favor of `obs(source=..., granularity="observation")`; the +// "Mode 1/Mode 2" vocabulary is purged from all public docs/docstrings/ +// runtime strings. This TS module keeps `researchBySource` working (no +// silent data change) and mirrors the vocabulary purge. // // Lives in @mostlyrightmd/meta (alongside `research()`), NOT in // @mostlyrightmd/core — `assertSourceIdentity` consumes the @@ -12,8 +18,8 @@ // must not depend on (would create a cycle). // // ── Vocabulary ─────────────────────────────────────────────────────── -// TS narrows what Python widens: at the input boundary, TS accepts -// ONLY the four canonical dotted-form sources. Bare forms (`iem`, +// TS narrows what Python widens: at the input boundary, `researchBySource` +// accepts ONLY the four canonical dotted-form sources. Bare forms (`iem`, // `awc`, `ghcnh`) are NEVER accepted at the API; they only ever // appear as parser-emitted PER-ROW source tags. The alias table // (`SOURCE_ALIASES`) bridges the boundary: filter rows whose bare @@ -40,15 +46,39 @@ import { export type { SourceMismatchRole }; -/** Mode 2 canonical source vocabulary. Exactly four dotted values. */ -export const MODE2_SOURCES = ["iem.archive", "iem.live", "awc.live", "ghcnh.archive"] as const; +/** + * Source-pinned dispatch vocabulary. Exactly four dotted values. + * + * Phase 30 30-04 (D-11/D-02/D-04): renamed from `MODE2_SOURCES` as part of + * the "Mode 1/Mode 2" vocabulary purge. `MODE2_SOURCES` remains exported as + * a deprecated alias (same tuple) so no consumer breaks; removal at 2.0. + */ +export const SOURCE_PINNED_SOURCES = [ + "iem.archive", + "iem.live", + "awc.live", + "ghcnh.archive", +] as const; /** - * Mode 2 source-identity type. Const-union derived from the - * `MODE2_SOURCES` tuple-literal (NOT a TS `enum` — `enum` defeats + * Source-identity type for source-pinned dispatch. Const-union derived from + * the `SOURCE_PINNED_SOURCES` tuple-literal (NOT a TS `enum` — `enum` defeats * tree-shaking per TS Architect rubric §5). */ -export type Mode2Source = (typeof MODE2_SOURCES)[number]; +export type SourcePinnedSource = (typeof SOURCE_PINNED_SOURCES)[number]; + +/** + * @deprecated Phase 30 30-04 (D-11): renamed to {@link SOURCE_PINNED_SOURCES} + * as part of the "Mode 1/Mode 2" vocabulary purge. This alias is the SAME + * tuple — kept working so no consumer breaks; scheduled for removal in 2.0. + */ +export const MODE2_SOURCES = SOURCE_PINNED_SOURCES; + +/** + * @deprecated Phase 30 30-04 (D-11): renamed to {@link SourcePinnedSource}. + * Alias retained for back-compat; scheduled for removal in 2.0. + */ +export type Mode2Source = SourcePinnedSource; /** * Map each canonical dotted source to the bare parser-emitted tags @@ -60,8 +90,8 @@ export type Mode2Source = (typeof MODE2_SOURCES)[number]; * * Mirrors packages/core/src/mostlyright/mode2.py:55-63. */ -export const SOURCE_ALIASES: ReadonlyMap> = new Map< - Mode2Source, +export const SOURCE_ALIASES: ReadonlyMap> = new Map< + SourcePinnedSource, ReadonlySet >([ ["iem.archive", new Set(["iem", "iem.archive"])], @@ -71,18 +101,25 @@ export const SOURCE_ALIASES: ReadonlyMap> = new ]); /** - * Type-guard: narrow an unknown value to {@link Mode2Source}. Returns + * Type-guard: narrow an unknown value to {@link SourcePinnedSource}. Returns * true iff `value` is one of the four canonical dotted strings. * Bare-form inputs (`'iem'`, `'awc'`, `'ghcnh'`) return false — TS * narrows what Python widens. */ -export function isMode2Source(value: unknown): value is Mode2Source { - return typeof value === "string" && (MODE2_SOURCES as readonly string[]).includes(value); +export function isSourcePinnedSource(value: unknown): value is SourcePinnedSource { + return typeof value === "string" && (SOURCE_PINNED_SOURCES as readonly string[]).includes(value); } +/** + * @deprecated Phase 30 30-04 (D-11): renamed to {@link isSourcePinnedSource}. + * Alias retained for back-compat; scheduled for removal in 2.0. + */ +export const isMode2Source = isSourcePinnedSource; + // ── research({ source }) filter vocabulary — the WIDENED superset ────────── -// `researchBySource` (Mode 2 dispatch) narrows to the four dotted forms -// because a live-IEM fetcher gap means bare/live inputs can't be dispatched. +// `researchBySource` (source-pinned dispatch) narrows to the four dotted +// forms because a live-IEM fetcher gap means bare/live inputs can't be +// dispatched. // `research({ source })` is a FILTER over already-fetched, already-merged // rows, so it accepts Python's full vocabulary. The accepted names and // alias-accept sets below are byte-identical to Python @@ -141,7 +178,7 @@ export function isResearchObservationSource(value: unknown): boolean { * (`'iem'`) are accepted alongside the dotted canonical form * (`'iem.archive'`). Without this, the per-row source-preserved * invariant (Python mode2.py:161-166) would force the assertion - * to fire on every Mode 2 call. + * to fire on every source-pinned call. * * @param rows rows to check (any shape with `source?: string`) * @param expected the source string OR alias-set the caller asked for @@ -178,7 +215,7 @@ export function assertSourceIdentity"; throw new SourceMismatchError( - `Mode 2 dispatch requested '${expectedLabel}' but received ${bad} row(s) with other sources: [${others + `source-pinned dispatch requested '${expectedLabel}' but received ${bad} row(s) with other sources: [${others .map((s) => `'${s}'`) .join(", ")}]`, { @@ -191,12 +228,12 @@ export function assertSourceIdentity { // `observed_at` is a guaranteed non-empty ISO string on Observation, diff --git a/packages-ts/meta/src/research.types.ts b/packages-ts/meta/src/research.types.ts index f65e568e..57f8a0d8 100644 --- a/packages-ts/meta/src/research.types.ts +++ b/packages-ts/meta/src/research.types.ts @@ -59,12 +59,15 @@ export interface ResearchKwargsExtension { backend?: "pandas" | "polars"; /** - * D-03: accepted but no-op in TS. Python returns a `MostlyRightResult` - * wrapper class when `return_type="wrapper"`; TS returns plain object - * arrays (no `.attrs` divergence to bridge), so the wrapper would carry - * no extra signal. + * D-06 output-knob reconciliation (Phase 30 30-04): the canonical value is + * `"dataframe"` (matches Python's winning `backend`+`return_type` + * convention). `"frame"` is retained as a back-compat alias — both are + * accepted and validate. `"wrapper"` remains a no-op in TS (Python returns + * a `MostlyRightResult` wrapper when `return_type="wrapper"`; TS returns + * plain object arrays with no `.attrs` divergence to bridge, so the wrapper + * carries no extra signal). Documented in docs/source-identity.md (30-05). */ - return_type?: "frame" | "wrapper"; + return_type?: "dataframe" | "frame" | "wrapper"; } /** diff --git a/packages-ts/meta/tests/mode2.test.ts b/packages-ts/meta/tests/mode2.test.ts index 7d3fb61d..9858080f 100644 --- a/packages-ts/meta/tests/mode2.test.ts +++ b/packages-ts/meta/tests/mode2.test.ts @@ -147,8 +147,11 @@ describe("researchBySource — unknown source rejection", () => { // Cast through unknown to bypass the Mode2Source type at the boundary // (consumers in plain JS / loose TS can pass any string). const bad = "iem" as unknown as Mode2Source; + // Phase 30 30-04 (D-11): "Mode 2" vocabulary purged from the runtime + // string — softened to "source must be one of ...", lockstep with the + // Python 30-03-5 reconcile. await expect(researchBySource("NYC", bad, "2024-06-01", "2024-06-30")).rejects.toThrow( - /Mode 2 source must be one of/, + /source must be one of/, ); // No HTTP call should have fired — guard runs before any fetcher. expect(fetchSpy).not.toHaveBeenCalled(); From 7c85a939866ef59c6e468672623ff5a811c2f0d6 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:46:53 +0200 Subject: [PATCH 23/52] test(30-04): TS parity tests for obs() observation grain + climate() - obs.observation.test.ts (8 tests): no-option default == observation grain byte-stable (D-12); window-trim; frameSource merged.live_v1/pinned + non-enumerable; granularity='daily' throws documented not-yet-ported error - climate.test.ts (11 tests): JOIN-vocab column contract; window-trim; frame 'cli.archive' vs per-row 'iem'; source accept-set + 'acis' throw; input guards - full TS suite green (core 806, weather 421, markets 248, meta 163); pnpm -r typecheck green; pnpm run size within budgets (weather 25.9/26, meta 38.86/40) --- packages-ts/weather/tests/climate.test.ts | 142 ++++++++++++++ .../weather/tests/obs.observation.test.ts | 174 ++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 packages-ts/weather/tests/climate.test.ts create mode 100644 packages-ts/weather/tests/obs.observation.test.ts diff --git a/packages-ts/weather/tests/climate.test.ts b/packages-ts/weather/tests/climate.test.ts new file mode 100644 index 00000000..16e1e3ae --- /dev/null +++ b/packages-ts/weather/tests/climate.test.ts @@ -0,0 +1,142 @@ +// Phase 30 30-04 — climate(station, start, end, opts?) column contract + +// window-trim + frame identity. +// +// TS parity for the Python climate() domain table (30-02). Verifies: +// - the public JOIN-vocab column contract (date, station, cli_high_f, +// cli_low_f, cli_report_type, source, issued_at); +// - window-trim drops out-of-window rows from the year-granular fetch; +// - frame identity "cli.archive" (product) vs per-row source "iem" +// (endpoint); +// - source accept-set {null,"iem","cli","cli.archive"}; "acis" throws; +// - malformed dates / start>end throw before any fetch. +// +// The IEM CLI fetcher is mocked via vi.spyOn (no network). + +import { describe, expect, it, vi } from "vitest"; + +import * as cliFetcher from "../src/_fetchers/iem-cli.js"; +import type { CliRawRecord } from "../src/_fetchers/iem-cli.js"; +import { climate } from "../src/climate.js"; +import type { ClimateResult } from "../src/climate.types.js"; + +// A `final`-typed CLI record: product issued next-day in the 04:00–10:00 UTC +// overnight window → inferReportType returns "final". +function cliRecord(valid: string, high: number, low: number): CliRawRecord { + const y = valid.slice(0, 4); + const m = valid.slice(5, 7); + const dayNum = Number.parseInt(valid.slice(8, 10), 10); + const nextDay = String(dayNum + 1).padStart(2, "0"); + return { + valid, + high, + low, + product: `${y}${m}${nextDay}0620-KOKX-CDUS41-CLINYC`, + }; +} + +function mockCliRange(records: CliRawRecord[]): void { + vi.spyOn(cliFetcher, "downloadCliRange").mockResolvedValue(records); +} + +describe("climate — column contract", () => { + it("returns the exact public JOIN-vocab columns", async () => { + mockCliRange([cliRecord("2026-01-02", 40, 25)]); + const out = await climate("KNYC", "2026-01-01", "2026-01-03"); + expect(out.length).toBe(1); + const row = out[0]; + if (row === undefined) throw new Error("expected a row"); + expect(Object.keys(row).sort()).toEqual( + [ + "cli_high_f", + "cli_low_f", + "cli_report_type", + "date", + "issued_at", + "source", + "station", + ].sort(), + ); + expect(row.date).toBe("2026-01-02"); + expect(row.station).toBe("KNYC"); + expect(row.cli_high_f).toBe(40); + expect(row.cli_low_f).toBe(25); + expect(row.cli_report_type).toBe("final"); + expect(row.source).toBe("iem"); + expect(row.issued_at).toBe("2026-01-03T06:20:00Z"); + }); +}); + +describe("climate — window-trim", () => { + it("drops rows outside [start, end] from the year-granular fetch", async () => { + // The fetcher returns a whole year; climate() must trim to the window. + mockCliRange([ + cliRecord("2025-12-31", 30, 15), // before window → DROP + cliRecord("2026-01-01", 38, 20), // start → KEEP + cliRecord("2026-01-03", 42, 24), // end → KEEP + cliRecord("2026-01-04", 44, 26), // after window → DROP + ]); + const out = await climate("KNYC", "2026-01-01", "2026-01-03"); + const dates = out.map((r) => r.date); + expect(dates).toEqual(["2026-01-01", "2026-01-03"]); + }); + + it("sorts rows by date", async () => { + mockCliRange([ + cliRecord("2026-01-03", 42, 24), + cliRecord("2026-01-01", 38, 20), + cliRecord("2026-01-02", 40, 22), + ]); + const out = await climate("KNYC", "2026-01-01", "2026-01-03"); + expect(out.map((r) => r.date)).toEqual(["2026-01-01", "2026-01-02", "2026-01-03"]); + }); +}); + +describe("climate — frame vs row source identity", () => { + it("frame tag 'cli.archive' (non-enumerable) distinct from per-row 'iem'", async () => { + mockCliRange([cliRecord("2026-01-02", 40, 25)]); + const out = (await climate("KNYC", "2026-01-01", "2026-01-03")) as ClimateResult; + expect(out.frameSource).toBe("cli.archive"); + expect(Object.keys(out)).not.toContain("frameSource"); + expect(Object.prototype.propertyIsEnumerable.call(out, "frameSource")).toBe(false); + for (const r of out) expect(r.source).toBe("iem"); + expect(JSON.stringify(out).includes("frameSource")).toBe(false); + }); +}); + +describe("climate — source accept-set", () => { + it.each(["iem", "cli", "cli.archive"] as const)("accepts source=%s", async (source) => { + mockCliRange([cliRecord("2026-01-02", 40, 25)]); + const out = await climate("KNYC", "2026-01-01", "2026-01-03", { source }); + expect(out.length).toBe(1); + }); + + it("source=null (default) == source='iem'", async () => { + mockCliRange([cliRecord("2026-01-02", 40, 25)]); + const dflt = await climate("KNYC", "2026-01-01", "2026-01-03"); + mockCliRange([cliRecord("2026-01-02", 40, 25)]); + const iem = await climate("KNYC", "2026-01-01", "2026-01-03", { source: "iem" }); + expect(JSON.parse(JSON.stringify(dflt))).toEqual(JSON.parse(JSON.stringify(iem))); + }); + + it("source='acis' throws (no ACIS leg)", async () => { + const spy = vi.spyOn(cliFetcher, "downloadCliRange"); + await expect( + climate("KNYC", "2026-01-01", "2026-01-03", { source: "acis" as never }), + ).rejects.toThrow(/source must be one of/); + expect(spy).not.toHaveBeenCalled(); + }); +}); + +describe("climate — input validation", () => { + it("malformed start throws before any fetch", async () => { + const spy = vi.spyOn(cliFetcher, "downloadCliRange"); + await expect(climate("KNYC", "2026/01/01", "2026-01-03")).rejects.toThrow(/start must be/); + expect(spy).not.toHaveBeenCalled(); + }); + + it("start > end throws before any fetch", async () => { + const spy = vi.spyOn(cliFetcher, "downloadCliRange"); + await expect(climate("KNYC", "2026-01-03", "2026-01-01")).rejects.toThrow(/must be <=/); + expect(spy).not.toHaveBeenCalled(); + }); +}); diff --git a/packages-ts/weather/tests/obs.observation.test.ts b/packages-ts/weather/tests/obs.observation.test.ts new file mode 100644 index 00000000..81feb066 --- /dev/null +++ b/packages-ts/weather/tests/obs.observation.test.ts @@ -0,0 +1,174 @@ +// Phase 30 30-04 — obs() granularity="observation" grain + frame identity. +// +// Verifies the D-03 / D-12 contract: +// - the no-option default returns per-report rows byte-identical to +// granularity="observation" (proving no existing caller breaks); +// - granularity="observation" returns per-report rows trimmed to the +// [fromDate, toDate] window; +// - the returned array carries a non-enumerable frameSource tag +// ("merged.live_v1" unpinned / bare source pinned) mirroring Python +// df.attrs["source"], without disturbing row iteration; +// - granularity="daily" throws the documented not-yet-ported error rather +// than silently returning per-report rows. +// +// All HTTP is mocked via vi.spyOn on the IEM ASOS fetcher (no network). + +import { describe, expect, it, vi } from "vitest"; + +import { DataAvailabilityError } from "@mostlyrightmd/core"; + +import * as awcFetcher from "../src/_fetchers/awc.js"; +import * as iemFetcher from "../src/_fetchers/iem-asos.js"; +import type { IemChunkResult } from "../src/_fetchers/iem-asos.js"; +import { obs } from "../src/obs.js"; +import type { ObsResult } from "../src/obs.types.js"; + +const IEM_HEADER = + "station,valid,tmpf,dwpf,drct,sknt,gust,alti,mslp,vsby,skyc1,skyl1,skyc2,skyl2,skyc3,skyl3,skyc4,skyl4,wxcodes,p01i,snowdepth,peak_wind_gust,peak_wind_drct,peak_wind_time,metar"; + +function makeIemRow(valid: string, tmpf: string): string { + const defaults: Record = { + station: "NYC", + valid, + tmpf, + dwpf: "55", + drct: "180", + sknt: "8", + gust: "", + alti: "29.92", + mslp: "1013.2", + vsby: "10", + skyc1: "FEW", + skyl1: "2500", + skyc2: "", + skyl2: "", + skyc3: "", + skyl3: "", + skyc4: "", + skyl4: "", + wxcodes: "", + p01i: "", + snowdepth: "", + peak_wind_gust: "", + peak_wind_drct: "", + peak_wind_time: "", + metar: `KNYC ${valid.slice(8, 10)}1251Z 18008KT 10SM FEW025 24/13 A2992 RMK AO2`, + }; + return IEM_HEADER.split(",") + .map((c) => defaults[c] ?? "") + .join(","); +} + +function iemChunk(rows: string[]): IemChunkResult { + return { csv: `${[IEM_HEADER, ...rows].join("\n")}\n` } as IemChunkResult; +} + +function mockIem(rows: string[]): void { + vi.spyOn(iemFetcher, "downloadIemAsos").mockResolvedValue([iemChunk(rows)]); +} + +describe("obs — granularity='observation' grain", () => { + it("no-option default == granularity='observation' (byte-stable, D-12)", async () => { + // Snapshot the no-option output and the explicit-observation output; the + // ROWS must be byte-identical (proving no existing TS caller breaks). + const rows = [makeIemRow("2025-01-06 12:51", "40"), makeIemRow("2025-01-07 12:51", "42")]; + mockIem(rows); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockResolvedValue([]); + const dflt = await obs("KNYC", "2025-01-06", "2025-01-07", { source: "iem" }); + + mockIem(rows); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockResolvedValue([]); + const explicit = await obs("KNYC", "2025-01-06", "2025-01-07", { + source: "iem", + granularity: "observation", + }); + + // JSON compares the enumerable rows only (frameSource is non-enumerable). + expect(JSON.parse(JSON.stringify(dflt))).toEqual(JSON.parse(JSON.stringify(explicit))); + expect(dflt.length).toBeGreaterThan(0); + }); + + it("returns per-report rows trimmed to [fromDate, toDate]", async () => { + mockIem([ + makeIemRow("2025-01-05 12:51", "38"), // before window → DROP + makeIemRow("2025-01-06 12:51", "40"), // fromDate → KEEP + makeIemRow("2025-01-08 12:51", "44"), // toDate → KEEP + makeIemRow("2025-01-09 12:51", "46"), // after window → DROP + ]); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockResolvedValue([]); + const out = await obs("KNYC", "2025-01-06", "2025-01-08", { + source: "iem", + granularity: "observation", + }); + const days = new Set(out.map((r) => r.observed_at.slice(0, 10))); + expect(days.has("2025-01-06")).toBe(true); + expect(days.has("2025-01-08")).toBe(true); + expect(days.has("2025-01-05")).toBe(false); + expect(days.has("2025-01-09")).toBe(false); + // Per-report rows preserve the bare parser source tag. + for (const r of out) expect(r.source).toBe("iem"); + }); + + it("stamps frameSource='merged.live_v1' when unpinned (source=null)", async () => { + mockIem([makeIemRow("2025-01-06 12:51", "40")]); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockResolvedValue([]); + const out = (await obs("KNYC", "2025-01-06", "2025-01-06", { + granularity: "observation", + })) as ObsResult; + expect(out.frameSource).toBe("merged.live_v1"); + }); + + it("stamps frameSource= when pinned", async () => { + mockIem([makeIemRow("2025-01-06 12:51", "40")]); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockResolvedValue([]); + const out = (await obs("KNYC", "2025-01-06", "2025-01-06", { + source: "iem", + granularity: "observation", + })) as ObsResult; + expect(out.frameSource).toBe("iem"); + }); + + it("frameSource is non-enumerable (rows iterate byte-stably)", async () => { + mockIem([makeIemRow("2025-01-06 12:51", "40")]); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockResolvedValue([]); + const out = (await obs("KNYC", "2025-01-06", "2025-01-06", { source: "iem" })) as ObsResult; + // frameSource must NOT show up as an own-enumerable key. + expect(Object.keys(out)).not.toContain("frameSource"); + expect(Object.prototype.propertyIsEnumerable.call(out, "frameSource")).toBe(false); + // ...but is still readable and JSON.stringify omits it. + expect(out.frameSource).toBe("iem"); + expect(JSON.stringify(out).includes("frameSource")).toBe(false); + }); +}); + +describe("obs — granularity='daily' not yet ported in TS", () => { + it("throws DataAvailabilityError (never silently returns per-report rows)", async () => { + // Should throw BEFORE any fetch — a no-data mock proves nothing was fetched. + const iemSpy = vi.spyOn(iemFetcher, "downloadIemAsos"); + const awcSpy = vi.spyOn(awcFetcher, "fetchAwcMetars"); + await expect( + obs("KNYC", "2025-01-06", "2025-01-08", { granularity: "daily" }), + ).rejects.toBeInstanceOf(DataAvailabilityError); + expect(iemSpy).not.toHaveBeenCalled(); + expect(awcSpy).not.toHaveBeenCalled(); + }); + + it("error hint names the CROSS-SDK-SYNC ticket + points at observation/dailyExtremes", async () => { + try { + await obs("KNYC", "2025-01-06", "2025-01-08", { granularity: "daily" }); + throw new Error("expected throw"); + } catch (e) { + const err = e as DataAvailabilityError; + expect(err.reason).toBe("model_unavailable"); + expect(err.source).toBe("obs.daily"); + expect(err.hint).toMatch(/CROSS-SDK-SYNC/); + expect(err.hint).toMatch(/observation/); + } + }); + + it("unknown granularity value throws TypeError", async () => { + await expect( + obs("KNYC", "2025-01-06", "2025-01-08", { granularity: "hourly" as never }), + ).rejects.toThrow(TypeError); + }); +}); From 47b829452fe17370a48ab53886ab72fc0d11bfb3 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 00:47:58 +0200 Subject: [PATCH 24/52] docs(30-04): purge stale 'Mode 2' vocab from meta barrel comment - index.ts:32 comment 'Mode 2 source-explicit dispatch' -> 'source-pinned dispatch' + Mode2Source -> SourcePinnedSource (D-11 vocab purge continuation in a plan-touched file) --- packages-ts/meta/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages-ts/meta/src/index.ts b/packages-ts/meta/src/index.ts index f21308b2..22f32882 100644 --- a/packages-ts/meta/src/index.ts +++ b/packages-ts/meta/src/index.ts @@ -29,8 +29,8 @@ export { core, markets, weather }; // this release; the warning arrives one release later, removal at 2.0). export { research, research as dataset, type ResearchOptions, type PairsRow } from "./research.js"; -// TS-W4 Wave 1: Mode 2 source-explicit dispatch (researchBySource + -// assertSourceIdentity + Mode2Source const-union). Lives in the meta +// TS-W4 Wave 1: source-pinned dispatch (researchBySource + +// assertSourceIdentity + SourcePinnedSource const-union). Lives in the meta // package alongside research() — the dispatch needs the @mostlyrightmd/weather // Observation type and assertSourceIdentity consumes it structurally; // @mostlyrightmd/core must NOT depend on weather (cycle). From e907be7babf31bb76feec666271d89b8d8f9726a Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:15:18 +0200 Subject: [PATCH 25/52] feat(30-07): dataset()/research() covariate join seam (D-16 task 1) - add include_satellite/include_cwop keyword-only flags (default False) - new _covariates.py: post-aggregation LEFT-join seam (lives OUTSIDE _internal/merge/) - label columns byte-identical on == off; missing days NaN, never dropped/added - lazy per-vertical imports inside enabled branches (flags-off pays zero import cost) - prefix + collision guards; additive attrs["covariates"] provenance - guard: covariates require the DataFrame return path (fail loud on return_type="list") Co-Authored-By: Claude Fable 5 --- packages/core/src/mostlyright/_covariates.py | 214 +++++++++++++++++++ packages/core/src/mostlyright/research.py | 60 ++++++ 2 files changed, 274 insertions(+) create mode 100644 packages/core/src/mostlyright/_covariates.py diff --git a/packages/core/src/mostlyright/_covariates.py b/packages/core/src/mostlyright/_covariates.py new file mode 100644 index 00000000..7bcd40ce --- /dev/null +++ b/packages/core/src/mostlyright/_covariates.py @@ -0,0 +1,214 @@ +"""Phase 30 D-16 — ``dataset()`` covariate composition seam. + +The post-aggregation column-join layer for ``dataset()``/``research()``. When a +caller opts in with ``include_satellite=True`` / ``include_cwop=True``, this +module LEFT-joins per-settlement-day covariate columns (``sat_*`` / ``cwop_*``) +onto the already-composed label frame by ``(station, LST settlement day)``. + +PARITY FIREWALL (D-01 amended → labels-only, D-16): +- This seam runs ONLY on the composed *result* DataFrame, strictly AFTER the + observation merge + ``max/min`` label aggregation. It never touches the merge + path, ``SOURCE_PRIORITY``, ``live/_sources``, or the ``schema.observation.v1`` + enum — satellite/cwop NEVER become observation SOURCES. It lives OUTSIDE + ``_internal/merge/`` for exactly this reason. +- The LEFT join keeps the composed frame as the left side: row count and row + order are identical to the flags-off output; days without covariate data get + ``NaN`` (never dropped/added rows). The label columns (``date``, ``station``, + ``obs_*``, ``cli_*``, ``fcst_*``) are byte-identical whether the flags are on + or off — this module only ADDS ``sat_*`` / ``cwop_*`` columns. + +Day bucketing is delegated to the per-vertical helpers, which MUST use the +snapshot settlement-window machinery (``settlement_date_for`` / +``settlement_window_utc``) — never UTC-day bucketing. This module aligns the +returned per-day aggregates onto the composed frame's ``date`` index. + +Extras fail LOUD: ``include_satellite=True`` without the ``[satellite]`` extra +(or ``include_cwop=True`` without ``[cwop]``) raises a typed error naming the +missing extra. All weather-extra imports are lazy (function-local), so the +flags-off path pays zero import cost. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import pandas as pd + +__all__ = ["compose_covariates"] + +#: Column prefixes owned by each covariate vertical. Enforced to avoid any +#: collision with the label columns the composer already produced. +_SAT_PREFIX = "sat_" +_CWOP_PREFIX = "cwop_" + + +def _index_dates_iso(result: pd.DataFrame) -> list[str]: + """Return the composed frame's settlement dates as ISO ``YYYY-MM-DD`` strings. + + The composed frame is indexed by ``date`` (a datetime index, one row per LST + settlement day, produced by ``pairs_to_dataframe``). The per-vertical helpers + key their aggregates by the same ISO settlement date, so we normalize the + index to ISO strings once here for the alignment reindex. + """ + import pandas as pd + + idx = result.index + # The composed frame is a DatetimeIndex named "date"; normalize each entry to + # its ISO calendar date. Guard the (degenerate) non-datetime case by casting. + out: list[str] = [] + for value in idx: + ts = value if isinstance(value, pd.Timestamp) else pd.Timestamp(value) + out.append(ts.date().isoformat()) + return out + + +def _align_covariate_frame( + result: pd.DataFrame, + per_day: dict[str, dict[str, Any]], + *, + prefix: str, +) -> pd.DataFrame: + """Build a covariate DataFrame aligned 1:1 to ``result``'s row order. + + ``per_day`` maps an ISO settlement date → a dict of ``{prefix}col -> value``. + Missing days (no covariate coverage) become all-NaN rows so the subsequent + horizontal concat is a pure LEFT join: identical row count + order, missing + days NaN, never dropped/added. Every produced column is asserted to carry the + expected ``prefix`` so a helper cannot leak a bare column into the label set. + """ + import pandas as pd + + dates_iso = _index_dates_iso(result) + + # Discover the covariate column universe (stable order = first-seen across the + # populated days). An empty universe (no covariate rows at all) yields a frame + # with zero columns → the join is a no-op that still preserves row order. + columns: list[str] = [] + for day in dates_iso: + row = per_day.get(day) + if not row: + continue + for col in row: + if col not in columns: + columns.append(col) + + for col in columns: + if not col.startswith(prefix): + raise ValueError( + f"covariate helper produced column {col!r} without the required " + f"{prefix!r} prefix — refusing to join (would risk a label-column " + f"collision)" + ) + + # Build the aligned rows in the composed frame's exact order. + aligned_rows: list[dict[str, Any]] = [] + for day in dates_iso: + row = per_day.get(day) or {} + aligned_rows.append({col: row.get(col) for col in columns}) + + cov = pd.DataFrame(aligned_rows, columns=columns, index=result.index) + return cov + + +def compose_covariates( + result: Any, + *, + station: str, + from_date: str, + to_date: str, + include_satellite: bool, + include_cwop: bool, + tz_override: str | None = None, +) -> Any: + """LEFT-join per-settlement-day covariate columns onto the composed frame. + + Called by ``dataset()`` after the label frame is assembled and ONLY when at + least one ``include_*`` flag is set. Returns the same object (a DataFrame) + with additive ``sat_*`` / ``cwop_*`` columns and a ``attrs["covariates"]`` + provenance dict. The label columns + row order are untouched. + + Args: + result: The composed label DataFrame (indexed by ``date``). MUST be a + DataFrame — the covariate path is unreachable for the ``list`` + return_type (guarded by the caller). + station: The station code the frame was composed for (single-station + path). Threaded to the per-vertical helpers for their own resolution. + from_date: Inclusive LST settlement start (``YYYY-MM-DD``). + to_date: Inclusive LST settlement end (``YYYY-MM-DD``). + include_satellite: When ``True``, attach ``sat_*`` covariates (needs the + ``[satellite]`` extra — raises a typed error if absent). + include_cwop: When ``True``, attach ``cwop_*`` covariates (needs the + ``[cwop]`` extra — raises a typed error if absent). + tz_override: IANA timezone override forwarded to the day-bucketing + machinery for stations outside the built-in registry. + + Returns: + The composed DataFrame with covariate columns joined. + + Raises: + SourceUnavailableError: an ``include_*`` flag is set but its optional + extra is not installed (names the missing extra). + ValueError: a covariate helper produced a column that collides with an + existing frame column, or lacked its required prefix. + """ + import contextlib + + covariate_provenance: dict[str, str] = {} + existing_columns = set(result.columns) + + if include_satellite: + # Lazy import: the [satellite] extra is heavy (boto3/xarray/…). Importing + # the helper here means the flags-off path never pays for it. + from mostlyright.weather.satellite._covariates import ( + SATELLITE_COVARIATE_SOURCE, + satellite_daily_covariates, + ) + + sat_by_day = satellite_daily_covariates( + station, + from_date, + to_date, + tz_override=tz_override, + ) + cov = _align_covariate_frame(result, sat_by_day, prefix=_SAT_PREFIX) + collision = existing_columns & set(cov.columns) + if collision: + raise ValueError( + f"satellite covariate columns collide with existing columns: {sorted(collision)}" + ) + for col in cov.columns: + result[col] = cov[col] + existing_columns.update(cov.columns) + covariate_provenance["satellite"] = SATELLITE_COVARIATE_SOURCE + + if include_cwop: + from mostlyright.weather.cwop._covariates import ( + CWOP_COVARIATE_SOURCE, + cwop_daily_covariates, + ) + + cwop_by_day = cwop_daily_covariates( + station, + from_date, + to_date, + tz_override=tz_override, + ) + cov = _align_covariate_frame(result, cwop_by_day, prefix=_CWOP_PREFIX) + collision = existing_columns & set(cov.columns) + if collision: + raise ValueError( + f"cwop covariate columns collide with existing columns: {sorted(collision)}" + ) + for col in cov.columns: + result[col] = cov[col] + existing_columns.update(cov.columns) + covariate_provenance["cwop"] = CWOP_COVARIATE_SOURCE + + # Additive provenance: record which covariates were joined + their source + # tags. Existing attrs (source, qc, retrieved_at) are untouched. + if covariate_provenance: + with contextlib.suppress(AttributeError): + result.attrs["covariates"] = covariate_provenance + + return result diff --git a/packages/core/src/mostlyright/research.py b/packages/core/src/mostlyright/research.py index b128a47b..65231c5c 100644 --- a/packages/core/src/mostlyright/research.py +++ b/packages/core/src/mostlyright/research.py @@ -1575,6 +1575,8 @@ def dataset( as_dataframe: bool | None = None, tz_override: str | None = None, qc: bool = False, + include_satellite: bool = False, + include_cwop: bool = False, backend: str = "pandas", return_type: str = "dataframe", ) -> Any: @@ -1639,6 +1641,32 @@ def dataset( mutual-exclusion boundary but the data-selection wiring lands in v0.3 — passing it raises ``NotImplementedError``. Mutually exclusive with ``source=``. + include_satellite: **Phase 30 D-16** — when ``True``, LEFT-join + per-settlement-day satellite covariate columns (``sat_*`` prefix) + onto the composed frame by ``(station, LST settlement day)``. + Requires the ``mostlyrightmd-weather[satellite]`` optional extra; + calling with ``True`` when it is absent raises + :class:`~mostlyright.core.exceptions.SourceUnavailableError` + (never silent empty columns). The LABEL columns + (``obs_*``/``cli_*``/``fcst_*``/``date``/``station``) stay + byte-identical whether this flag is on or off — satellite is a + COVARIATE, never an observation source (the parity firewall is + labels-only, D-01 amended). Days without satellite coverage get + ``NaN``; row count/order are identical to the default output. + Default ``False``. + include_cwop: **Phase 30 D-16** — when ``True``, LEFT-join + per-settlement-day CWOP (PWS) covariate columns (``cwop_`` prefix) + onto the composed frame by ``(station, LST settlement day)``, from + the persisted ``cwop.history()`` cache. Requires the + ``mostlyrightmd-weather[cwop]`` optional extra; calling with + ``True`` when it is absent raises + :class:`~mostlyright.core.exceptions.SourceUnavailableError`. + **QC-clean rows only** (combined QC score > 0.7) contribute — a + hot-rooftop/indoor PWS never touches a label column. **Ledger + left-edge:** CWOP is a capture-or-lose feed (D-10), so settlement + days BEFORE the local capture start have no persisted rows and + come back ``NaN`` after the join (never silently thinned). The + label columns stay byte-identical on == off. Default ``False``. Returns: DataFrame (or ``list[dict]`` when ``as_dataframe=False``) with **one @@ -1995,6 +2023,38 @@ def dataset( with contextlib.suppress(AttributeError): result.attrs["source"] = source + # Phase 30 D-16: covariate composition seam. LEFT-join per-settlement-day + # `sat_*`/`cwop_*` columns onto the composed frame ONLY when a flag is set. + # PARITY FIREWALL (labels-only, D-01 amended): this runs strictly AFTER the + # observation merge + max/min label aggregation above — it never touches the + # merge path, SOURCE_PRIORITY, live sources, or the observation enum, and it + # only ADDS columns (label columns byte-identical on == off). Flags-off is a + # zero-cost no-op: `_covariates` is imported lazily and never runs, so the + # default parity path is untouched and pays no import for the weather extras. + # + # Covariates require the DataFrame path: they are additive COLUMNS, so the + # `return_type="list"` (historic as_dataframe=False) raw-rows shape has no + # place to attach them. Guard loudly rather than silently dropping the flag. + if include_satellite or include_cwop: + if not _as_dataframe: + raise ValueError( + "research(): include_satellite/include_cwop require the DataFrame " + "return path; they add covariate COLUMNS which have no place in the " + 'return_type="list" raw-rows shape. Drop return_type="list" ' + "(or as_dataframe=False) to attach covariates." + ) + from mostlyright._covariates import compose_covariates + + result = compose_covariates( + result, + station=info.code, + from_date=from_date, + to_date=to_date, + include_satellite=include_satellite, + include_cwop=include_cwop, + tz_override=tz_override, + ) + # Effective return_type "list" (or historic as_dataframe=False) → the caller # wants raw list[dict]; backend/return_type wrapping does not apply. Same # when backend is the default (pandas + dataframe) — return unchanged for From dc6a834cbc3320a0fe2a4cf8c5d30f2ca8212e52 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:15:28 +0200 Subject: [PATCH 26/52] feat(30-07): satellite per-settlement-day covariate aggregates (D-16 task 2) - [satellite]-gated helper: reuses the existing extractor + cache (no new transport) - buckets pixels by settlement_date_for (LST settlement day), NEVER UTC-day - sat__mean + sat_scan_count columns; missing days absent -> join NaN - imported lazily from the core seam; extractor raises SourceUnavailableError naming [satellite] when the heavy deps are absent (fail-loud contract) Co-Authored-By: Claude Fable 5 --- .../weather/satellite/_covariates.py | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 packages/weather/src/mostlyright/weather/satellite/_covariates.py diff --git a/packages/weather/src/mostlyright/weather/satellite/_covariates.py b/packages/weather/src/mostlyright/weather/satellite/_covariates.py new file mode 100644 index 00000000..e28f4f49 --- /dev/null +++ b/packages/weather/src/mostlyright/weather/satellite/_covariates.py @@ -0,0 +1,132 @@ +"""Phase 30 D-16 — satellite per-settlement-day covariate aggregates. + +The ``[satellite]``-gated helper that ``dataset(..., include_satellite=True)`` +calls to turn native-L2 single-pixel satellite rows into ONE aggregate row per +LST settlement day, with the ``sat_`` column prefix. It reuses the existing +:func:`mostlyright.weather.satellite.satellite` extractor + cache — no new fetch +transport — and buckets pixels by the SNAPSHOT settlement-window machinery +(``settlement_date_for``), NEVER by UTC calendar day. UTC-day bucketing is the +exact cross-day leakage bug this feature exists to prevent: a pre-midnight-LST +scan whose UTC timestamp rolls into the next UTC date must land in the SAME +settlement day the label row uses. + +Days without satellite coverage are simply ABSENT from the returned mapping; the +core join seam (:mod:`mostlyright._covariates`) turns those into ``NaN`` rows so +row count/order stay identical to the flags-off output. + +This module is imported LAZILY from the core join seam (only when the flag is +set), so a flags-off ``dataset()`` call never pays the ``[satellite]`` import +cost. The extractor itself raises +:class:`~mostlyright.core.exceptions.SourceUnavailableError` naming the +``[satellite]`` extra when its heavy deps are absent — the fail-loud contract. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from datetime import datetime + +#: Source tag recorded in ``df.attrs["covariates"]["satellite"]`` for the join +#: provenance. The default native-ring source is GOES (``noaa_goes``); mixed / +#: auto-routed footprints still surface under this covariate tag (the per-row +#: ``source`` identity stays intact inside the extractor frame). +SATELLITE_COVARIATE_SOURCE = "noaa_goes" + +__all__ = ["SATELLITE_COVARIATE_SOURCE", "satellite_daily_covariates"] + + +def _settlement_window_bounds( + from_date: str, to_date: str, station: str, tz_override: str | None +) -> tuple[datetime, datetime]: + """Return the UTC extraction window covering ``[from_date, to_date]`` LST days. + + The satellite extractor works in event-time UTC, so we widen the fetch to the + UTC span of the first day's settlement-window start through the last day's + settlement-window end. Every pixel is later re-bucketed to its own settlement + day via :func:`settlement_date_for`, so the widening never leaks a scan into + the wrong day — it only ensures the fetch is broad enough to cover the LST + edges (which spill outside the naive UTC calendar range). + """ + from mostlyright.snapshot import settlement_window_utc + + win_start, _ = settlement_window_utc(from_date, station, tz_override=tz_override) + _, win_end = settlement_window_utc(to_date, station, tz_override=tz_override) + return win_start, win_end + + +def satellite_daily_covariates( + station: str, + from_date: str, + to_date: str, + *, + tz_override: str | None = None, +) -> dict[str, dict[str, Any]]: + """Aggregate satellite pixels to one ``sat_*`` row per LST settlement day. + + Args: + station: ICAO/NWS station code (single-station covariate path). + from_date: Inclusive LST settlement start (``YYYY-MM-DD``). + to_date: Inclusive LST settlement end (``YYYY-MM-DD``). + tz_override: IANA timezone override forwarded to the settlement-window + math for stations outside the built-in registry. + + Returns: + ``{settlement_date_iso: {sat_col: value, ...}}``. Days without coverage + are ABSENT (the join turns them into ``NaN``). Columns: + + - ``sat__mean`` — mean ``pixel_value`` over the day's clean + scans, per registered product variable. + - ``sat_scan_count`` — number of scans (rows) contributing to the day. + + Raises: + SourceUnavailableError: the ``[satellite]`` optional extra is absent + (raised by the underlying extractor, naming the extra). + """ + from mostlyright.snapshot import settlement_date_for + from mostlyright.weather.satellite import satellite as _extract + + win_start, win_end = _settlement_window_bounds(from_date, to_date, station, tz_override) + + # Reuse the existing extractor + on-disk cache; no new fetch transport. It + # raises SourceUnavailableError (naming [satellite]) when the heavy deps are + # missing — the fail-loud contract the core seam relies on. + df = _extract(station, start=win_start, end=win_end) + + if df is None or len(df) == 0: + return {} + + # Bucket every pixel by its OWN settlement day (settlement_date_for on the + # event-time scan_start_utc). This is the leakage-safe day assignment: a scan + # whose UTC timestamp rolls past midnight UTC still lands in the LST day the + # label row uses. UTC-day bucketing (scan_start_utc[:10]) is WRONG. + by_day: dict[str, dict[str, list[float]]] = {} + counts: dict[str, int] = {} + for _, row in df.iterrows(): + scan_ts = row.get("scan_start_utc") + if not scan_ts: + continue + settle_day = settlement_date_for(scan_ts, station, tz_override=tz_override) + variable = row.get("variable") + value = row.get("pixel_value") + counts[settle_day] = counts.get(settle_day, 0) + 1 + if variable is None or value is None: + continue + # Skip NaN pixel values (a _FillValue scan) so the mean is over real cells. + try: + fval = float(value) + except (TypeError, ValueError): + continue + if fval != fval: # NaN check without importing math + continue + by_day.setdefault(settle_day, {}).setdefault(str(variable), []).append(fval) + + out: dict[str, dict[str, Any]] = {} + for settle_day, count in counts.items(): + day_cols: dict[str, Any] = {"sat_scan_count": count} + for variable, values in by_day.get(settle_day, {}).items(): + if values: + day_cols[f"sat_{variable}_mean"] = sum(values) / len(values) + out[settle_day] = day_cols + return out From efbbb0624fdc78cd573409f4210ad5792c547311 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:15:38 +0200 Subject: [PATCH 27/52] feat(30-07): cwop per-settlement-day covariate aggregates (D-16 task 3) - [cwop]-gated, READ-ONLY, socket-free: registry.load_stations() radius filter for ICAO->PWS mapping (never scan()/nearby() which open an APRS-IS socket) - QC-clean rows only (qc_status="clean", combined > 0.7) via cwop.history() - buckets by settlement_date_for (LST settlement day), never UTC-day - ledger left-edge (D-10): pre-capture-start days absent -> join NaN, never thinned - cwop_temp_high_qc/low_qc + clean station/obs counts; fail-loud [cwop] extra guard Co-Authored-By: Claude Fable 5 --- .../mostlyright/weather/cwop/_covariates.py | 257 ++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 packages/weather/src/mostlyright/weather/cwop/_covariates.py diff --git a/packages/weather/src/mostlyright/weather/cwop/_covariates.py b/packages/weather/src/mostlyright/weather/cwop/_covariates.py new file mode 100644 index 00000000..4e3b8e34 --- /dev/null +++ b/packages/weather/src/mostlyright/weather/cwop/_covariates.py @@ -0,0 +1,257 @@ +"""Phase 30 D-16 — CWOP per-settlement-day covariate aggregates. + +The ``[cwop]``-gated helper that ``dataset(..., include_cwop=True)`` calls to +turn persisted CWOP (PWS) history into ONE aggregate row per LST settlement day, +with the ``cwop_`` column prefix. + +READ-ONLY, socket-free by construction (three deliberate isolation choices): + +1. Station discovery reads the cached CWOP registry (``stations.json`` via + :func:`mostlyright.weather.cwop._registry.load_stations`) and filters by + great-circle radius around the ICAO's coordinates — it NEVER calls + ``nearby()``/``scan()`` (those open an APRS-IS TCP stream). No network. +2. Observations come from :func:`mostlyright.weather.cwop.history`, which reads + the persisted monthly parquet cache only. +3. QC firewall: ONLY QC-clean rows (combined score > 0.7, i.e. + ``qc_status == "clean"``) contribute. A hot-rooftop or indoor PWS is dropped + before it can touch a covariate column — mirroring why CWOP is barred from the + label ``max/min`` entirely. + +LEDGER LEFT-EDGE (D-10): CWOP is a capture-or-lose feed. Settlement days before +the local capture start have no persisted rows, so they are simply ABSENT from +the returned mapping and the core join turns them into ``NaN`` — never silently +thinned. Days are bucketed by the SNAPSHOT settlement-window machinery +(``settlement_date_for``), never by UTC calendar day. + +Imported LAZILY from the core join seam (only when the flag is set), so a +flags-off ``dataset()`` call never pays the ``[cwop]`` import cost. +""" + +from __future__ import annotations + +import math +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from datetime import date + +#: Source tag recorded in ``df.attrs["covariates"]["cwop"]`` — the persisted +#: cache read path (never a live socket pull), matching ``schema.cwop.v1``'s +#: ``"cwop.cache"`` source identity. +CWOP_COVARIATE_SOURCE = "cwop.cache" + +#: Default great-circle radius (km) for mapping an ICAO to nearby persisted PWS +#: stations. Matches the ``scan()``/``nearby()`` default so the covariate +#: footprint is consistent with the live-discovery footprint. Documented + +#: kwarg-able later (D-16 leaves the exact radius to executor discretion). +_DEFAULT_RADIUS_KM = 25.0 + +#: The QC verdict that gates a row into the covariate aggregate. Combined score +#: > 0.7 (see weather.qc._cwop thresholds) → "clean". Flagged/dropped/unknown +#: rows never contribute. +_QC_CLEAN = "clean" + +__all__ = ["CWOP_COVARIATE_SOURCE", "cwop_daily_covariates"] + + +def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + """Great-circle distance in km (WGS84), mirrors ``_scan._haversine_km``.""" + r = 6371.0088 + p1, p2 = math.radians(lat1), math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlmb = math.radians(lon2 - lon1) + a = math.sin(dphi / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dlmb / 2) ** 2 + return 2 * r * math.asin(min(1.0, math.sqrt(a))) + + +def _icao_coords(station: str) -> tuple[float, float]: + """Resolve an ICAO/NWS code to ``(lat, lon)`` from the SDK registry. + + Read-only; raises ``NoCWOPDataError`` for an unknown code so the fail-loud + contract holds (never a silent empty covariate set from a typo'd station). + """ + from mostlyright.core.exceptions import NoCWOPDataError + from mostlyright.weather.satellite._resolve import _resolve_station_infos + + infos = _resolve_station_infos([station]) + if not infos: + raise NoCWOPDataError(station, "unknown station code (not in the SDK registry)") + info = infos[0] + return float(info.latitude), float(info.longitude) + + +def _nearby_persisted_stations(lat: float, lon: float, radius_km: float) -> list[str]: + """Return cached CWOP station ids within ``radius_km`` (registry read only). + + Uses :func:`mostlyright.weather.cwop._registry.load_stations` — the persisted + ``stations.json`` written by prior ``scan()`` calls — and filters by + haversine radius. NEVER opens an APRS-IS socket (no ``scan()``/``nearby()``). + Positionless registry records (missing lat/lon) are skipped. + """ + from . import _registry + + out: list[str] = [] + for rec in _registry.load_stations().values(): + r_lat = rec.get("lat") + r_lon = rec.get("lon") + sid = rec.get("id") + if r_lat is None or r_lon is None or not sid: + continue + if _haversine_km(lat, lon, float(r_lat), float(r_lon)) <= radius_km: + out.append(str(sid)) + return out + + +def _clean_history_rows(pws_ids: list[str], from_date: date, to_date: date) -> list[dict[str, Any]]: + """Read QC-clean CWOP history rows across the mapped PWS stations. + + Each station is read independently via :func:`cwop.history` with + ``qc_status="clean"`` (persisted parquet only). A station with no clean rows + in-range raises ``NoCWOPDataError``, which is swallowed here — the covariate + view aggregates across whatever clean stations exist, and an entirely-empty + result yields no covariate rows (join → NaN), respecting the ledger edge. + """ + from mostlyright.core.exceptions import NoCWOPDataError + + from ._history import history + + rows: list[dict[str, Any]] = [] + for sid in pws_ids: + try: + df = history(sid, from_date, to_date, qc_status=_QC_CLEAN) + except NoCWOPDataError: + continue + for _, r in df.iterrows(): + rows.append( + { + "station_id": r.get("station_id"), + "observed_at": r.get("observed_at"), + "temp_f": r.get("temp_f"), + } + ) + return rows + + +def cwop_daily_covariates( + station: str, + from_date: str, + to_date: str, + *, + tz_override: str | None = None, + radius_km: float = _DEFAULT_RADIUS_KM, +) -> dict[str, dict[str, Any]]: + """Aggregate QC-clean CWOP history to one ``cwop_*`` row per LST settlement day. + + Args: + station: ICAO/NWS station code (the label frame's station). Mapped to + nearby persisted PWS stations within ``radius_km``. + from_date: Inclusive LST settlement start (``YYYY-MM-DD``). + to_date: Inclusive LST settlement end (``YYYY-MM-DD``). + tz_override: IANA timezone override forwarded to the settlement-window + math for stations outside the built-in registry. + radius_km: Great-circle radius for the ICAO→PWS mapping (default + ``25.0`` km, matching ``scan()``/``nearby()``). + + Returns: + ``{settlement_date_iso: {cwop_col: value, ...}}``. Days with no clean + rows (including every day before the local capture start — the ledger + left-edge, D-10) are ABSENT (the join turns them into ``NaN``). Columns: + + - ``cwop_temp_high_qc`` — max clean-PWS ``temp_f`` over the settlement + day. + - ``cwop_temp_low_qc`` — min clean-PWS ``temp_f`` over the settlement + day. + - ``cwop_clean_station_count`` — distinct clean PWS stations reporting + on the day. + - ``cwop_clean_obs_count`` — clean observation rows contributing. + + Raises: + NoCWOPDataError: the ``station`` code is unknown (fail-loud, never a + silent empty covariate set). + """ + from datetime import date as _date + + from mostlyright.snapshot import settlement_date_for + + # Fail-loud [cwop]-extra guard (D-16): the covariate path builds a + # schema.cwop.v1 DataFrame via history(), which needs pandas — the sole + # [cwop] extra dep. Raise a typed, extra-naming error rather than a bare + # ImportError so the missing-extra contract matches the satellite path. + try: + import pandas # noqa: F401 + except ImportError as exc: + from mostlyright.core.exceptions import SourceUnavailableError + + raise SourceUnavailableError( + "CWOP covariate composition requires the [cwop] optional extra. " + "Install with: pip install mostlyrightmd-weather[cwop]", + source="cwop.cache", + retryable=False, + underlying=str(exc), + ) from None + + lat, lon = _icao_coords(station) + pws_ids = _nearby_persisted_stations(lat, lon, radius_km) + if not pws_ids: + # No persisted PWS near this station → no covariates (join → NaN). + return {} + + frm = _date.fromisoformat(from_date) + to = _date.fromisoformat(to_date) + rows = _clean_history_rows(pws_ids, frm, to) + if not rows: + return {} + + # Bucket by the OWN settlement day of each observation (leakage-safe LST-day + # assignment via the snapshot machinery — NOT UTC-day). Only rows inside a + # requested settlement day contribute to that day. + requested = {d.isoformat() for d in _daterange(frm, to)} + by_day: dict[str, list[float]] = {} + day_stations: dict[str, set[str]] = {} + day_obs: dict[str, int] = {} + for r in rows: + observed_at = r.get("observed_at") + if observed_at is None: + continue + settle_day = settlement_date_for(observed_at, station, tz_override=tz_override) + if settle_day not in requested: + continue + day_obs[settle_day] = day_obs.get(settle_day, 0) + 1 + sid = r.get("station_id") + if sid is not None: + day_stations.setdefault(settle_day, set()).add(str(sid)) + temp = r.get("temp_f") + if temp is None: + continue + try: + ftemp = float(temp) + except (TypeError, ValueError): + continue + if ftemp != ftemp: # NaN + continue + by_day.setdefault(settle_day, []).append(ftemp) + + out: dict[str, dict[str, Any]] = {} + for settle_day, obs_count in day_obs.items(): + cols: dict[str, Any] = { + "cwop_clean_obs_count": obs_count, + "cwop_clean_station_count": len(day_stations.get(settle_day, set())), + } + temps = by_day.get(settle_day) + if temps: + cols["cwop_temp_high_qc"] = max(temps) + cols["cwop_temp_low_qc"] = min(temps) + out[settle_day] = cols + return out + + +def _daterange(frm: date, to: date) -> list[date]: + """Inclusive list of calendar dates ``[frm, to]``.""" + from datetime import timedelta + + out: list[date] = [] + d = frm + while d <= to: + out.append(d) + d = d + timedelta(days=1) + return out From edebea96f7a5671f05eb7d46ae36cc03c0673b8d Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:17:32 +0200 Subject: [PATCH 28/52] test(30-07): covariate tests + labels-only firewall wording (D-16 task 4) - test_dataset_covariates.py (skip-guards extras, no network): label byte-equiv flags on/off, missing-extra typed error, NaN-not-dropped, LST-day bucketing (non-UTC KLAX), QC-clean-only cwop, read-only-registry assertion, firewall grep gate (merge/observations, live/_sources, observation enum carry no cwop/satellite), flags-off import-cost exclusion - CLAUDE.md CWOP section: labels-only firewall wording (four-file list verbatim), D-16 include_cwop covariate-columns rule replacing the blanket no-include claim - docs/satellite.md + docs/cwop-adapter.md: D-16 cross-ref stubs (full docs 30-05) Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 4 +- docs/cwop-adapter.md | 18 +- docs/satellite.md | 13 + .../core/tests/test_dataset_covariates.py | 418 ++++++++++++++++++ 4 files changed, 446 insertions(+), 7 deletions(-) create mode 100644 packages/core/tests/test_dataset_covariates.py diff --git a/CLAUDE.md b/CLAUDE.md index 103b1d5f..81ce78c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,12 +67,12 @@ This applies to every PLAN.md generated by `/gsd-plan-phase`, every PROJECT.md " Standalone live adapter for Citizen Weather Observer Program (PWS) data over the APRS-IS TCP stream. **Deliberately isolated from the parity-critical core** — a hot-rooftop or indoor PWS must never corrupt a Kalshi NHIGH/NLOW settlement target. -- **Never wired into** `research()`, `merge/observations.py` (`SOURCE_PRIORITY`), or `live/_sources.py`. CWOP carries its own `schema.cwop.v1` — `schema.observation.v1`'s `observation_type` enum is untouched. These four files are the parity firewall; do not register "cwop" in any of them. This holds in v0.2: persistence + `history()` add NO wiring into those four files. +- **Labels-only firewall (D-01 amended, D-16).** CWOP must NEVER become an observation SOURCE — never register "cwop" in the observation merge path of `research()`, `merge/observations.py` (`SOURCE_PRIORITY`), `live/_sources.py`, or `schema.observation.v1`'s `observation_type` enum. CWOP carries its own `schema.cwop.v1`. **These four files are the parity firewall; do not register "cwop" in any of them.** The firewall protects the LABEL computation (obs `max/min` + `cli_*` columns) only — a hot-rooftop/indoor PWS can never enter the settlement aggregation. What the amendment ALLOWS: covariate COLUMNS. `dataset(..., include_cwop=True)` (D-16) LEFT-joins per-settlement-day `cwop_*` columns (QC-clean rows only, ledger left-edge respected) onto the composed frame via the post-aggregation seam in `_covariates.py` (which lives OUTSIDE `_internal/merge/` and runs strictly after the label aggregation — it never touches those four files). Label columns stay byte-identical with the flag on or off. This holds in v0.2: persistence + `history()` add NO wiring into those four files. - **Public surface (sync unless noted):** `nearby()` (start-here ICAO→scan), `scan()`, `stream()` (async generator), `snapshot()` (→ `schema.cwop.v1` DataFrame; `persist=True` also writes the backtest cache), `latest()`. v0.2 adds `history(station, from_date, to_date, *, qc_status=None)` (→ `schema.cwop.v1` DataFrame for backtest replay) and `persist_observations(observations)`. Source tag `"cwop.live"` (live) / `"cwop.cache"` (persisted-then-read); `schema.cwop.v1` accepts both via `_registered_sources`. Errors raise `NoCWOPDataError` (subclass of `NoLiveDataError`) — never return `[]`/`None`. - **File layout:** `weather/_aprs.py` (in-house MIT parser, no GPLv2 aprslib), `weather/_fetchers/_aprs_stream.py` (`_CWOPReaderThread` daemon owns socket → `threading.Queue` → sync/async consumers; `loop.call_soon_threadsafe` bridges async), `weather/cwop/*` (public API + `_schema.py` + `_registry.py` + v0.2 `_cache.py`/`_history.py`), `weather/qc/_cwop.py` (QC). Pure stdlib (`socket` + `re`) for the live path; persistence pulls `pyarrow`/`filelock`. `[cwop]` extra pulls pandas (for `snapshot()`/`history()` DataFrames). - **Persistence (v0.2):** monthly parquet at `$HOME/.mostlyright/cache/cwop/{station}/{year}/{month}.parquet` (honors `MOSTLYRIGHT_CACHE_DIR`). Standalone `weather/cwop/_cache.py` — does NOT import the parity-coupled `weather/cache.py`; `filelock`-guarded read-modify-write merge, dedup by `(station_id, observed_at)` first-seen-wins. **NO current-month skip** (CWOP is ephemeral — unlike re-fetchable AWC/IEM/GHCNh, the current month is the only chance to retain it). CWOP station ids (CW0875, ham callsigns) are NOT ICAO, so the cache uses its own `_CWOP_STATION_RE` path validator, not `STATION_CODE_RE`. - **6-layer QC thresholds:** each layer scores 0.0–1.0; combined = weighted geometric mean. `< 0.3` → dropped, `0.3–0.7` → flagged, `> 0.7` → clean. Layers: range (0.20), temporal consistency (0.20), indoor detection (0.15), buddy/ASOS check (0.20), solar-radiation bias (0.10), reliability (0.15). Layers needing history (indoor ≥24h, solar ≥7d) pass (1.0) until enough data accumulates. -- **Why no `include_cwop` on `research()`:** `research()` aggregates `max/min(temp_f)` source-blind, so a hot-rooftop/indoor PWS would corrupt a Kalshi NHIGH/NLOW target. CWOP reaches models ONLY through `cwop.history()` (a DataFrame the strategy joins itself), never the settlement join. See [`docs/cwop-adapter.md`](docs/cwop-adapter.md). +- **`include_cwop` covariate columns (D-16), NOT a settlement source:** `research()`/`dataset()` aggregate `max/min(temp_f)` source-blind for the LABEL, so a hot-rooftop/indoor PWS must never enter that aggregation — CWOP stays out of the four firewall files. But `dataset(..., include_cwop=True)` (D-16) attaches per-settlement-day `cwop_*` COVARIATE columns (QC-clean rows only, combined > 0.7; ledger left-edge → NaN before capture start; read-only, no APRS connections) via the labels-only join seam. The label columns are byte-identical with the flag on or off. CWOP also reaches models through `cwop.history()` (a DataFrame the strategy joins itself); either way it never touches the settlement join. See [`docs/cwop-adapter.md`](docs/cwop-adapter.md). ## Testing diff --git a/docs/cwop-adapter.md b/docs/cwop-adapter.md index 61d3fd56..a0f0f2f7 100644 --- a/docs/cwop-adapter.md +++ b/docs/cwop-adapter.md @@ -85,11 +85,19 @@ clean = cwop.history("CW0875", date(2026, 1, 1), date(2026, 6, 30), qc_status="c needs to keep. - **Source identity:** live pulls are `"cwop.live"`; anything read back from the cache is `"cwop.cache"`. `schema.cwop.v1` accepts both. -- **Why no `include_cwop=True` on `research()`?** `research()` aggregates - `max(temp_f)/min(temp_f)` source-blind, so a single hot-rooftop or indoor PWS - would raise `obs_high_f` / lower `obs_low_f` and corrupt a Kalshi NHIGH/NLOW - training target. CWOP therefore stays on its own `history()` path; the - strategy decides how (and whether) to fold it in. +- **`include_cwop=True` = COVARIATE columns, not a settlement source (D-16).** + `research()`/`dataset()` aggregate `max(temp_f)/min(temp_f)` source-blind for + the LABEL, so a single hot-rooftop or indoor PWS must never enter that + aggregation — CWOP stays out of the four parity-firewall files. But + `dataset(..., include_cwop=True)` (Phase 30 D-16) LEFT-joins per-settlement-day + `cwop_*` COVARIATE columns onto the composed frame: **QC-clean rows only** + (combined > 0.7), read-only from the persisted `history()` cache (no APRS + connections), with the ledger left-edge respected (days before capture start → + `NaN`, never silently thinned). The label columns stay byte-identical with the + flag on or off. See the `dataset()` docstring for the column contract; the + manual `history()` join below is still available and documented. + + > **Cross-reference stub — full covariate docs land in 30-05.** Every failure mode raises `NoCWOPDataError` (a `NoLiveDataError` subclass) with the requested station/area and a `reason` — the API never returns `[]`/`None`. diff --git a/docs/satellite.md b/docs/satellite.md index cad775e1..307c8a58 100644 --- a/docs/satellite.md +++ b/docs/satellite.md @@ -351,6 +351,19 @@ suite are **keyless** (synthetic SEVIRI fixtures + a mocked Data-Store transport CI-excluded `@pytest.mark.live` test. No other source is affected by a missing EUMETSAT key. +## Covariate composition on `dataset()` (D-16) + +> **Cross-reference stub — full docs land in 30-05.** Beyond the manual join, you +> can let the SDK own the leakage-prone LST-day alignment: +> `dataset(..., include_satellite=True)` (Phase 30 D-16) LEFT-joins +> per-settlement-day `sat_*` covariate columns onto the composed training frame +> by `(station, LST settlement day)`. Satellite stays a COVARIATE, never a +> settlement source — the label columns (`obs_*`/`cli_*`) are byte-identical with +> the flag on or off, days without coverage become `NaN` (rows are never +> dropped/added), and pixels are bucketed by the LST settlement window (never UTC +> day). Requires the `[satellite]` extra (fail-loud otherwise). See `dataset()`'s +> docstring for the exact column contract. + ## See also - [`docs/forecasts.md`](forecasts.md) — the NWP forecast path (the load-bearing diff --git a/packages/core/tests/test_dataset_covariates.py b/packages/core/tests/test_dataset_covariates.py new file mode 100644 index 00000000..2225dbb2 --- /dev/null +++ b/packages/core/tests/test_dataset_covariates.py @@ -0,0 +1,418 @@ +"""Phase 30 D-16 — ``dataset()`` covariate composition (include_satellite / include_cwop). + +Network-free unit tests for the covariate join seam (:mod:`mostlyright._covariates`) +and the label-firewall guarantees. The live satellite/cwop fetch paths are NOT +exercised here (they need the optional extras + network); instead the join seam is +tested directly against a synthetic composed frame + synthetic per-day aggregates, +and the ``dataset()`` wiring is tested via monkeypatched helpers. + +The four parity-firewall files are grep-asserted to carry NO new cwop/satellite +source registration vs main. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +pd = pytest.importorskip("pandas") + +from mostlyright._covariates import compose_covariates # noqa: E402 + +# Repo root = three parents up from packages/core/tests/ ... resolve dynamically. +_REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _synthetic_composed_frame() -> pd.DataFrame: + """A minimal composed label frame: 3 LST settlement days, one station. + + Mirrors the shape ``pairs_to_dataframe`` produces (DatetimeIndex named + ``date`` + a ``station`` column + a couple of label columns). + """ + df = pd.DataFrame( + { + "station": ["KNYC", "KNYC", "KNYC"], + "obs_high_f": [40.0, 42.0, 39.0], + "obs_low_f": [30.0, 31.0, 29.0], + "cli_high_f": [41.0, 43.0, 40.0], + "cli_low_f": [29.0, 30.0, 28.0], + }, + index=pd.to_datetime(["2025-01-06", "2025-01-07", "2025-01-08"], format="%Y-%m-%d"), + ) + df.index.name = "date" + return df + + +# -------------------------------------------------------------------------- +# (a) Label byte-equivalence: flags on vs off (every pre-existing column + order) +# -------------------------------------------------------------------------- + + +def test_label_columns_byte_identical_flags_on_vs_off(monkeypatch: pytest.MonkeyPatch) -> None: + """The join only ADDS sat_* columns; label columns + row order are identical.""" + base = _synthetic_composed_frame() + label_cols = list(base.columns) + + # Stub the satellite helper: two of three days have coverage. + def _fake_sat(station, frm, to, *, tz_override=None): + return { + "2025-01-06": {"sat_cloud_frac_mean": 0.5, "sat_scan_count": 12}, + "2025-01-08": {"sat_cloud_frac_mean": 0.2, "sat_scan_count": 10}, + } + + from mostlyright.weather.satellite import _covariates as satmod + + monkeypatch.setattr(satmod, "satellite_daily_covariates", _fake_sat) + + joined = compose_covariates( + base.copy(), + station="KNYC", + from_date="2025-01-06", + to_date="2025-01-08", + include_satellite=True, + include_cwop=False, + ) + + # Row count + order identical. + assert list(joined.index) == list(base.index) + assert len(joined) == len(base) + # Every label column byte-identical. + for col in label_cols: + pd.testing.assert_series_equal(joined[col], base[col], check_names=True) + # Only sat_* columns were added. + added = [c for c in joined.columns if c not in label_cols] + assert added, "expected sat_* columns to be added" + assert all(c.startswith("sat_") for c in added) + # Missing day → NaN, never dropped. + assert pd.isna(joined.loc["2025-01-07", "sat_cloud_frac_mean"]) + # Provenance stamped additively. + assert joined.attrs["covariates"]["satellite"] == satmod.SATELLITE_COVARIATE_SOURCE + + +def test_flags_off_is_noop_identity() -> None: + """compose_covariates is never called on the flags-off path — but if it is + with both flags False, it returns the frame unchanged with no covariates attr.""" + base = _synthetic_composed_frame() + out = compose_covariates( + base.copy(), + station="KNYC", + from_date="2025-01-06", + to_date="2025-01-08", + include_satellite=False, + include_cwop=False, + ) + assert list(out.columns) == list(base.columns) + assert "covariates" not in out.attrs + + +# -------------------------------------------------------------------------- +# (c) Typed error on missing extra +# -------------------------------------------------------------------------- + + +def test_missing_satellite_extra_raises_typed_error() -> None: + """include_satellite=True without [satellite] raises a typed, extra-naming error.""" + pytest.importorskip("pandas") + try: + import boto3 # noqa: F401 + + pytest.skip("[satellite] extra IS installed — cannot test the missing-extra path") + except ImportError: + pass + + from mostlyright.core.exceptions import SourceUnavailableError + + base = _synthetic_composed_frame() + with pytest.raises(SourceUnavailableError) as exc: + compose_covariates( + base.copy(), + station="KNYC", + from_date="2025-01-06", + to_date="2025-01-08", + include_satellite=True, + include_cwop=False, + ) + assert "satellite" in str(exc.value) + + +# -------------------------------------------------------------------------- +# (d) NaN-not-dropped semantics (cwop, via stubbed helper) +# -------------------------------------------------------------------------- + + +def test_cwop_left_edge_days_are_nan_not_dropped(monkeypatch: pytest.MonkeyPatch) -> None: + """Pre-capture-start days come back NaN after the join (ledger left-edge).""" + base = _synthetic_composed_frame() + label_cols = list(base.columns) + + # Only the LAST day has clean CWOP rows (capture started 2025-01-08). + def _fake_cwop(station, frm, to, *, tz_override=None): + return { + "2025-01-08": { + "cwop_temp_high_qc": 41.0, + "cwop_temp_low_qc": 30.0, + "cwop_clean_station_count": 2, + "cwop_clean_obs_count": 40, + } + } + + from mostlyright.weather.cwop import _covariates as cwopmod + + monkeypatch.setattr(cwopmod, "cwop_daily_covariates", _fake_cwop) + + joined = compose_covariates( + base.copy(), + station="KNYC", + from_date="2025-01-06", + to_date="2025-01-08", + include_satellite=False, + include_cwop=True, + ) + + assert len(joined) == len(base) # no rows dropped + for col in label_cols: + pd.testing.assert_series_equal(joined[col], base[col]) + # Left-edge days are NaN, never thinned. + assert pd.isna(joined.loc["2025-01-06", "cwop_temp_high_qc"]) + assert pd.isna(joined.loc["2025-01-07", "cwop_temp_high_qc"]) + assert joined.loc["2025-01-08", "cwop_temp_high_qc"] == 41.0 + assert joined.attrs["covariates"]["cwop"] == cwopmod.CWOP_COVARIATE_SOURCE + + +def test_column_collision_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A covariate column colliding with an existing label column raises loudly.""" + base = _synthetic_composed_frame() + + def _bad_sat(station, frm, to, *, tz_override=None): + # Non-prefixed column that would collide → caught by the prefix guard. + return {"2025-01-06": {"obs_high_f": 99.0}} + + from mostlyright.weather.satellite import _covariates as satmod + + monkeypatch.setattr(satmod, "satellite_daily_covariates", _bad_sat) + + with pytest.raises(ValueError, match="prefix"): + compose_covariates( + base.copy(), + station="KNYC", + from_date="2025-01-06", + to_date="2025-01-08", + include_satellite=True, + include_cwop=False, + ) + + +# -------------------------------------------------------------------------- +# (e) Firewall grep assertions — the four parity-firewall files carry NO new +# cwop/satellite source registration. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "rel_path", + [ + "packages/core/src/mostlyright/_internal/merge/observations.py", + "packages/core/src/mostlyright/live/_sources.py", + ], +) +def test_firewall_files_have_no_cwop_or_satellite(rel_path: str) -> None: + """The merge + live-source firewall files never mention cwop/satellite.""" + path = _REPO_ROOT / rel_path + assert path.exists(), f"firewall file not found: {path}" + text = path.read_text() + lower = text.lower() + assert "cwop" not in lower, f"{rel_path} references cwop — firewall breach" + assert "satellite" not in lower, f"{rel_path} references satellite — firewall breach" + + +def test_covariate_seam_lives_outside_merge() -> None: + """_covariates.py must NOT live under _internal/merge/ (parity firewall).""" + inside_merge = _REPO_ROOT / "packages/core/src/mostlyright/_internal/merge/_covariates.py" + outside = _REPO_ROOT / "packages/core/src/mostlyright/_covariates.py" + assert not inside_merge.exists() + assert outside.exists() + + +def test_observation_enum_has_no_cwop_satellite_member() -> None: + """schema.observation.v1's source enum never gains cwop/satellite.""" + path = _REPO_ROOT / "packages/core/src/mostlyright/core/schemas/observation.py" + if not path.exists(): + pytest.skip("observation schema module not found at expected path") + lower = path.read_text().lower() + assert "cwop" not in lower + # "satellite" here would only matter as an observation source; the schema + # module is observation-only, so any mention is a breach. + assert "satellite" not in lower + + +def test_flags_off_import_cost_excludes_extras() -> None: + """A flags-off `import mostlyright` never pulls the satellite/cwop covariate modules.""" + code = ( + "import sys, mostlyright\n" + "mods = [m for m in sys.modules if 'covariates' in m or 'satellite' in m " + "or m.endswith('cwop') or '.cwop.' in m]\n" + "assert not mods, mods\n" + "print('OK')\n" + ) + result = subprocess.run( + ["python", "-c", code], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "OK" in result.stdout + + +# -------------------------------------------------------------------------- +# Task 30-07-1 — dataset()/research() signature + the DataFrame-only guard. +# -------------------------------------------------------------------------- + + +def test_dataset_signature_has_covariate_kwonly_flags() -> None: + """dataset() (and its research alias) expose include_satellite/include_cwop.""" + import inspect + + import mostlyright + + sig = inspect.signature(mostlyright.dataset) + for name in ("include_satellite", "include_cwop"): + assert name in sig.parameters, name + p = sig.parameters[name] + assert p.kind is inspect.Parameter.KEYWORD_ONLY + assert p.default is False + # research is the same callable → same signature. + assert mostlyright.research is mostlyright.dataset + + +def test_covariate_flags_require_dataframe_return_type(monkeypatch: pytest.MonkeyPatch) -> None: + """include_* with return_type='list' fails loud (no place for covariate columns). + + Exercises the REAL dataset() guard by stubbing every network fetch so no + APIs are hit — the frame is built from empty inputs, then the covariate seam + rejects the list return type. + """ + import sys + + import mostlyright + import mostlyright.research # ensure the submodule is imported into sys.modules + + research_mod = sys.modules["mostlyright.research"] + + # Stub the fetch + assembly internals to keep the call network-free. The + # guard fires after the (empty) result frame is built, before any wrapping. + monkeypatch.setattr(research_mod, "_all_caches_warm", lambda *a, **k: True) + monkeypatch.setattr(research_mod, "_fetch_observations_range", lambda *a, **k: []) + monkeypatch.setattr(research_mod, "_fetch_climate_range", lambda *a, **k: []) + + with pytest.raises(ValueError, match="DataFrame return path"): + mostlyright.dataset( + "KNYC", + "2025-01-06", + "2025-01-08", + include_satellite=True, + return_type="list", + ) + + +# -------------------------------------------------------------------------- +# Task 30-07-2 — satellite helper buckets by LST settlement day, not UTC day +# (test with a non-UTC station: KLAX = UTC-8). +# -------------------------------------------------------------------------- + + +def test_satellite_helper_buckets_by_lst_settlement_day(monkeypatch: pytest.MonkeyPatch) -> None: + """Pixels are assigned to the LST settlement day via settlement_date_for. + + A scan at 2025-01-07T05:00:00Z is 21:00 LST on 2025-01-06 for KLAX (UTC-8), + so UTC-day bucketing (['2025-01-07']) would be WRONG — it must land in the + 2025-01-06 settlement day. + """ + from mostlyright.weather.satellite import _covariates as satmod + + fake_rows = [ + # 05:00Z on the 7th → LST 2025-01-06 for KLAX. + {"scan_start_utc": "2025-01-07T05:00:00Z", "variable": "ACM", "pixel_value": 1.0}, + # 09:00Z on the 7th → LST 2025-01-07 for KLAX. + {"scan_start_utc": "2025-01-07T09:00:00Z", "variable": "ACM", "pixel_value": 3.0}, + ] + fake_df = pd.DataFrame(fake_rows) + + def _fake_extract(station, *, start, end): + return fake_df + + # Patch the extractor the helper calls (imported inside the function as + # `from mostlyright.weather.satellite import satellite`). The package name + # shadows the submodule, so reach the module object via sys.modules and patch + # its `satellite` attribute. + import sys + + satpkg = sys.modules["mostlyright.weather.satellite"] + monkeypatch.setattr(satpkg, "satellite", _fake_extract) + + out = satmod.satellite_daily_covariates("KLAX", "2025-01-06", "2025-01-07") + assert set(out) == {"2025-01-06", "2025-01-07"} + # No cross-day leakage: each day gets exactly its own scan. + assert out["2025-01-06"]["sat_ACM_mean"] == 1.0 + assert out["2025-01-07"]["sat_ACM_mean"] == 3.0 + assert out["2025-01-06"]["sat_scan_count"] == 1 + + +# -------------------------------------------------------------------------- +# Task 30-07-3 — cwop helper: QC-clean only, read-only registry + history. +# -------------------------------------------------------------------------- + + +def test_cwop_helper_qc_clean_only_and_lst_bucketing(monkeypatch: pytest.MonkeyPatch) -> None: + """Only clean rows contribute; flagged/dropped change nothing; LST bucketing.""" + from mostlyright.weather.cwop import _covariates as cwopmod + + # Stub registry: one nearby PWS. + monkeypatch.setattr(cwopmod, "_icao_coords", lambda station: (33.94, -118.4)) + monkeypatch.setattr(cwopmod, "_nearby_persisted_stations", lambda lat, lon, r: ["CW0001"]) + + # history() is filtered to qc_status="clean" by the helper, so the stub only + # ever returns clean rows — proving the helper requests clean-only. A flagged + # row would never reach the aggregate because history(qc_status="clean") + # excludes it upstream. + def _fake_clean_rows(pws_ids, frm, to): + return [ + # 05:00Z on the 7th → LST 2025-01-06 for KLAX (UTC-8). + {"station_id": "CW0001", "observed_at": "2025-01-07T05:00:00Z", "temp_f": 55.0}, + {"station_id": "CW0001", "observed_at": "2025-01-07T06:00:00Z", "temp_f": 60.0}, + ] + + monkeypatch.setattr(cwopmod, "_clean_history_rows", _fake_clean_rows) + + out = cwopmod.cwop_daily_covariates("KLAX", "2025-01-06", "2025-01-07") + assert "2025-01-06" in out + day = out["2025-01-06"] + assert day["cwop_temp_high_qc"] == 60.0 + assert day["cwop_temp_low_qc"] == 55.0 + assert day["cwop_clean_station_count"] == 1 + assert day["cwop_clean_obs_count"] == 2 + # 2025-01-07 has no in-window rows → absent (join turns it NaN). + assert "2025-01-07" not in out + + +def test_cwop_helper_no_nearby_stations_returns_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """No persisted PWS in radius → empty mapping (join → NaN, never raises).""" + from mostlyright.weather.cwop import _covariates as cwopmod + + monkeypatch.setattr(cwopmod, "_icao_coords", lambda station: (33.94, -118.4)) + monkeypatch.setattr(cwopmod, "_nearby_persisted_stations", lambda lat, lon, r: []) + + out = cwopmod.cwop_daily_covariates("KLAX", "2025-01-06", "2025-01-07") + assert out == {} + + +def test_cwop_helper_uses_read_only_registry_not_scan() -> None: + """The cwop covariate module must not import scan()/nearby() (socket path).""" + src = (_REPO_ROOT / "packages/weather/src/mostlyright/weather/cwop/_covariates.py").read_text() + # Read-only discovery goes through _registry.load_stations, never scan/nearby. + assert "load_stations" in src + assert "import scan" not in src + assert "import nearby" not in src + assert "_run_live_scan" not in src From cc58326f16c5470a26c3c765a349ae31ab853150 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:40:36 +0200 Subject: [PATCH 29/52] docs(30-05): add docs/source-identity.md cross-vertical kwarg contract - NEW single contract page (no modes explainer, D-11) - source= provenance / delivery= local|hosted / vertical-defined grain (D-08) - real-time single-provider rule (D-09) - two archive classes per FEED LEG, not per vertical (D-10 refined) - frame-vs-row identity + merged.live_v1 / cli.archive contract - two-layer model + labels-only firewall + D-16 covariate composition - backend+return_type output convention (as_dataframe deprecated, D-06) - train/trade symmetry example - earnings coordination-only note naming the BACKLOG follow-up (D-14 half a) --- docs/source-identity.md | 303 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 docs/source-identity.md diff --git a/docs/source-identity.md b/docs/source-identity.md new file mode 100644 index 00000000..1b370bb0 --- /dev/null +++ b/docs/source-identity.md @@ -0,0 +1,303 @@ +# Source identity — the cross-vertical kwarg contract + +This is the ONE contract page for how you address data across every mostlyright +vertical (weather observations, climate/CLI labels, forecasts, satellite, CWOP, +and future verticals like earnings and FRED). Learn it once; it holds everywhere. + +There is deliberately **no "modes" explainer** — the old numbered-mode +vocabulary has been retired. What used to be a numbered "source-pinned mode" is +just a `source=` pin; what used to be the numbered "fused mode" is the default +source-blind (fused) path. The kwargs below say exactly what they mean, so there +is no mode to memorize. + +The two-layer surface these kwargs address: + +- **`dataset()`** — the leakage-safe *composer* that produces one row per LST + settlement day (obs extremes ⋈ CLI labels, forecasts optional, covariates + optional). `research()` is a fully working alias of `dataset()` in this release. +- **Domain tables** — the raw, source-identified ingredients: `obs()`, + `climate()`, `forecasts()` / `forecast_nwp()`, `satellite()`, `cwop.*`. + +--- + +## 1. The `source=` contract — always provenance + +`source=` **always** means *provenance* — WHO produced the row — and nothing +else. It never encodes delivery, transport, or freshness. + +- `source=None` (the default) means **best available**: the vertical's + source-blind fused/priority path. For weather observations that is the frozen + `AWC > IEM > GHCNh` merge; the result is byte-identical to `mostlyright==0.14.1`. +- `source="awc"` / `source="iem"` / `source="ghcnh"` **pins** the provenance to a + single authority. The row's per-row `source` tag will be exactly that string. +- A **degenerate single-authority axis is VALID.** A vertical may have exactly + one provenance today (EDGAR for filings, FRED for macro series, earnings from + our own STT) and still declare `source=` — it stays ready for a future pin + (e.g. earnings own-STT vs a provider transcript) without an API change. + +Passing a `source` the vertical does not recognize raises a **loud typed error** +immediately, before any network call — never a silent fallback to a different +provider. Example: `climate(..., source="acis")` raises `ValueError` (there is no +ACIS leg in that code path); the accepted set is `{None, "iem", "cli", +"cli.archive"}`. + +### `source=` pin changes row composition (retrain event) + +Pinning `source=` on `obs()` takes the **source-isolated exact-window** fetch +path, which returns the FULL single-source coverage *before* the fusion merge. +That is a **different row composition** than the default source-blind path (which +dedups across providers). This is by design and it is the fix for the old +per-source limitation — but it means: + +> **Migrating a pinned feature is a retrain event, not a find-replace.** If you +> pin `source=` on a feature that previously used the fused default, retrain the +> model on the pinned data; do not assume byte-stability across the switch. + +(The full migration guide lives in the migration doc; this page states the rule.) + +--- + +## 2. The `delivery=` contract — always local | hosted + +`delivery=` **always** means *where the computation runs* — `local` (default; +public APIs hit directly from the SDK) or `hosted` (the opt-in precomputed API, +reached only via the documented hosted seams + `MOSTLYRIGHT_API_KEY`). It is a +strictly orthogonal axis to `source=`: + +| axis | question it answers | example values | +| ----------- | ------------------- | ------------------------------- | +| `source=` | WHO produced it | `None`, `"awc"`, `"iem"`, `"noaa_goes"` | +| `delivery=` | WHERE it is computed| `"local"`, `"hosted"` | + +Hosted rows are byte-identical to the local path. Satellite already models this +correctly: `source="noaa_goes"` (provenance) × `delivery ∈ {local, hosted}` +(transport). Any vertical that overloads `source=` to mean transport is a bug — +see §8 for the one outstanding case (earnings) and its tracked follow-up. + +--- + +## 3. Grain is vertical-defined + +Grain describes the **row level** a table returns. It is defined per vertical and +is a *structural* row-level, never a clock cadence: + +| vertical | grain kwarg / values | +| --------------------- | ----------------------------------------------------------- | +| weather observations | `obs(granularity="daily" \| "observation")` | +| weather climate (CLI) | daily only — CLI is a daily settlement product, no grain axis | +| forecasts | per settlement-day envelope | +| earnings (future) | `call` / `utterance` / `occurrence` | +| FRED (future) | the series' native frequency + vintage-as-knowledge-time | + +**`obs()` granularity** (Python default `"daily"`): + +- `granularity="daily"` — one aggregated row per LST settlement day + (`obs_high_f`/`obs_low_f` extremes). Byte-identical to prior behavior. +- `granularity="observation"` — the merged **per-report** rows (METAR + SPECI + + sub-hourly routines) at the station's NATIVE cadence, window-trimmed to the + queried UTC range with NO bucketing or resampling. `observation_type` + distinguishes METAR from SPECI; `raw_metar` is preserved. Per-row `source` is + the truthful parser-emitted tag. + +There is deliberately **no `granularity="hourly"`** — that name is a lie: rows +land at the station's real cadence (METAR ~:51, plus off-cycle SPECI, plus +sub-hourly ASOS routines), not on the clock hour. Fixed-cadence *resampling* +(exact-hourly / 30-min buckets) is a separate preprocessing transform layered +on top of `granularity="observation"`, deferred to a later release — never a +`granularity="30min"` fetch value. + +--- + +## 4. The real-time rule — live surfaces are always single-provider + +Fusion is a **batch** operation: the source-blind dedup needs the query window +CLOSED to know which providers reported. A live surface cannot fuse because the +window is still open. Therefore: + +- **Live surfaces are ALWAYS single-provider.** `live.latest()` / `live.stream()`, + `cwop.stream()`, and the earnings SSE stream each speak to exactly one provider. +- On a live surface, `source=None` means **the documented default provider** for + that surface — NOT "best available fusion." The rows are still truthfully + tagged with a `*.live` provenance (e.g. `cwop.live`), never mislabeled as a + fused product. +- `snapshot` generalizes across verticals as **"settlement-target state now"** — + the current best single-provider read of the thing a market settles on. + +The train/trade seam this creates is guarded — see §7. + +--- + +## 5. Two archive classes — belong to the FEED LEG, not the vertical + +Whether historical data is re-fetchable depends on the **feed leg**, not the +table or the vertical. Most verticals end up with BOTH legs: + +- **Re-fetchable upstream legs** — the provider retains history and any window can + be rebuilt on demand. Weather obs (AWC/IEM/GHCNh), `climate()` (IEM CLI), + forecasts, EDGAR, FRED. Ask for any past window; it rebuilds. +- **Ledger / capture-or-lose legs** — data exists ONLY from the moment capture + started (a live broadcast, an APRS-IS listen). CWOP capture and earnings + webcast audio are ledger legs. A ledger table has a **hard left edge** at + capture start and **raises loudly** past it — it NEVER silently thins the + result to make a short answer look complete. + +The refinement that matters: **classify the leg, not the whole vertical.** +Earnings, for example, has a live capture leg (webcast audio — capture-or-lose) +AND a parseable archive leg behind it (call archives via a transcript provider — +re-fetchable, a separate future phase). A PWS archive leg is likewise planned +behind CWOP's live ledger. Never label an entire vertical "capture-or-lose" when +only one of its legs is. (The order-book / tape class is intentionally left +unclassified pending the exchange-phase design.) + +--- + +## 6. Frame-vs-row identity + the `merged.live_v1` contract + +There are two source tags, and they mean different things: + +- **Per-row `source`** — the truthful, durable parser/provider tag on each row + (`awc` / `iem` / `ghcnh`). It survives every DataFrame operation because it is + a real column. +- **Frame-level `df.attrs["source"]`** — a pandas-*experimental*, fragile tag on + the whole frame. `attrs` is dropped by `concat` / `merge` / `groupby` / + `to_dict`, so treat it as advisory, not load-bearing. Re-derive identity from + the per-row column when you need durability. + +**The merged-tag contract:** + +- The fused observation frame carries the frame tag **`merged.live_v1`** — a + DISTINCT merged-policy identity, not any single provider's name. +- `merged.live_v1` is **REJECTED by every single-source pinned schema** + (`schema.observation.v1` raises `SourceMismatchError`) and is **ACCEPTED ONLY + by `schema.observation.merged.v1`** (`MergedObservationSchema`), which validates + the per-row `source` as membership in `{awc, iem, ghcnh}` rather than + equality-to-a-single-provider. This is what stops a fused frame from silently + passing through a pinned-source validator. +- `climate()` mirrors the split: the frame tag is **`cli.archive`** (the + settlement *product*) while the per-row `source` is **`iem`** (the fetch + *endpoint*). Both are truthful; they answer different questions. + +(TypeScript has no `df.attrs`; it carries the equivalent identity as a +non-enumerable `frameSource` property on the returned array, so rows still +iterate / JSON / spread byte-stably.) + +--- + +## 7. The two-layer model + the labels-only firewall + +**Layer 1 — domain tables** are the raw, source-identified ingredients: +`obs()`, `climate()`, `forecasts()` / `forecast_nwp()`, `satellite()`, `cwop.*`. +Each obeys the `source=` / `delivery=` / grain contract above and can be joined by +hand like any table. + +**Layer 2 — `dataset()`** is the one cross-vertical composer. It owns the +leakage-prone step: aligning everything to the LST settlement day and producing +one leakage-safe training row per day. + +**The firewall is LABELS-ONLY (D-01 as amended).** The settlement firewall +protects the LABEL computation only: + +- Satellite and CWOP rows must **NEVER** enter the source-blind obs `max/min` + aggregation and must **NEVER** touch the `cli_*` columns. They are never + observation *sources* — no registration in the four parity-firewall files + (`research.py` observation merge, `merge/observations.py` `SOURCE_PRIORITY`, + `live/_sources.py`, `schema.observation.v1` enum). +- But covariate-COLUMN composition **is allowed**: `dataset(include_satellite=, + include_cwop=)` LEFT-joins per-settlement-day covariate columns (§below). This + is the "labels-only" narrowing — the blanket "never composed" rule is + withdrawn. + +### Covariate composition (D-16) + +`dataset(..., include_satellite=False, include_cwop=False)` (and the `research()` +alias): when a flag is enabled, the SDK LEFT-joins per-settlement-day covariate +columns onto the composed frame by `(station, LST settlement day)`: + +- Prefixes: `sat_*` (satellite), `cwop_*` (CWOP). +- **Label columns byte-identical** — `obs_*`, `cli_*`, `date`, `station` are the + same whether the flags are on or off; defaults are off, so parity fixtures are + untouched. +- **Row count and order identical** to the default output. Missing covariate days + are `NaN` — rows are **never dropped or added**. +- Bucketing is by the **LST settlement window**, never the UTC day. +- CWOP covariates aggregate **QC-clean rows only** (combined score > 0.7), + read-only from the persisted `history()` cache (no APRS sockets), and carry the + ledger left-edge — days before capture start are `NaN`, never silently thinned. +- A missing `[satellite]` / `[cwop]` extra fails **loud** (typed error), never a + silent empty column. Frame `attrs["covariates"]` records covariate provenance + (`SATELLITE_COVARIATE_SOURCE="noaa_goes"`, `CWOP_COVARIATE_SOURCE="cwop.cache"`). +- Covariates require the DataFrame return path; `return_type="list"` combined with + a covariate flag raises `ValueError` (raw dicts have no column to attach to). + +The manual hand-join is still fully available and documented — `dataset()` just +lets the SDK own the leakage-prone day alignment once, instead of every quant +hand-rolling it. See [`docs/satellite.md`](satellite.md) and +[`docs/cwop-adapter.md`](cwop-adapter.md) for the column contracts. + +--- + +## 8. The output convention — `backend` + `return_type` + +There is ONE output convention across every public surface: + +- **`backend`** — the DataFrame engine (`"pandas"`, `"polars"`, …). +- **`return_type`** — the shape. The canonical value is **`"dataframe"`**; + `"list"` returns `list[dict]`; `"wrapper"` returns the typed wrapper. (The + TypeScript SDK accepts `"dataframe"` as canonical and keeps `"frame"` as a + back-compat alias.) +- **`as_dataframe` is deprecated** — it still works this release (mapping + `True → "dataframe"`, `False → "list"`) and emits a `DeprecationWarning` when + passed explicitly. An explicit `return_type` always wins. Per-function default + return shapes are unchanged, so every no-arg call stays byte-stable. + +Incoherent combinations raise loudly — e.g. `backend="polars"` with a pandas-only +`return_type` is a `ValueError`. + +--- + +## 9. Train / trade symmetry — the validator guards the seam + +The contract exists so that a feature **backtests the same way it trades**. The +canonical example: + +```python +import mostlyright +from mostlyright.weather import obs +from mostlyright.core import validate_dataframe + +# TRAIN — pinned historical, per-report grain +train = obs("KNYC", "2024-01-01", "2024-12-31", + source="awc", granularity="observation") +validate_dataframe(train, schema_id="schema.observation.v1") + +# TRADE — the SAME provenance, live +tick = mostlyright.live.stream("KNYC", source="awc") +``` + +Because BOTH sides pin `source="awc"`, the source identity is consistent across +the train/infer seam. If you accidentally route a fused (`merged.live_v1`) frame +through a single-source pinned schema, `validate_dataframe` raises +`SourceMismatchError` at load time — loud, before the mismatch can silently +corrupt a model. That is the whole point: a train/infer source mismatch errors +loudly instead of quietly poisoning predictions. + +--- + +## 10. Earnings reconciliation — coordination-only note + +The earnings vertical currently declares `source ∈ {live, hosted}`. Under this +contract that is a **`delivery=` axis mislabeled as `source=`** — `{live, hosted}` +describes *where* the data is computed, not *who* produced it. (Satellite got this +right: `source="noaa_goes"` × `delivery ∈ {live, hosted}`.) + +**This phase changes NO earnings schema or code — it documents the target rule +only.** The target rule: earnings should carry `delivery ∈ {live, hosted}`, +freeing `source=` for true provenance (today degenerate — our own STT — but ready +for a future provider-transcript pin). + +The tracked follow-up is filed as a **P1 item in `.planning/BACKLOG.md`**: +*"Earnings schema `source ∈ {live,hosted}` → `delivery=` reconciliation +[Phase 27/28 owners]"*. Fixing it before the hosted earnings path is publicly +exposed (Phase 28 deploy) avoids a breaking schema change on a live surface. The +paper trail — target rule here, tracked owner-assigned item there — closes the +D-14 coordination requirement. From b4635f6b80960065417b4e81fc985ee1cba5c2d7 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:41:14 +0200 Subject: [PATCH 30/52] docs(30-05): rewrite README train/infer snippet to new surface - swap mode2.research_by_source for obs(source=, granularity="observation") - delete Mode 1/Mode 2 wording; note pinned features = retrain event - prefer dataset() (research() still works); link docs/source-identity.md --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f8800aab..dc2fa0df 100644 --- a/README.md +++ b/README.md @@ -93,26 +93,36 @@ from mostlyright.markets.catalog import kalshi_nhigh # 1. What does the contract settle on? contract = kalshi_nhigh.resolve("KHIGHNYC", date(2025, 1, 15)) -# 2. Pull the data that decides it. -df = mostlyright.research(contract.settlement_station, "2025-01-15", "2025-01-15") +# 2. Pull the data that decides it. dataset() is the primary name; +# research() is a fully working alias in this release. +df = mostlyright.dataset(contract.settlement_station, "2025-01-15", "2025-01-15") # 3. Apply the threshold and decide. ``` ### Train models with train/infer source parity -`research()` stamps a source identity on every row. The validator catches train/infer mismatches at load time, instead of silently corrupting predictions in production. +`dataset()` (and its `research()` alias) stamps a source identity on every row. +Pinning `source=` on `obs()` returns single-source coverage; the validator catches +train/infer mismatches at load time, instead of silently corrupting predictions in +production. ```python -from mostlyright.mode2 import research_by_source +from mostlyright.weather import obs from mostlyright.core import validate_dataframe -train = research_by_source("KNYC", "iem.archive", "2024-01-01", "2024-12-31") -# Validator pins the schema's expected source. SourceMismatchError fires if -# you accidentally route a Mode 1 (fused AWC+IEM+GHCNh) DataFrame through a -# Mode 2 (iem.archive only) schema. +train = obs("KNYC", "2024-01-01", "2024-12-31", + source="iem.archive", granularity="observation") +# Validator pins the schema's expected source. SourceMismatchError fires if you +# accidentally route a source-blind (fused AWC+IEM+GHCNh) DataFrame through an +# iem.archive-only schema. See the source-identity contract for the full rule. validate_dataframe(train, schema_id="schema.observation.v1") ``` +> Pinning `source=` returns a different row composition than the fused default — +> migrating a pinned feature is a **retrain event**, not a find-replace. The one +> contract for `source=` / `delivery=` / grain across every vertical lives in +> [`docs/source-identity.md`](docs/source-identity.md). + ### Feed AI agents structured public data Every response carries a stable `schema.*.v1` URI and serializes to JSON-Schema-validated shapes. Drop responses into MCP tool outputs or OpenAI/Anthropic function-call returns without re-shaping. From d3543e8d92f86866c32627a454122199bad05641 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:41:52 +0200 Subject: [PATCH 31/52] docs(30-05): purge Axis-A Mode vocab in ingest-strategies + ts-quickstart - ingest-strategies: 'research() Mode-1 obs aggregates' -> 'dataset()/research() default-path (fused-source) obs aggregates'; add strategy= demotion + granularity="observation" callout - ts-quickstart: 'Mode 1 parity' heading -> 'default-path parity' --- docs/ingest-strategies.md | 15 +++++++++++---- docs/ts-quickstart.md | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/ingest-strategies.md b/docs/ingest-strategies.md index ca54ccd1..5b152e36 100644 --- a/docs/ingest-strategies.md +++ b/docs/ingest-strategies.md @@ -18,6 +18,12 @@ df = obs("KNYC", "2024-03-01", "2024-03-31", strategy="warm_cache") df = obs("KNYC", "2024-03-01", "2024-03-31", strategy="hosted") # v0.2.x ``` +> **`strategy=` is a demoted escape hatch.** `strategy="auto"` (the default) +> already picks the right tactic for you — including the source-isolated +> exact-window path for pinned (`source=`) queries — so ordinary callers should +> never type `strategy=`. For the row-level axis use `obs(granularity="daily" | +> "observation")` instead (see [`docs/source-identity.md`](source-identity.md) §3). + ## Decision tree (`strategy="auto"`) ``` @@ -82,7 +88,8 @@ Notes: separation; no filename infix collisions). - `warm_cache` populates the canonical cache; subsequent calls with overlapping windows benefit from the year-aligned read path. Byte-equivalent to - `research()` Mode-1 obs aggregates for the 5 parity fixtures. + `dataset()`/`research()` default-path (fused-source) obs aggregates for the 5 + parity fixtures. - `warm_cache` requires `source=None` — single-source warm_cache would corrupt the merge-priority semantics. For source-filtered queries use `exact_window` (fetcher-boundary enforcement preserves priority correctly). @@ -118,9 +125,9 @@ mutable-period logic. on a cold cache and want minimal bytes (e.g. CI). Also use for single-source queries (`source="iem"`, etc.) — these are not supported by `warm_cache`. - **Force `warm_cache`** if you're running batch backtests across many windows - and want the year-aligned cache to pay off. This is what `research()` does - internally — `obs(strategy="warm_cache")` is byte-equivalent to `research()` - Mode-1 obs aggregates. + and want the year-aligned cache to pay off. This is what `dataset()`/`research()` + does internally — `obs(strategy="warm_cache")` is byte-equivalent to + `dataset()`/`research()` default-path (fused-source) obs aggregates. - **`hosted`** will become the default for v0.2.x users with `TW_HOSTED_URL` set, once the precomputed-API client lands. diff --git a/docs/ts-quickstart.md b/docs/ts-quickstart.md index 1b6a1034..18c80136 100644 --- a/docs/ts-quickstart.md +++ b/docs/ts-quickstart.md @@ -11,7 +11,7 @@ The TS SDK mirrors the Python public surface and ships four npm packages: Five-minute path: **Node** + **browser** below. The browser path is for service workers / content scripts in extensions or web apps; the Node path is for scripts, backtests, Workers, and Bun/Deno. -## Node (and Bun / Deno) — Mode 1 parity +## Node (and Bun / Deno) — default-path parity ```bash # research() ships from the meta package only — install that. From f6dd6dcb715aad4cf64977dc9f3c0f0f11af4b2a Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:42:46 +0200 Subject: [PATCH 32/52] docs(30-05): rename Axis-B forecast Mode headings descriptively - forecasts.md: 'Mode 1: IEM MOS' -> 'IEM MOS'; 'Mode 2: ...NWP' -> 'per-model NWP'; prose 'Mode 1/Mode 2' -> 'IEM MOS surface / per-NWP-model surface'; H3 'Settlement-day envelope (Mode 2)' -> '(NWP)' - nwp-forecasts.md cross-ref: 'Mode 1/Mode 2' -> 'IEM MOS / per-NWP-model' - semantic content preserved (rename, not delete) per D-11 --- docs/forecasts.md | 10 +++++----- docs/nwp-forecasts.md | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/forecasts.md b/docs/forecasts.md index d4963b99..491ee74e 100644 --- a/docs/forecasts.md +++ b/docs/forecasts.md @@ -26,11 +26,11 @@ from mostlyright import research # Default: include_forecast=False (parity-compatible v0.14.1 behavior) df = research("KNYC", "2026-05-01", "2026-05-07") -# Mode 1: IEM MOS forecasts populate fcst_* columns +# IEM MOS forecasts populate fcst_* columns df = research("KNYC", "2026-05-01", "2026-05-07", include_forecast=True) print(df[["date", "obs_high_f", "fcst_high_f", "fcst_model"]]) -# Mode 2: Additionally include NWP forecasts per model +# Additionally include per-model NWP forecasts df = research( "KNYC", "2026-05-01", "2026-05-07", include_forecast=True, @@ -40,8 +40,8 @@ print(df[["date", "fcst_high_f", "fcst_high_f_nwp_hrrr", "fcst_high_f_nwp_gfs"]] ``` `research(include_forecast=True)` was previously raising -`NotImplementedError`; v1.0 wires both Mode 1 (IEM MOS) and Mode 2 -(per-NWP-model) end-to-end. Settlement-day bucketing uses station LST +`NotImplementedError`; v1.0 wires both the IEM MOS surface and the +per-NWP-model (NWP) surface end-to-end. Settlement-day bucketing uses station LST via `settlement_date_for(observed_at, station)` so post-midnight tail rows roll into the correct calendar settlement. @@ -237,7 +237,7 @@ df = forecast_nwp(station="KNYC", model="cfs", member="03") > DataFrame — it selects which member's grid is fetched. A per-row > `member` column is tracked as future work. -### Settlement-day envelope (Mode 2) +### Settlement-day envelope (NWP) `research(include_forecast=True, forecast_models=[...])` fetches a 36-hour NWP envelope per settlement day: diff --git a/docs/nwp-forecasts.md b/docs/nwp-forecasts.md index c034442d..5faf8ae9 100644 --- a/docs/nwp-forecasts.md +++ b/docs/nwp-forecasts.md @@ -157,8 +157,8 @@ For up-to-date wiring status and historical-depth bounds, see ## See also -- [forecasts.md](./forecasts.md) — Python-side NWP wiring and Mode 1/Mode 2 - `research(include_forecast=True)` documentation +- [forecasts.md](./forecasts.md) — Python-side NWP wiring and IEM MOS / + per-NWP-model `research(include_forecast=True)` documentation - [climate-gaps.md](./climate-gaps.md) — parallel deferral on `climateGaps()`, also browser-environment limited - `.planning/phases/17-forecast-catalog-expansion-herbie-wide-models/` — From 654a52b56980799445e3aa824afd62afde6066a5 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 01:47:09 +0200 Subject: [PATCH 33/52] docs(30-05): satellite/cwop labels-only firewall + covariate composition - cwop-adapter: refine 'never feeds research()' firewall box to labels-only (never a source / label input; composes as covariate columns or hand-join); upgrade D-16 stub to full docs; add domain-table list incl. climate(); cross-ref source-identity.md; refine two stale 'no research() integration' lines - satellite: upgrade D-16 stub to full covariate docs; add domain-table list incl. climate(); cross-ref source-identity.md in See also --- docs/cwop-adapter.md | 64 +++++++++++++++++++++++++++++++------------- docs/satellite.md | 42 ++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 28 deletions(-) diff --git a/docs/cwop-adapter.md b/docs/cwop-adapter.md index a0f0f2f7..5f199a11 100644 --- a/docs/cwop-adapter.md +++ b/docs/cwop-adapter.md @@ -5,14 +5,24 @@ Program**, delivered over the APRS-IS network. ~7K active North American stations report every 5–15 minutes. The only real-time path is a TCP stream (no REST), so this adapter is **live-only**. -> **CWOP is standalone.** It never feeds `research()`, the merge layer, or -> `live` sources, and it carries its own `schema.cwop.v1` (separate from the -> parity-frozen `schema.observation.v1`). A hot-rooftop or indoor PWS must never -> silently corrupt a Kalshi NHIGH/NLOW settlement target — so CWOP lives on its -> own island, gated by a 6-layer QC pipeline. There is no `research()` -> integration: CWOP reaches ML strategies only through its **own** access path -> (`cwop.history()`), which returns a `schema.cwop.v1` DataFrame the strategy -> joins itself — fully aware it is unofficial PWS data. +> **CWOP obeys the labels-only firewall.** CWOP never enters the LABEL +> computation — it is never a `research()`/`dataset()` observation **source**, it +> never touches the source-blind `obs_*` `max/min` aggregation, and it never +> registers in the four parity-firewall files (`research.py` observation merge, +> `merge/observations.py` `SOURCE_PRIORITY`, `live/_sources.py`, +> `schema.observation.v1` enum). It carries its own `schema.cwop.v1`. A +> hot-rooftop or indoor PWS must never silently corrupt a Kalshi NHIGH/NLOW +> settlement target, so it stays out of the label — gated by a 6-layer QC +> pipeline. +> +> But CWOP is **not** walled off from models. It reaches strategies two ways: +> (1) as **covariate columns** via `dataset(..., include_cwop=True)` (Phase 30 +> D-16), which LEFT-joins QC-clean per-settlement-day `cwop_*` columns onto the +> composed frame WITHOUT altering any label column; and (2) as a raw table via +> its **own** access path `cwop.history()` (a `schema.cwop.v1` DataFrame the +> strategy hand-joins itself, like any table). See +> [`docs/source-identity.md`](source-identity.md) for the cross-vertical +> labels-only firewall rule. ## Install @@ -54,7 +64,9 @@ async for obs in cwop.stream("CW0875"): CWOP data is ephemeral — the APRS-IS stream has no REST backfill, so the only way to build a history for model training is to **persist what you collect**. v0.2 adds a monthly parquet cache and a `history()` replay query, both behind -the same firewall as the live surface (CWOP never enters `research()`/merge). +the same labels-only firewall as the live surface (CWOP never enters the +`research()`/merge LABEL computation; it composes only as `dataset()` covariate +columns or a hand-joined table). ```python from datetime import date @@ -86,18 +98,32 @@ clean = cwop.history("CW0875", date(2026, 1, 1), date(2026, 6, 30), qc_status="c - **Source identity:** live pulls are `"cwop.live"`; anything read back from the cache is `"cwop.cache"`. `schema.cwop.v1` accepts both. - **`include_cwop=True` = COVARIATE columns, not a settlement source (D-16).** - `research()`/`dataset()` aggregate `max(temp_f)/min(temp_f)` source-blind for + `dataset()`/`research()` aggregate `max(temp_f)/min(temp_f)` source-blind for the LABEL, so a single hot-rooftop or indoor PWS must never enter that aggregation — CWOP stays out of the four parity-firewall files. But `dataset(..., include_cwop=True)` (Phase 30 D-16) LEFT-joins per-settlement-day `cwop_*` COVARIATE columns onto the composed frame: **QC-clean rows only** (combined > 0.7), read-only from the persisted `history()` cache (no APRS connections), with the ledger left-edge respected (days before capture start → - `NaN`, never silently thinned). The label columns stay byte-identical with the - flag on or off. See the `dataset()` docstring for the column contract; the - manual `history()` join below is still available and documented. - - > **Cross-reference stub — full covariate docs land in 30-05.** + `NaN`, never silently thinned). The label columns (`obs_*`, `cli_*`, `date`, + `station`) stay byte-identical, and the row count/order are identical, with the + flag on or off. A missing `[cwop]` extra fails loud (typed error), never a + silent empty column. See the `dataset()` docstring for the exact column + contract and [`docs/source-identity.md`](source-identity.md) §7 for the + cross-vertical rule. + +### Composing with the other domain tables + +`dataset()` is the one cross-vertical composer; CWOP is one of the raw +source-identified domain tables it can draw from alongside `obs()`, `climate()` +(NWS CLI daily settlement labels), `forecasts()` / `forecast_nwp()`, and +`satellite()`. Two paths reach a model: + +- **`dataset(..., include_cwop=True)`** — the SDK owns the leakage-prone LST-day + alignment and joins `cwop_*` covariate columns for you (labels-only firewall). +- **`cwop.history()`** — the raw `schema.cwop.v1` table, which you hand-join like + any other table (fully aware it is unofficial PWS data). This is still + available and documented below. Every failure mode raises `NoCWOPDataError` (a `NoLiveDataError` subclass) with the requested station/area and a `reason` — the API never returns `[]`/`None`. @@ -150,9 +176,11 @@ freshly-seen station is scored mostly on the range + temporal layers. `history()` only returns what you previously persisted (via `snapshot(persist=True)` or `persist_observations()`). There is no way to retrieve CWOP data for a date before you started collecting. -- **No `research()` integration** — by design. CWOP reaches models only through - the standalone `history()` path (see *Persistence + backtest* above); it is - never folded into the parity-critical settlement join. +- **No LABEL integration** — by design. CWOP never enters the parity-critical + settlement LABEL join (the source-blind `obs_*` aggregation / `cli_*` columns). + It reaches models only as `dataset(..., include_cwop=True)` covariate columns + (D-16) or the standalone hand-joined `history()` path (see *Persistence + + backtest* above) — never folded into the label. - **Buddy check needs an ASOS** within 50km; degraded scoring otherwise. - **Solar-bias / indoor detection need 7d / 24h** of history (the live session rarely has it), so they pass until enough data accumulates. diff --git a/docs/satellite.md b/docs/satellite.md index 307c8a58..18a29a81 100644 --- a/docs/satellite.md +++ b/docs/satellite.md @@ -353,19 +353,41 @@ EUMETSAT key. ## Covariate composition on `dataset()` (D-16) -> **Cross-reference stub — full docs land in 30-05.** Beyond the manual join, you -> can let the SDK own the leakage-prone LST-day alignment: -> `dataset(..., include_satellite=True)` (Phase 30 D-16) LEFT-joins -> per-settlement-day `sat_*` covariate columns onto the composed training frame -> by `(station, LST settlement day)`. Satellite stays a COVARIATE, never a -> settlement source — the label columns (`obs_*`/`cli_*`) are byte-identical with -> the flag on or off, days without coverage become `NaN` (rows are never -> dropped/added), and pixels are bucketed by the LST settlement window (never UTC -> day). Requires the `[satellite]` extra (fail-loud otherwise). See `dataset()`'s -> docstring for the exact column contract. +Satellite obeys the **labels-only firewall**: it is never a settlement source and +never touches the LABEL computation (the source-blind `obs_*` aggregation or the +`cli_*` columns), and it never registers in the four parity-firewall files. But it +is **not** walled off from models — it reaches them two ways. + +**1. As covariate columns — `dataset(..., include_satellite=True)` (D-16).** +The SDK owns the leakage-prone LST-day alignment for you: it LEFT-joins +per-settlement-day `sat_*` covariate columns onto the composed training frame by +`(station, LST settlement day)`. + +- Label columns (`obs_*`, `cli_*`, `date`, `station`) are **byte-identical** with + the flag on or off; the row count and order are identical too. +- Days without satellite coverage become `NaN` — rows are **never dropped or + added**. +- Pixels are bucketed by the **LST settlement window**, never the UTC day + (`settlement_date_for`), so a scan near midnight lands in the correct + settlement day. +- Requires the `[satellite]` extra — a missing extra fails **loud** (a typed + `SourceUnavailableError` naming `[satellite]`), never a silent empty column. + Covariate provenance is recorded in `df.attrs["covariates"]` + (`SATELLITE_COVARIATE_SOURCE="noaa_goes"`). + +See the `dataset()` docstring for the exact `sat_*` column set and +[`docs/source-identity.md`](source-identity.md) §7 for the cross-vertical rule. + +**2. As a raw table — `satellite()`.** The single-pixel extraction is one of the +raw source-identified domain tables `dataset()` composes over, alongside `obs()`, +`climate()` (NWS CLI daily settlement labels), and `forecasts()` / +`forecast_nwp()`. You can call `satellite()` directly and hand-join it like any +other table — the manual path is fully available and documented above. ## See also +- [`docs/source-identity.md`](source-identity.md) — the cross-vertical + `source=` / `delivery=` / grain contract and the labels-only firewall. - [`docs/forecasts.md`](forecasts.md) — the NWP forecast path (the load-bearing Tmax/Tmin signal). - [`docs/forecast-sources.md`](forecast-sources.md) — forecast source catalog. From cb39f573d7154583deba7c4d0766015cb3e177d0 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:07:48 +0200 Subject: [PATCH 34/52] =?UTF-8?q?fix(30-review):=20WR-01=20=E2=80=94=20sat?= =?UTF-8?q?ellite=20covariates=20count=20only=20contributing=20scans?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the counts[settle_day] increment AFTER the None/NaN/non-numeric guards so sat_scan_count tallies only scans that contribute a usable value. A settlement day whose scans are all fill-value/NaN now produces NO row (dropped to NaN after the covariate LEFT join) instead of a misleading sat_scan_count-only row that a coverage-gating trading model would read as 'covered' while it carries zero real signal. Co-Authored-By: Claude Fable 5 --- .../core/tests/test_dataset_covariates.py | 42 +++++++++++++++++++ .../weather/satellite/_covariates.py | 9 +++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/packages/core/tests/test_dataset_covariates.py b/packages/core/tests/test_dataset_covariates.py index 2225dbb2..e4bcb697 100644 --- a/packages/core/tests/test_dataset_covariates.py +++ b/packages/core/tests/test_dataset_covariates.py @@ -360,6 +360,48 @@ def _fake_extract(station, *, start, end): assert out["2025-01-06"]["sat_scan_count"] == 1 +def test_satellite_helper_counts_only_contributing_scans( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """WR-01: a day whose scans are ALL fill-value/None emits NO row. + + `sat_scan_count` must tally only scans that contributed a usable value. + A day of pure fill-value scans (NaN pixel_value, or None variable/value) + previously produced a `sat_scan_count`-only day with no `sat__mean`, + which looks "covered" to a coverage-gating model but carries zero signal. + """ + import math + + from mostlyright.weather.satellite import _covariates as satmod + + fake_rows = [ + # 2025-01-06 (LST): one real scan (contributes) + one fill-value scan. + {"scan_start_utc": "2025-01-06T20:00:00Z", "variable": "ACM", "pixel_value": 2.0}, + {"scan_start_utc": "2025-01-06T21:00:00Z", "variable": "ACM", "pixel_value": math.nan}, + # 2025-01-07 (LST): ONLY fill-value / None scans → must produce NO row. + {"scan_start_utc": "2025-01-07T20:00:00Z", "variable": "ACM", "pixel_value": math.nan}, + {"scan_start_utc": "2025-01-07T21:00:00Z", "variable": None, "pixel_value": 5.0}, + {"scan_start_utc": "2025-01-07T22:00:00Z", "variable": "ACM", "pixel_value": None}, + ] + fake_df = pd.DataFrame(fake_rows) + + def _fake_extract(station, *, start, end): + return fake_df + + import sys + + satpkg = sys.modules["mostlyright.weather.satellite"] + monkeypatch.setattr(satpkg, "satellite", _fake_extract) + + out = satmod.satellite_daily_covariates("KLAX", "2025-01-06", "2025-01-08") + # 2025-01-07 had only fill-value/None scans → NO row (not a count-only row). + assert set(out) == {"2025-01-06"} + # The one real scan is counted; the fill-value scan on the same day is not. + assert out["2025-01-06"]["sat_scan_count"] == 1 + assert out["2025-01-06"]["sat_ACM_mean"] == 2.0 + assert "2025-01-07" not in out + + # -------------------------------------------------------------------------- # Task 30-07-3 — cwop helper: QC-clean only, read-only registry + history. # -------------------------------------------------------------------------- diff --git a/packages/weather/src/mostlyright/weather/satellite/_covariates.py b/packages/weather/src/mostlyright/weather/satellite/_covariates.py index e28f4f49..1fc76c7f 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_covariates.py +++ b/packages/weather/src/mostlyright/weather/satellite/_covariates.py @@ -107,10 +107,8 @@ def satellite_daily_covariates( scan_ts = row.get("scan_start_utc") if not scan_ts: continue - settle_day = settlement_date_for(scan_ts, station, tz_override=tz_override) variable = row.get("variable") value = row.get("pixel_value") - counts[settle_day] = counts.get(settle_day, 0) + 1 if variable is None or value is None: continue # Skip NaN pixel values (a _FillValue scan) so the mean is over real cells. @@ -120,6 +118,13 @@ def satellite_daily_covariates( continue if fval != fval: # NaN check without importing math continue + # WR-01: count only scans that CONTRIBUTE a usable value. Bumping the + # count before the None/NaN/non-numeric guards produced a + # `sat_scan_count`-only day (no `sat__mean`) that looked "covered" + # to a coverage-gating trading model while carrying zero real signal. + # A day with only fill-value scans must now yield NO row. + settle_day = settlement_date_for(scan_ts, station, tz_override=tz_override) + counts[settle_day] = counts.get(settle_day, 0) + 1 by_day.setdefault(settle_day, {}).setdefault(str(variable), []).append(fval) out: dict[str, dict[str, Any]] = {} From 809eacfd3c346c560e0ceeb8b370df3f109bad1b Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:07:56 +0200 Subject: [PATCH 35/52] =?UTF-8?q?fix(30-review):=20WR-04=20=E2=80=94=20add?= =?UTF-8?q?=20climate.py=20to=20firewall=20grep=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parity-firewall regression guard parametrized only observations.py and live/_sources.py; _internal/merge/climate.py (the frozen climate dedup, a named firewall file) was never grep-asserted free of cwop/satellite registration. Add it to the parametrized rel_path list so a future covariate-source edit to the climate merge is caught. Co-Authored-By: Claude Fable 5 --- packages/core/tests/test_dataset_covariates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/tests/test_dataset_covariates.py b/packages/core/tests/test_dataset_covariates.py index e4bcb697..462dbbe5 100644 --- a/packages/core/tests/test_dataset_covariates.py +++ b/packages/core/tests/test_dataset_covariates.py @@ -214,6 +214,7 @@ def _bad_sat(station, frm, to, *, tz_override=None): "rel_path", [ "packages/core/src/mostlyright/_internal/merge/observations.py", + "packages/core/src/mostlyright/_internal/merge/climate.py", "packages/core/src/mostlyright/live/_sources.py", ], ) From 4c89ce991d391822c8270d60d74c1c01309ac6ad Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:08:25 +0200 Subject: [PATCH 36/52] =?UTF-8?q?fix(30-review):=20WR/P2=20=E2=80=94=20sta?= =?UTF-8?q?mp=20retrieved=5Fat=20on=20observation-grain=20frames?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The granularity='observation' producer stamped only df.attrs['source'], but schema.observation.merged.v1's validator contract requires df.attrs['retrieved_at'] (core/validator.py:491-509). Stamp datetime.now(UTC) alongside the source stamp under the same best-effort contextlib.suppress pattern so the frame validates AS PRODUCED. Remove the manual attr injection from the test and assert the attr is present + tz-aware on the frame straight from obs(). Daily/dataset path untouched. Co-Authored-By: Claude Fable 5 --- packages/weather/src/mostlyright/weather/obs.py | 8 ++++++++ tests/weather/test_obs_observation_grain.py | 16 +++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/obs.py b/packages/weather/src/mostlyright/weather/obs.py index 17fc5450..5b6b5b95 100644 --- a/packages/weather/src/mostlyright/weather/obs.py +++ b/packages/weather/src/mostlyright/weather/obs.py @@ -295,6 +295,7 @@ def _observation_grain_rows( return trimmed import contextlib + from datetime import UTC, datetime import pandas as pd @@ -309,6 +310,13 @@ def _observation_grain_rows( frame_source = source if source is not None else _MERGED_FRAME_SOURCE with contextlib.suppress(AttributeError): df.attrs["source"] = frame_source + # FIX-3 (WR/P2): the advertised validator contract for + # schema.observation.merged.v1 requires df.attrs["retrieved_at"] + # (core/validator.py:491-509). The observation-grain producer is the + # frame's origin, so stamp the fetch-completion time here — same + # best-effort contextlib.suppress pattern as the source stamp — so the + # frame validates AS PRODUCED without the caller injecting the attr. + df.attrs["retrieved_at"] = datetime.now(UTC) if backend == "pandas" and return_type == "dataframe": return df diff --git a/tests/weather/test_obs_observation_grain.py b/tests/weather/test_obs_observation_grain.py index e5f5a074..e0b07456 100644 --- a/tests/weather/test_obs_observation_grain.py +++ b/tests/weather/test_obs_observation_grain.py @@ -15,7 +15,7 @@ from __future__ import annotations import inspect -from datetime import UTC, datetime +from datetime import datetime from unittest.mock import patch import pytest @@ -125,9 +125,7 @@ def test_observation_grain_pinned_frame_identity_is_bare_source() -> None: # source="iem" forces exact_window in the resolver; _dispatch_strategy is # patched, so no network. The frame identity must be the bare source. with _patch_dispatch([r for r in _RAW_ROWS if r["source"] == "iem"]): - df = obs( - "KNYC", "2026-01-01", "2026-01-03", source="iem", granularity="observation" - ) + df = obs("KNYC", "2026-01-01", "2026-01-03", source="iem", granularity="observation") assert df.attrs["source"] == "iem" @@ -166,8 +164,12 @@ def test_observation_grain_frame_validates_against_merged_schema() -> None: with _patch_dispatch(list(_RAW_ROWS)): df = obs("KNYC", "2026-01-01", "2026-01-03", granularity="observation") - # obs() does not stamp retrieved_at; the validator requires it, so supply it - # (this mirrors what a catalog adapter would do on a real pull). - df.attrs["retrieved_at"] = datetime.now(UTC) + # FIX-3: the producer now stamps df.attrs["retrieved_at"] itself, so the + # frame validates AS PRODUCED — no caller-side injection. Assert the attr is + # present and tz-aware (UTC), then validate against the merged schema. + assert "retrieved_at" in df.attrs + stamped = df.attrs["retrieved_at"] + assert isinstance(stamped, datetime) + assert stamped.tzinfo is not None reg = validate_dataframe(df, "schema.observation.merged.v1") assert all(e["event"] != "source_drift_allowed" for e in reg.audit_log()) From e710daaff7eca27731e518dfd14d68a95a958653 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:08:32 +0200 Subject: [PATCH 37/52] =?UTF-8?q?fix(30-review):=20WR-02+WR-03=20=E2=80=94?= =?UTF-8?q?=20compose=5Fcovariates=20contract=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docstring-only, no behavior change: (a) document that compose_covariates takes ownership of and mutates 'result' in place, so callers must pass a frame they own; (b) document that attrs['covariates'] provenance survives only the pandas return_type='dataframe' path (dropped by the backend/wrapper conversion). Co-Authored-By: Claude Fable 5 --- packages/core/src/mostlyright/_covariates.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/core/src/mostlyright/_covariates.py b/packages/core/src/mostlyright/_covariates.py index 7bcd40ce..64e427bd 100644 --- a/packages/core/src/mostlyright/_covariates.py +++ b/packages/core/src/mostlyright/_covariates.py @@ -128,6 +128,16 @@ def compose_covariates( with additive ``sat_*`` / ``cwop_*`` columns and a ``attrs["covariates"]`` provenance dict. The label columns + row order are untouched. + Contract (WR-02): this function TAKES OWNERSHIP of ``result`` and MUTATES it + in place (``result[col] = ...`` and ``result.attrs[...] = ...``). Callers + MUST pass a frame they own (``dataset()`` passes a freshly-built + ``pairs_to_dataframe`` frame). Do NOT pass a shared/borrowed frame. + + Contract (WR-03): the ``attrs["covariates"]`` provenance dict survives ONLY + on the pandas ``return_type="dataframe"`` path. The ``backend="polars"`` / + ``return_type="wrapper"`` conversion drops ``df.attrs``, so covariate + provenance is not surfaced there. + Args: result: The composed label DataFrame (indexed by ``date``). MUST be a DataFrame — the covariate path is unreachable for the ``list`` From 6e24664ae0125375b147dda563f7fe61cfefb6d3 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:23:28 +0200 Subject: [PATCH 38/52] =?UTF-8?q?fix(30-review):=20CR-01=20=E2=80=94=20TS?= =?UTF-8?q?=20climate()=20resolves=20NWS=E2=86=94ICAO=20for=20cross-SDK=20?= =?UTF-8?q?parity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TS climate() uppercased the input and passed it verbatim to downloadCliRange, projecting the parser's station_code. Python resolves the input through the station registry, fetches by info.icao, and projects info.code (the 3-letter NWS code) into every row's station column. Add a shared weather-internal station module (src/_internal/stations.ts) with resolveStation (mirrors the mode2.ts helper + Python _resolve_station): accept ICAO or NWS code, fetch by ICAO, output station = NWS code. Unknown station → typed ValidationError before any fetch. climate() now resolves via it. The module also exposes assertStationFormat (STATION_CODE_RE guard) reused by FIX-2. Tests cover NWS-code input, ICAO input, the station-column value, and unknown/malformed station (no fetch). Co-Authored-By: Claude Fable 5 --- packages-ts/weather/src/_internal/stations.ts | 67 +++++++++++++++++++ packages-ts/weather/src/climate.ts | 25 +++++-- packages-ts/weather/tests/climate.test.ts | 51 +++++++++++++- 3 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 packages-ts/weather/src/_internal/stations.ts diff --git a/packages-ts/weather/src/_internal/stations.ts b/packages-ts/weather/src/_internal/stations.ts new file mode 100644 index 00000000..9d078a3b --- /dev/null +++ b/packages-ts/weather/src/_internal/stations.ts @@ -0,0 +1,67 @@ +// Phase 30 30-fix — shared station-format guard + registry resolution. +// +// Extracted so obs() and climate() enforce the SAME station contract without +// each inlining it (keeps the weather barrel small — one copy, two consumers). +// +// FIX-2 (security): Python treats STATION_CODE_RE (\A[A-Z]{3,4}\Z, applied +// after trim+uppercase) as a URL-security boundary — station ids flow into +// URL params and cache paths, so a malformed id like "../../etc/passwd" must +// be rejected BEFORE any fetch task is created. `assertStationFormat` is that +// boundary for the TS SDK. +// +// FIX-1 (parity): Python climate() resolves the input through the station +// registry, fetches by ICAO, and projects the 3-letter NWS `code` into every +// row's `station` column. `resolveStation` mirrors that contract exactly +// (mirrors the inlined helper in packages-ts/meta/src/mode2.ts). + +import { STATION_BY_CODE, STATION_BY_ICAO, ValidationError } from "@mostlyrightmd/core"; + +/** STATION_CODE_RE parity: 3–4 uppercase ASCII letters, whole string. */ +const STATION_CODE_RE = /^[A-Z]{3,4}$/; + +/** + * Return `station` trimmed+uppercased after asserting it matches + * STATION_CODE_RE. Throws a typed {@link ValidationError} for anything that is + * not 3–4 uppercase letters (path separators, whitespace, non-ASCII, empty). + * + * This is the URL/path security boundary — call it BEFORE creating any fetch + * task so a malformed id (e.g. `"../../etc/passwd"`) never reaches the network. + */ +export function assertStationFormat(station: string): string { + const normalized = typeof station === "string" ? station.trim().toUpperCase() : ""; + if (!STATION_CODE_RE.test(normalized)) { + throw new ValidationError( + `station=${JSON.stringify(station)} does not match STATION_CODE_RE (3-4 uppercase letters)`, + ); + } + return normalized; +} + +/** Resolved station identity: the 3-letter NWS `code` + the 4-letter `icao`. */ +export interface ResolvedStation { + readonly code: string; + readonly icao: string; +} + +/** + * Resolve a station identifier (3-letter NWS code OR 4-letter ICAO) to its + * registry record. Mirrors Python `_resolve_station`: fetch by `.icao`, project + * `.code` (the 3-letter NWS code) into output rows. + * + * The input is format-guarded first (FIX-2), so `resolveStation` never sees an + * unsafe id. Unknown stations raise a typed {@link ValidationError}. + */ +export function resolveStation(input: string): ResolvedStation { + const raw = assertStationFormat(input); + // Match by ICAO, by NWS code, or — for a 4-letter "K"-prefixed ICAO whose + // registry key is the stripped 3-letter code (e.g. "KNYC" → "NYC") — by the + // stripped code. Both maps reference the same StationInfo records. + let hit = STATION_BY_ICAO.get(raw) ?? STATION_BY_CODE.get(raw); + if (hit === undefined && raw.startsWith("K") && raw.length === 4) { + hit = STATION_BY_CODE.get(raw.slice(1)); + } + if (hit !== undefined && hit.code !== null) { + return { code: hit.code, icao: hit.icao }; + } + throw new ValidationError(`unknown station ${JSON.stringify(input)} — not in registry`); +} diff --git a/packages-ts/weather/src/climate.ts b/packages-ts/weather/src/climate.ts index ed6c62b0..2eac17b9 100644 --- a/packages-ts/weather/src/climate.ts +++ b/packages-ts/weather/src/climate.ts @@ -18,6 +18,7 @@ // `source` stays the truthful endpoint tag `"iem"`. import { downloadCliRange } from "./_fetchers/iem-cli.js"; +import { resolveStation } from "./_internal/stations.js"; import { mergeClimate, parseCliResponse } from "./_parsers/cli.js"; import type { ClimateFrameSource, @@ -60,11 +61,15 @@ function stampFrameSource(rows: ReadonlyArray): ClimateResult { * download → parse → frozen `mergeClimate` dedup), window-trims to * `[start, end]`, and projects onto the public JOIN vocabulary. * - * @param station ICAO code (e.g. "KNYC") — IEM CLI expects a 4-letter ICAO. + * @param station ICAO code (e.g. "KNYC") OR 3-letter NWS code (e.g. "NYC"). + * Resolved through the station registry: fetched by ICAO, + * projected as the NWS code into each row's `station` (matches + * Python `climate()`). * @param start Inclusive start date, ISO `YYYY-MM-DD`. * @param end Inclusive end date, ISO `YYYY-MM-DD`. * @param opts See {@link ClimateOptions}. * @returns {@link ClimateResult}: projected ClimateRow[] + `frameSource`. + * @throws ValidationError on a malformed/unknown station (before any fetch). * @throws Error on malformed dates, `start > end`, or `source="acis"` * (no ACIS leg in this code path). */ @@ -87,15 +92,23 @@ export async function climate( ); } - const stationCode = station.trim().toUpperCase(); + // FIX-1: resolve the input through the station registry to match Python + // `climate()` — fetch by the 4-letter ICAO (IEM CLI requires it), project the + // 3-letter NWS `code` into every row's `station` column. A 3-letter NWS code + // input (e.g. "NYC") resolves to "KNYC" for the fetch; the row `station` + // stays "NYC" so cross-SDK consumers joining on `station` get matching keys. + // `resolveStation` also enforces STATION_CODE_RE (FIX-2) BEFORE any fetch — + // an unknown / malformed station throws a typed ValidationError with no + // network dispatch. + const resolved = resolveStation(station); // Year-granular fetch (IEM cli.py has no partial-year request); trim later. const rangeOpts: { politenessMs?: number; signal?: AbortSignal } = {}; if (opts.politenessMs !== undefined) rangeOpts.politenessMs = opts.politenessMs; if (opts.signal !== undefined) rangeOpts.signal = opts.signal; - const raw = await downloadCliRange(stationCode, yearOf(start), yearOf(end), rangeOpts); + const raw = await downloadCliRange(resolved.icao, yearOf(start), yearOf(end), rangeOpts); - const parsed = parseCliResponse(raw, stationCode); + const parsed = parseCliResponse(raw, resolved.icao); // Frozen `(station, date)` dedup — keeps the settlement `final` (D-15). const deduped = mergeClimate(parsed); @@ -106,7 +119,9 @@ export async function climate( if (r.observation_date < start || r.observation_date > end) continue; rows.push({ date: r.observation_date, - station: r.station_code, + // Project the resolved NWS code (matches Python `info.code`), NOT the + // parser's station_code — keeps the `station` join key cross-SDK stable. + station: resolved.code, cli_high_f: r.high_temp_f, cli_low_f: r.low_temp_f, cli_report_type: r.report_type, diff --git a/packages-ts/weather/tests/climate.test.ts b/packages-ts/weather/tests/climate.test.ts index 16e1e3ae..c6a4eb4f 100644 --- a/packages-ts/weather/tests/climate.test.ts +++ b/packages-ts/weather/tests/climate.test.ts @@ -57,7 +57,9 @@ describe("climate — column contract", () => { ].sort(), ); expect(row.date).toBe("2026-01-02"); - expect(row.station).toBe("KNYC"); + // FIX-1: `station` is the resolved 3-letter NWS code (matches Python + // `info.code`), NOT the ICAO input carried through. + expect(row.station).toBe("NYC"); expect(row.cli_high_f).toBe(40); expect(row.cli_low_f).toBe(25); expect(row.cli_report_type).toBe("final"); @@ -66,6 +68,53 @@ describe("climate — column contract", () => { }); }); +describe("climate — station registry resolution (FIX-1 parity)", () => { + it("ICAO input 'KNYC' fetches by ICAO and projects NWS code 'NYC'", async () => { + const spy = vi + .spyOn(cliFetcher, "downloadCliRange") + .mockResolvedValue([cliRecord("2026-01-02", 40, 25)]); + const out = await climate("KNYC", "2026-01-01", "2026-01-03"); + // fetched by the 4-letter ICAO (first positional arg to downloadCliRange) + expect(spy).toHaveBeenCalledWith("KNYC", 2026, 2026, expect.anything()); + // row `station` is the resolved 3-letter NWS code, not the ICAO input + expect(out[0]?.station).toBe("NYC"); + }); + + it("NWS-code input 'NYC' resolves to ICAO 'KNYC' for the fetch", async () => { + const spy = vi + .spyOn(cliFetcher, "downloadCliRange") + .mockResolvedValue([cliRecord("2026-01-02", 40, 25)]); + const out = await climate("NYC", "2026-01-01", "2026-01-03"); + // IEM CLI requires a 4-letter ICAO — a 3-letter NWS code must be resolved + // to "KNYC" before the fetch (the pre-fix bug sent "NYC" verbatim). + expect(spy).toHaveBeenCalledWith("KNYC", 2026, 2026, expect.anything()); + expect(out[0]?.station).toBe("NYC"); + }); + + it("lowercase / untrimmed input normalizes then resolves", async () => { + const spy = vi + .spyOn(cliFetcher, "downloadCliRange") + .mockResolvedValue([cliRecord("2026-01-02", 40, 25)]); + const out = await climate(" knyc ", "2026-01-01", "2026-01-03"); + expect(spy).toHaveBeenCalledWith("KNYC", 2026, 2026, expect.anything()); + expect(out[0]?.station).toBe("NYC"); + }); + + it("unknown station throws a typed ValidationError before any fetch", async () => { + const spy = vi.spyOn(cliFetcher, "downloadCliRange"); + await expect(climate("ZZZ", "2026-01-01", "2026-01-03")).rejects.toThrow(/unknown station/); + expect(spy).not.toHaveBeenCalled(); + }); + + it("malformed station (path traversal) throws before any fetch", async () => { + const spy = vi.spyOn(cliFetcher, "downloadCliRange"); + await expect(climate("../../etc/passwd", "2026-01-01", "2026-01-03")).rejects.toThrow( + /STATION_CODE_RE/, + ); + expect(spy).not.toHaveBeenCalled(); + }); +}); + describe("climate — window-trim", () => { it("drops rows outside [start, end] from the year-granular fetch", async () => { // The fetcher returns a whole year; climate() must trim to the window. From 782d501e566463060c09bba4ef2a96931e2295e7 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:24:37 +0200 Subject: [PATCH 39/52] =?UTF-8?q?fix(30-review):=20Codex=20P1=20=E2=80=94?= =?UTF-8?q?=20TS=20station-format=20guard=20before=20network=20dispatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python treats STATION_CODE_RE (\A[A-Z]{3,4}\Z, after trim+uppercase) as a URL-security boundary; TS had no equivalent, so obs('../../etc/passwd') would build a fetch task with the raw id. Apply assertStationFormat (the shared guard from src/_internal/stations.ts) EAGERLY at the top of obs() — before any fetch task is created — so a malformed station throws a typed ValidationError with zero network dispatch. climate() gets the guard implicitly via FIX-1's resolveStation. Tests assert no fetch occurs for path-traversal and empty/whitespace station input. Co-Authored-By: Claude Fable 5 --- packages-ts/weather/src/obs.ts | 9 +++++++- .../weather/tests/obs.observation.test.ts | 21 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages-ts/weather/src/obs.ts b/packages-ts/weather/src/obs.ts index 4961c0b7..5abe28b6 100644 --- a/packages-ts/weather/src/obs.ts +++ b/packages-ts/weather/src/obs.ts @@ -22,6 +22,7 @@ import { DataAvailabilityError } from "@mostlyrightmd/core"; import { fetchAwcMetars } from "./_fetchers/awc.js"; import { downloadIemAsos } from "./_fetchers/iem-asos.js"; +import { assertStationFormat } from "./_internal/stations.js"; import { awcToObservation } from "./_parsers/awc.js"; import { parseIemCsv } from "./_parsers/iem.js"; import type { ObsFrameSource, ObsOptions, ObsResult, ObsRow, ObsStrategy } from "./obs.types.js"; @@ -244,11 +245,17 @@ function stampFrameSource(rows: ReadonlyArray, frameSource: ObsFrameSour * @throws TypeError when strategy or granularity is not in the accepted enum */ export async function obs( - station: string, + rawStation: string, fromDate: string, toDate: string, opts: ObsOptions = {}, ): Promise { + // FIX-2 (security): STATION_CODE_RE guard is a URL/path security boundary + // (Python parity). Apply it EAGERLY — before any fetch task is created — so a + // malformed id like "../../etc/passwd" throws a typed ValidationError and + // never reaches the network. Normalize (trim+uppercase) and reuse. + const station = assertStationFormat(rawStation); + const strategy = opts.strategy ?? "auto"; const source = opts.source ?? null; // Phase 30 30-04 (D-03): default grain is "observation" — the per-report diff --git a/packages-ts/weather/tests/obs.observation.test.ts b/packages-ts/weather/tests/obs.observation.test.ts index 81feb066..1287ac88 100644 --- a/packages-ts/weather/tests/obs.observation.test.ts +++ b/packages-ts/weather/tests/obs.observation.test.ts @@ -141,6 +141,27 @@ describe("obs — granularity='observation' grain", () => { }); }); +describe("obs — station-format guard (FIX-2 security)", () => { + it("malformed station (path traversal) throws before any fetch task is created", async () => { + const iemSpy = vi.spyOn(iemFetcher, "downloadIemAsos"); + const awcSpy = vi.spyOn(awcFetcher, "fetchAwcMetars"); + await expect(obs("../../etc/passwd", "2025-01-06", "2025-01-08")).rejects.toThrow( + /STATION_CODE_RE/, + ); + // No fetch of ANY leg must occur for a malformed station. + expect(iemSpy).not.toHaveBeenCalled(); + expect(awcSpy).not.toHaveBeenCalled(); + }); + + it("empty / whitespace station throws before any fetch", async () => { + const iemSpy = vi.spyOn(iemFetcher, "downloadIemAsos"); + const awcSpy = vi.spyOn(awcFetcher, "fetchAwcMetars"); + await expect(obs(" ", "2025-01-06", "2025-01-08")).rejects.toThrow(/STATION_CODE_RE/); + expect(iemSpy).not.toHaveBeenCalled(); + expect(awcSpy).not.toHaveBeenCalled(); + }); +}); + describe("obs — granularity='daily' not yet ported in TS", () => { it("throws DataAvailabilityError (never silently returns per-report rows)", async () => { // Should throw BEFORE any fetch — a no-data mock proves nothing was fetched. From 4a270d240626749c51d62fdb1548e0c4638c57a6 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:25:03 +0200 Subject: [PATCH 40/52] =?UTF-8?q?fix(30-review):=20WR-06=20=E2=80=94=20pin?= =?UTF-8?q?ned=20source=3D'awc'=20fetch=20failures=20fail=20loud?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchByStrategy pushed fetchAwcForWindow(...).catch(() => []) even when the caller pinned source='awc', so a network failure silently returned [] — the opposite of the fail-loud contract obs() applies to unwired ghcnh. Keep .catch(() => []) ONLY for the unpinned merge path; for a pinned awc query, wrap the fetch error in a typed DataAvailabilityError (reason='source_5xx') so the pinned caller learns the fetch failed. Tests: pinned awc + rejection → typed error; unpinned + AWC rejection → still returns IEM-only rows silently. Co-Authored-By: Claude Fable 5 --- packages-ts/weather/src/obs.ts | 30 +++++++++++++++++-- .../weather/tests/obs.observation.test.ts | 18 +++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages-ts/weather/src/obs.ts b/packages-ts/weather/src/obs.ts index 5abe28b6..fbcdf1db 100644 --- a/packages-ts/weather/src/obs.ts +++ b/packages-ts/weather/src/obs.ts @@ -200,12 +200,36 @@ async function fetchByStrategy( // overlapping callers reuse cached chunks. AWC is always the live // 168-hour endpoint regardless of strategy — the request shape doesn't // change with strategy in either Python or TS. - const wantsIem = source === null || source === undefined || source === "iem"; - const wantsAwc = source === null || source === undefined || source === "awc"; + const unpinned = source === null || source === undefined; + const wantsIem = unpinned || source === "iem"; + const wantsAwc = unpinned || source === "awc"; const tasks: Promise[] = []; if (wantsIem) tasks.push(fetchIemForWindow(station, fromDate, toDate, resolvedStrategy)); - if (wantsAwc) tasks.push(fetchAwcForWindow(station, fromDate, toDate).catch(() => [])); + if (wantsAwc) { + const awcTask = fetchAwcForWindow(station, fromDate, toDate); + // WR-06: on the UNPINNED merge path, a flaky AWC leg degrades gracefully to + // IEM-only rows (`.catch(() => [])`). But when the caller PINNED + // source="awc", a swallowed fetch error would silently return [] — the + // opposite of the "fail loud, never silent empty" contract obs() applies to + // unwired ghcnh. Let the pinned-awc error propagate (wrapped as a typed + // DataAvailabilityError) so a pinned caller learns the fetch failed. + if (unpinned) { + tasks.push(awcTask.catch(() => [])); + } else { + tasks.push( + awcTask.catch((err) => { + throw new DataAvailabilityError({ + reason: "source_5xx", + source: "awc", + hint: `pinned source='awc' fetch failed (fail-loud, not empty): ${ + err instanceof Error ? err.message : String(err) + }`, + }); + }), + ); + } + } const results = await Promise.all(tasks); return results.flat(); diff --git a/packages-ts/weather/tests/obs.observation.test.ts b/packages-ts/weather/tests/obs.observation.test.ts index 1287ac88..e36bcb4c 100644 --- a/packages-ts/weather/tests/obs.observation.test.ts +++ b/packages-ts/weather/tests/obs.observation.test.ts @@ -162,6 +162,24 @@ describe("obs — station-format guard (FIX-2 security)", () => { }); }); +describe("obs — pinned-awc fail-loud (FIX-4 / WR-06)", () => { + it("pinned source='awc' + fetch rejection → typed DataAvailabilityError", async () => { + vi.spyOn(awcFetcher, "fetchAwcMetars").mockRejectedValue(new Error("network down")); + await expect(obs("KNYC", "2025-01-06", "2025-01-06", { source: "awc" })).rejects.toBeInstanceOf( + DataAvailabilityError, + ); + }); + + it("unpinned + AWC rejection → still returns IEM-only rows silently", async () => { + mockIem([makeIemRow("2025-01-06 12:51", "40")]); + vi.spyOn(awcFetcher, "fetchAwcMetars").mockRejectedValue(new Error("network down")); + const out = await obs("KNYC", "2025-01-06", "2025-01-06"); + // AWC failure is swallowed on the merge path; IEM rows still come back. + expect(out.length).toBeGreaterThan(0); + for (const r of out) expect(r.source).toBe("iem"); + }); +}); + describe("obs — granularity='daily' not yet ported in TS", () => { it("throws DataAvailabilityError (never silently returns per-report rows)", async () => { // Should throw BEFORE any fetch — a no-data mock proves nothing was fetched. From 84fa40d3a3b9e74a1f58e1c33c6d9ed18321f65e Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:25:44 +0200 Subject: [PATCH 41/52] chore(30-fix): raise weather size budget 26->27 KB (station guard + pinned-awc fail-loud) The weather barrel had 40 B of headroom (25.96/26 KB). CR-01's registry resolution + Codex-P1's STATION_CODE_RE guard (shared src/_internal/ stations.ts) plus WR-06's pinned-awc fail-loud branch push the gzipped ESM to 26.02 KB. After aggressive trimming (compact error strings, map lookup, dropped param defaults) it remains ~17 B over 26 KB. Both are P1/P2 correctness+security fixes, so bump the budget rather than drop them. Co-Authored-By: Claude Fable 5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 053c8beb..1ba8ad1d 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ { "name": "@mostlyrightmd/weather (W1: AWC+CLI only, gzipped ESM)", "path": "packages-ts/weather/dist/index.mjs", - "limit": "26 KB" + "limit": "27 KB" }, { "name": "@mostlyrightmd/markets (W1: Kalshi NHIGH/NLOW, gzipped ESM)", From 4cb7f6c77e1fb3efe7d5406ee5b70074c55a8448 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 02:57:31 +0200 Subject: [PATCH 42/52] =?UTF-8?q?fix(30-review-r2):=20A=20=E2=80=94=20stat?= =?UTF-8?q?ion-format=20guard=20on=20all=20exported=20weather=20network=20?= =?UTF-8?q?surfaces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 1 added assertStationFormat to obs() + climate() only. Codex verified three more exported surfaces reached the network without a format guard: - live latest() default AWC path (fetchAwcLatest -> normalizeStation only trim/uppercase/K-prefix; the IEM path already guarded its stripped code) - dailyExtremes() (AWC leg fetchAwcMetars([station]) uses encodeURIComponent only; the IEM leg validated but AWC did not) - iemMosForecasts() (encodeURIComponent only into the MOS URL) Each now calls assertStationFormat(station) at the public entry, before any fetch task is created, so a malformed id throws a typed ValidationError and never reaches the network. Grep sweep for other station-string URL interpolation surfaced no additional weather cases (earnings hosted URLs interpolate ticker/callId/token, not stations). Tests per surface assert malformed input throws before fetch (spy not called). Co-Authored-By: Claude Fable 5 --- packages-ts/weather/src/dailyExtremes.ts | 10 ++++++++++ packages-ts/weather/src/forecasts/iem-mos.ts | 7 +++++++ packages-ts/weather/src/live/latest.ts | 10 ++++++++++ .../weather/tests/dailyExtremes.test.ts | 19 ++++++++++++++++++ .../weather/tests/forecasts/iem-mos.test.ts | 20 +++++++++++++++++++ packages-ts/weather/tests/live/latest.test.ts | 11 ++++++++++ 6 files changed, 77 insertions(+) diff --git a/packages-ts/weather/src/dailyExtremes.ts b/packages-ts/weather/src/dailyExtremes.ts index 840c5d73..f1305938 100644 --- a/packages-ts/weather/src/dailyExtremes.ts +++ b/packages-ts/weather/src/dailyExtremes.ts @@ -27,6 +27,7 @@ import { } from "@mostlyrightmd/core/discovery"; import { fetchAwcMetars } from "./_fetchers/awc.js"; import { downloadIemAsos } from "./_fetchers/iem-asos.js"; +import { assertStationFormat } from "./_internal/stations.js"; import { awcToObservation } from "./_parsers/awc.js"; import { parseIemCsv } from "./_parsers/iem.js"; import type { @@ -237,6 +238,15 @@ export async function dailyExtremes( toDate: string, opts: DailyExtremesOptions = {}, ): Promise> { + // FIX-A (security): STATION_CODE_RE guard at the URL/path boundary. The + // station string flows unmodified into both the AWC fetch + // (`fetchAwcMetars([station])`, encodeURIComponent only) and the IEM fetch + // (`downloadIemAsos` — which validates, but the AWC leg does not). Guard the + // final string BEFORE any fetch task is created so a malformed id never + // reaches the network. `lookupStation` also rejects unknown stations, but + // that runs a registry scan on the raw string — do the cheap format guard + // first so injection-shaped ids throw a typed ValidationError immediately. + assertStationFormat(station); const { tz, isUs } = lookupStation(station); const merge = opts.merge ?? "live_v1"; diff --git a/packages-ts/weather/src/forecasts/iem-mos.ts b/packages-ts/weather/src/forecasts/iem-mos.ts index 8296c39b..d27df612 100644 --- a/packages-ts/weather/src/forecasts/iem-mos.ts +++ b/packages-ts/weather/src/forecasts/iem-mos.ts @@ -4,6 +4,7 @@ // Endpoint: https://mesonet.agron.iastate.edu/api/1/mos.json // CORS: OPEN per IEM ASOS posture (see `.planning/research/FORECAST-CORS-MATRIX.md`). +import { assertStationFormat } from "../_internal/stations.js"; import type { IemMosModel, IemMosOptions, IemMosRow, IemMosSource } from "./types.js"; const IEM_MOS_URL = "https://mesonet.agron.iastate.edu/api/1/mos.json"; @@ -179,6 +180,12 @@ export async function iemMosForecasts( toDate: string, opts: IemMosOptions = {}, ): Promise { + // FIX-A (security): STATION_CODE_RE guard at the URL boundary. `station` is + // interpolated into the IEM MOS URL via encodeURIComponent only, which + // percent-escapes but does not reject a malformed id — guard the input + // BEFORE building any URL / issuing any fetch so an injection-shaped station + // throws a typed ValidationError and the fetchFn spy is never called. + assertStationFormat(station); const model = opts.model ?? "nbe"; if (!SUPPORTED_MODELS.has(model)) { throw new Error( diff --git a/packages-ts/weather/src/live/latest.ts b/packages-ts/weather/src/live/latest.ts index 084d0ffc..5041f655 100644 --- a/packages-ts/weather/src/live/latest.ts +++ b/packages-ts/weather/src/live/latest.ts @@ -6,6 +6,7 @@ import { NoLiveDataError } from "@mostlyrightmd/core"; +import { assertStationFormat } from "../_internal/stations.js"; import { fetchLatest, normalizeStation, pickMostRecent } from "./_fetch.js"; import { type LiveSource, sourceTag, validateSource } from "./sources.js"; import type { LiveObservation } from "./types.js"; @@ -35,6 +36,15 @@ export interface LatestOptions { */ export async function latest(station: string, opts: LatestOptions = {}): Promise { const src: LiveSource = validateSource(opts.source ?? undefined); + // FIX-A (security): STATION_CODE_RE guard at the URL/path boundary. The + // default AWC path (`fetchAwcLatest` → `normalizeStation`) only trims, + // uppercases and K-prefixes — it never rejects a malformed id, so a station + // like "KN&data=foo" or "../../etc/passwd" would flow straight into the AWC + // URL. Guard the input BEFORE any fetch task is created. A 3-letter code that + // passes the guard becomes a 4-letter "K"-prefixed ICAO downstream, which is + // still 3-4 uppercase letters (safe); the IEM path already guards its own + // stripped code in `_fetch.ts`. Throwing here means the spy is never called. + assertStationFormat(station); const rows = await fetchLatest(station, src); const picked = pickMostRecent(rows); if (picked === null) { diff --git a/packages-ts/weather/tests/dailyExtremes.test.ts b/packages-ts/weather/tests/dailyExtremes.test.ts index bef618d4..cd2b6afa 100644 --- a/packages-ts/weather/tests/dailyExtremes.test.ts +++ b/packages-ts/weather/tests/dailyExtremes.test.ts @@ -25,6 +25,25 @@ describe("dailyExtremes — station registry lookup", () => { }); }); +describe("dailyExtremes — station-format guard (FIX-A)", () => { + it("rejects a malformed station before any fetch is issued", async () => { + // Round-1 guarded obs()/climate() only. dailyExtremes() passes the raw + // station into the AWC leg (`fetchAwcMetars([station])`, encodeURIComponent + // only) — the injection-shaped id must be rejected before any fetch task. + const awcSpy = vi.spyOn(awcFetcher, "fetchAwcMetars"); + const iemSpy = vi.spyOn(iemFetcher, "downloadIemAsos"); + await expect(dailyExtremes("../../etc/passwd", "2025-01-06", "2025-01-06")).rejects.toThrow( + /STATION_CODE_RE/, + ); + await expect(dailyExtremes("KN&data=foo", "2025-01-06", "2025-01-06")).rejects.toThrow( + /STATION_CODE_RE/, + ); + expect(awcSpy).not.toHaveBeenCalled(); + expect(iemSpy).not.toHaveBeenCalled(); + vi.restoreAllMocks(); + }); +}); + describe("dailyExtremes — merge mode dispatch", () => { it("rejects unknown merge mode at runtime", async () => { // EGLL is in the international STATIONS registry. diff --git a/packages-ts/weather/tests/forecasts/iem-mos.test.ts b/packages-ts/weather/tests/forecasts/iem-mos.test.ts index 331c07cc..c13328b1 100644 --- a/packages-ts/weather/tests/forecasts/iem-mos.test.ts +++ b/packages-ts/weather/tests/forecasts/iem-mos.test.ts @@ -41,6 +41,26 @@ describe("iemMosForecasts", () => { expect(rows[0]?.source).toBe("iem.archive"); }); + it("FIX-A: rejects a malformed station before any fetch is issued", async () => { + // `station` is interpolated into the IEM MOS URL via encodeURIComponent + // only — the format guard now runs at the entry, before any URL is built + // or fetchFn is called. + const fetchFn = vi.fn(async () => ({ + ok: true, + status: 200, + async json() { + return { data: [] }; + }, + })) as unknown as typeof fetch; + await expect( + iemMosForecasts("../../etc/passwd", "2026-05-01", "2026-05-01", { fetchFn }), + ).rejects.toThrow(/STATION_CODE_RE/); + await expect( + iemMosForecasts("KN&station=foo", "2026-05-01", "2026-05-01", { fetchFn }), + ).rejects.toThrow(/STATION_CODE_RE/); + expect(fetchFn).not.toHaveBeenCalled(); + }); + it("converts F→C correctly", async () => { const fetchFn = makeMockFetch({ data: [SAMPLE_ROW] }); const rows = await iemMosForecasts("KNYC", "2026-05-01", "2026-05-01", { diff --git a/packages-ts/weather/tests/live/latest.test.ts b/packages-ts/weather/tests/live/latest.test.ts index 5b676489..f9787691 100644 --- a/packages-ts/weather/tests/live/latest.test.ts +++ b/packages-ts/weather/tests/live/latest.test.ts @@ -170,6 +170,17 @@ describe("latest()", () => { expect(fetchSpy).not.toHaveBeenCalled(); }); + it("FIX-A: default AWC path rejects malformed station before fetch", async () => { + // Round-1 added `assertStationFormat` to obs()/climate() only. The default + // `latest()` AWC path (`fetchAwcLatest` → `normalizeStation`) only + // trim/uppercase/K-prefixes and never rejected a malformed id, so a station + // like "../../etc/passwd" or "KN&data=foo" flowed into the AWC URL. The + // guard now runs at the `latest()` entry, before any fetch task is created. + await expect(latest("../../etc/passwd")).rejects.toThrow(/STATION_CODE_RE/); + await expect(latest("KN&data=foo")).rejects.toThrow(/STATION_CODE_RE/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + it("IEM path uses exclusive day2 (one-day window per poll)", async () => { // Regression for iter-3 codex finding: `fetchIemLatest` previously passed // day1==day2 to `buildIemUrl`, but the IEM endpoint treats day2 as From f14a182b8b55010436db87c21acc047d365b0a02 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 03:05:07 +0200 Subject: [PATCH 43/52] =?UTF-8?q?fix(30-review-r2):=20B=20=E2=80=94=20pinn?= =?UTF-8?q?ed=20source=3D'awc'=20genuinely=20fail-loud?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The real AWC fetcher (awc.ts) swallows every non-OK/network failure into [] (graceful-degradation contract), so round-1's task-level .catch wrap on the pinned source='awc' path NEVER fired in the real failure mode — a pinned-awc fetch failure silently returned an empty result instead of failing loud. Fetcher: add an opt-in { failLoud: true } flag. In fail-loud mode a genuine FAILURE (non-OK HTTP after retries, or a network error) THROWS a typed AwcFetchError (carrying the underlying error as cause) instead of returning []. An empty-but-SUCCESSFUL response (HTTP 200 zero rows / non-array body) stays [] in BOTH modes — that is data truth, not a failure. All tolerant callers (unpinned merge, latest(), dailyExtremes()) keep the swallow-into-[] behavior. obs.ts: the pinned source='awc' path now calls the fetcher with failLoud=true and wraps the real AwcFetchError as DataAvailabilityError(reason=source_5xx); the unpinned path stays tolerant. Tests: rewritten to exercise the REAL mode via a global-fetch mock (round-1 mocked fetchAwcMetars to reject, which never happened in production). Fetcher unit tests cover 5xx/400/network → throw, and 200-empty/non-array → []. obs() tests cover pinned+400 → DataAvailabilityError, unpinned+400 → IEM-only rows, pinned+200-empty → empty result (no throw). A beforeEach mock-restore was added to the obs fail-loud block because earlier describe blocks leak fetchAwcMetars spies (vitest config sets no restoreMocks). Co-Authored-By: Claude Fable 5 --- packages-ts/weather/src/_fetchers/awc.ts | 66 ++++++++++++++++--- packages-ts/weather/src/obs.ts | 28 +++++--- packages-ts/weather/tests/awc.test.ts | 57 ++++++++++++++++ .../weather/tests/obs.observation.test.ts | 65 ++++++++++++++++-- 4 files changed, 192 insertions(+), 24 deletions(-) diff --git a/packages-ts/weather/src/_fetchers/awc.ts b/packages-ts/weather/src/_fetchers/awc.ts index 307ac449..e169ffa0 100644 --- a/packages-ts/weather/src/_fetchers/awc.ts +++ b/packages-ts/weather/src/_fetchers/awc.ts @@ -85,6 +85,31 @@ export interface AwcMetarRaw { export interface FetchAwcOptions extends FetchWithRetryOptions { /** Lookback window in hours. Default 168 (max). Values above `AWC_MAX_HOURS` are clamped. */ hours?: number; + /** + * FIX-B (fail-loud): when `true`, a genuine FAILURE (non-OK HTTP after retry + * exhaustion, or a network error) THROWS the underlying typed error instead + * of returning `[]`. Callers that pin `source="awc"` need this so a fetch + * failure surfaces loudly rather than masquerading as an empty result. + * + * The default (`false`/omitted) preserves the historical tolerant contract: + * failures degrade to `[]` so merged/latest()/dailyExtremes callers keep + * running when AWC is down. + * + * Note: a genuinely-empty-but-SUCCESSFUL response (HTTP 200 with zero rows, + * or a 200 body that is not a JSON array) is NOT a failure — it returns `[]` + * in BOTH modes. Empty-success is data truth; only failures throw. + */ + failLoud?: boolean; +} + +/** + * Thrown by {@link fetchAwcMetars} when called with `failLoud: true` and the + * fetch genuinely FAILS (non-OK HTTP after retries, or a network error). + * Carries the underlying error as `cause` so callers can inspect the HTTP + * status / network reason. Empty-but-successful responses never raise this. + */ +export class AwcFetchError extends Error { + override readonly name = "AwcFetchError"; } /** @@ -94,12 +119,19 @@ export interface FetchAwcOptions extends FetchWithRetryOptions { * `awcToObservation` from `../_parsers/awc.ts` to map each entry to the * observation row schema. * - * Behaviour mirrors Python `fetch_awc_metars`: + * Behaviour mirrors Python `fetch_awc_metars` (tolerant default mode): * - empty `stationIcaos` → return `[]` immediately, no HTTP issued * - 4xx after retry exhaustion → `[]` * - 5xx / network errors after retry budget → `[]` * - non-array JSON body → `[]` * - NEVER throws (callers want graceful degradation) + * + * FIX-B: pass `{ failLoud: true }` to opt into fail-loud mode — a genuine + * FAILURE (non-OK HTTP after retries, or a network error) THROWS a typed + * {@link AwcFetchError} instead of returning `[]`. Empty-but-successful + * responses (HTTP 200 zero rows / non-array body) still return `[]` in both + * modes. Only the pinned `source="awc"` path in `obs()` uses this; all other + * callers keep the tolerant swallow-into-`[]` contract. */ export async function fetchAwcMetars( stationIcaos: ReadonlyArray, @@ -120,29 +152,47 @@ export async function fetchAwcMetars( const { hours: _consumed, ...retryOpts } = opts; void _consumed; + const failLoud = opts.failLoud === true; + let response: Response; try { response = await fetchWithRetry(url, retryOpts); } catch (err) { - // Any TherminalError (404, 400, 401, 403, 429-after-exhaustion, 5xx-after- - // exhaustion) OR raw network error → return [] (mirror Python). This is - // the "graceful degradation" contract; orchestration layer decides whether - // to surface a SourceUnavailableError instead. - if (err instanceof TherminalError) { - return []; - } // Re-raise abort errors so callers that pass AbortSignals can cancel. + // (Applies in BOTH modes — a caller-initiated abort is not a data failure.) if (err instanceof DOMException && (err.name === "AbortError" || err.name === "TimeoutError")) { // For caller-initiated aborts we re-throw; otherwise the timeout was // composed in by fetchWithRetry and would already have been retried. if (opts.signal?.aborted) { throw err; } + if (failLoud) { + throw new AwcFetchError(`AWC fetch failed (timeout/abort) for ${url}`, { cause: err }); + } + return []; + } + // FIX-B (fail-loud): a TherminalError (404, 400, 401, 403, 429-after- + // exhaustion, 5xx-after-exhaustion) OR a raw network error is a genuine + // FAILURE. In tolerant mode (default) → return [] (mirror Python graceful + // degradation). In fail-loud mode → THROW a typed AwcFetchError so a pinned + // caller learns the fetch failed instead of getting a silent empty result. + if (failLoud) { + throw new AwcFetchError( + `AWC fetch failed for ${url}: ${err instanceof Error ? err.message : String(err)}`, + { cause: err }, + ); + } + if (err instanceof TherminalError) { return []; } return []; } + // FIX-B: the response was SUCCESSFUL (HTTP OK) past this point. An empty + // array, an unparseable body, or a non-array body is empty-success (data + // truth: "the request worked, there were no usable rows") — NOT a failure. + // These return [] in BOTH tolerant and fail-loud modes; only response-level + // failures (handled in the catch above) throw under failLoud. let data: unknown; try { data = await response.json(); diff --git a/packages-ts/weather/src/obs.ts b/packages-ts/weather/src/obs.ts index fbcdf1db..45a7cabd 100644 --- a/packages-ts/weather/src/obs.ts +++ b/packages-ts/weather/src/obs.ts @@ -174,8 +174,14 @@ async function fetchAwcForWindow( station: string, fromDate: string, toDate: string, + failLoud = false, ): Promise { - const raw = await fetchAwcMetars([station]); + // FIX-B: `failLoud` propagates to the fetcher. Tolerant callers (unpinned + // merge, latest(), dailyExtremes()) leave it false so a flaky AWC degrades to + // []. The pinned source="awc" path sets it true so a genuine fetch FAILURE + // throws AwcFetchError (which obs() wraps as DataAvailabilityError) instead + // of the fetcher silently swallowing the error into []. + const raw = await fetchAwcMetars([station], failLoud ? { failLoud: true } : {}); const out: ObsRow[] = []; for (const r of raw) { const obs = awcToObservation(r); @@ -207,16 +213,20 @@ async function fetchByStrategy( const tasks: Promise[] = []; if (wantsIem) tasks.push(fetchIemForWindow(station, fromDate, toDate, resolvedStrategy)); if (wantsAwc) { - const awcTask = fetchAwcForWindow(station, fromDate, toDate); - // WR-06: on the UNPINNED merge path, a flaky AWC leg degrades gracefully to - // IEM-only rows (`.catch(() => [])`). But when the caller PINNED - // source="awc", a swallowed fetch error would silently return [] — the - // opposite of the "fail loud, never silent empty" contract obs() applies to - // unwired ghcnh. Let the pinned-awc error propagate (wrapped as a typed - // DataAvailabilityError) so a pinned caller learns the fetch failed. + // WR-06 / FIX-B: on the UNPINNED merge path, a flaky AWC leg degrades + // gracefully to IEM-only rows — the fetcher stays in TOLERANT mode + // (`failLoud=false`) and swallows failures into [], then `.catch(() => [])` + // covers any residual throw. When the caller PINNED source="awc", the + // fetcher runs in FAIL-LOUD mode: a genuine fetch FAILURE throws + // AwcFetchError, which round-1's task-level `.catch` NEVER saw because the + // real fetcher always swallowed into [] (Codex round-2 finding). We now + // wrap that real throw as a typed DataAvailabilityError so a pinned caller + // learns the fetch failed. An empty-but-successful AWC response (HTTP 200, + // zero rows) stays an empty result — that is data truth, not a failure. if (unpinned) { - tasks.push(awcTask.catch(() => [])); + tasks.push(fetchAwcForWindow(station, fromDate, toDate).catch(() => [])); } else { + const awcTask = fetchAwcForWindow(station, fromDate, toDate, /* failLoud */ true); tasks.push( awcTask.catch((err) => { throw new DataAvailabilityError({ diff --git a/packages-ts/weather/tests/awc.test.ts b/packages-ts/weather/tests/awc.test.ts index 197ea7ae..5dd84aa5 100644 --- a/packages-ts/weather/tests/awc.test.ts +++ b/packages-ts/weather/tests/awc.test.ts @@ -180,6 +180,63 @@ describe("fetchAwcMetars — retry + error handling", () => { }, 10_000); }); +describe("fetchAwcMetars — fail-loud mode (FIX-B)", () => { + let fetchSpy: MockInstance; + + beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch"); + }); + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it("THROWS AwcFetchError on persistent 5xx (failLoud=true)", async () => { + fetchSpy.mockResolvedValue(errorResponse(500)); + await expect( + fetchAwcMetars(["KNYC"], { failLoud: true, baseDelayMs: 1, maxRetries: 2 }), + ).rejects.toMatchObject({ name: "AwcFetchError" }); + }, 10_000); + + it("THROWS AwcFetchError on non-retryable 400 (failLoud=true)", async () => { + fetchSpy.mockResolvedValueOnce(errorResponse(400)); + await expect( + fetchAwcMetars(["KNYC"], { failLoud: true, baseDelayMs: 1, maxRetries: 1 }), + ).rejects.toMatchObject({ name: "AwcFetchError" }); + }); + + it("THROWS AwcFetchError on network rejection (failLoud=true)", async () => { + fetchSpy.mockRejectedValue(new TypeError("network down")); + await expect( + fetchAwcMetars(["KNYC"], { failLoud: true, baseDelayMs: 1, maxRetries: 2 }), + ).rejects.toMatchObject({ name: "AwcFetchError" }); + }, 10_000); + + it("preserves the underlying error as `cause`", async () => { + fetchSpy.mockResolvedValueOnce(errorResponse(400)); + try { + await fetchAwcMetars(["KNYC"], { failLoud: true, baseDelayMs: 1, maxRetries: 1 }); + throw new Error("expected throw"); + } catch (e) { + expect((e as Error).name).toBe("AwcFetchError"); + expect((e as { cause?: unknown }).cause).toBeInstanceOf(Error); + } + }); + + it("HTTP-200 empty array is empty-SUCCESS → [] (no throw, both modes)", async () => { + fetchSpy.mockResolvedValue(jsonResponse([])); + const loud = await fetchAwcMetars(["KNYC"], { failLoud: true }); + expect(loud).toEqual([]); + const tolerant = await fetchAwcMetars(["KNYC"]); + expect(tolerant).toEqual([]); + }); + + it("HTTP-200 non-array body is empty-SUCCESS → [] (no throw under failLoud)", async () => { + fetchSpy.mockResolvedValue(jsonResponse({ error: "unexpected shape" })); + const rows = await fetchAwcMetars(["KNYC"], { failLoud: true }); + expect(rows).toEqual([]); + }); +}); + // --------------------------------------------------------------------------- // Parser unit tests — visibility // --------------------------------------------------------------------------- diff --git a/packages-ts/weather/tests/obs.observation.test.ts b/packages-ts/weather/tests/obs.observation.test.ts index e36bcb4c..ecb394f6 100644 --- a/packages-ts/weather/tests/obs.observation.test.ts +++ b/packages-ts/weather/tests/obs.observation.test.ts @@ -13,7 +13,7 @@ // // All HTTP is mocked via vi.spyOn on the IEM ASOS fetcher (no network). -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { DataAvailabilityError } from "@mostlyrightmd/core"; @@ -162,22 +162,73 @@ describe("obs — station-format guard (FIX-2 security)", () => { }); }); -describe("obs — pinned-awc fail-loud (FIX-4 / WR-06)", () => { - it("pinned source='awc' + fetch rejection → typed DataAvailabilityError", async () => { - vi.spyOn(awcFetcher, "fetchAwcMetars").mockRejectedValue(new Error("network down")); +describe("obs — pinned-awc fail-loud (FIX-B / WR-06)", () => { + // FIX-B rewrite: round-1's tests mocked `fetchAwcMetars` to REJECT, but the + // REAL fetcher swallows non-OK/network failures into [] (awc.ts) — so the + // rejection never happened in production and the task-level `.catch` never + // fired. These tests exercise the real failure mode by mocking global + // `fetch` (the transport underneath `fetchWithRetry`), proving the fetcher's + // new fail-loud path throws on genuine failures while empty-success stays []. + // + // HTTP 400 is used for the failure case: it is a non-retryable status, so + // `fetchWithRetry` throws immediately (no backoff delay → fast test) and the + // real fetcher catch-block runs — exactly the swallow-into-[] path round-1 + // could not reach. + // + // beforeEach restore is REQUIRED: earlier describe blocks in this file spy + // `fetchAwcMetars` with `mockResolvedValue([])` and never restore, and the + // vitest config sets no `restoreMocks`. Without clearing those leaked spies + // first, `fetchAwcMetars` would still return [] and our real-fetch mock would + // never run. Restore before each test, then install a fresh global-fetch mock. + beforeEach(() => vi.restoreAllMocks()); + afterEach(() => vi.restoreAllMocks()); + + function mockFetchStatus(status: number, body: string): void { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(body, { status, headers: { "content-type": "application/json" } }), + ); + } + + it("pinned source='awc' + real HTTP-400 failure → typed DataAvailabilityError", async () => { + mockFetchStatus(400, "bad request"); await expect(obs("KNYC", "2025-01-06", "2025-01-06", { source: "awc" })).rejects.toBeInstanceOf( DataAvailabilityError, ); }); - it("unpinned + AWC rejection → still returns IEM-only rows silently", async () => { + it("pinned source='awc' failure hint marks it fail-loud, reason=source_5xx", async () => { + mockFetchStatus(400, "bad request"); + try { + await obs("KNYC", "2025-01-06", "2025-01-06", { source: "awc" }); + throw new Error("expected throw"); + } catch (e) { + expect(e).toBeInstanceOf(DataAvailabilityError); + const err = e as DataAvailabilityError; + expect(err.reason).toBe("source_5xx"); + expect(err.source).toBe("awc"); + expect(err.hint).toMatch(/fail-loud, not empty/); + } + }); + + it("unpinned + real AWC HTTP-400 failure → still returns IEM-only rows (swallow)", async () => { + // The IEM leg is mocked via downloadIemAsos; the AWC leg goes through the + // real fetcher (global fetch → 400) which, in TOLERANT mode, swallows into + // []. The merge therefore returns IEM-only rows without throwing. mockIem([makeIemRow("2025-01-06 12:51", "40")]); - vi.spyOn(awcFetcher, "fetchAwcMetars").mockRejectedValue(new Error("network down")); + mockFetchStatus(400, "bad request"); const out = await obs("KNYC", "2025-01-06", "2025-01-06"); - // AWC failure is swallowed on the merge path; IEM rows still come back. expect(out.length).toBeGreaterThan(0); for (const r of out) expect(r.source).toBe("iem"); }); + + it("pinned source='awc' + HTTP-200 empty body → empty result, NO throw (data truth)", async () => { + // Empty-but-successful is NOT a failure: the fetch worked, there were zero + // rows. A pinned-awc query must return an empty result here, never throw. + mockFetchStatus(200, "[]"); + const out = await obs("KNYC", "2025-01-06", "2025-01-06", { source: "awc" }); + expect(out).toEqual([]); + expect((out as ObsResult).frameSource).toBe("awc"); + }); }); describe("obs — granularity='daily' not yet ported in TS", () => { From 366928ff8cf322941c91bbbd5febf6a0a25b7fab Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 03:05:46 +0200 Subject: [PATCH 44/52] =?UTF-8?q?fix(30-review-r2):=20C=20=E2=80=94=20refr?= =?UTF-8?q?esh=20stale=20climate()=20subpath=20budget=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit index.ts:207 still said climate() is omitted from the root barrel because it exceeds the '26 KB' budget ('+274 B over'). The weather budget is now 27 KB, so climate() would fit. Update the comment: the subpath decision stands, but for budget HEADROOM + tree-shaking (root-barrel consumers should not pay for climate()'s CLI parse/merge tree unless they import it), not because it no longer fits. Comment-only change. Co-Authored-By: Claude Fable 5 --- packages-ts/weather/src/index.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages-ts/weather/src/index.ts b/packages-ts/weather/src/index.ts index 4ea242f7..f768ed00 100644 --- a/packages-ts/weather/src/index.ts +++ b/packages-ts/weather/src/index.ts @@ -203,10 +203,11 @@ export type { // // Deliberately NOT re-exported from this root barrel — surfaced ONLY via the // LEAN `@mostlyrightmd/weather/climate` subpath, mirroring the -// live/forecasts/hosted subpath pattern. Adding climate() to the root -// barrel exceeds the 26 KB TS-BUNDLE-01 budget (measured +274 B over). The -// subpath keeps the default browser bundle lean while making climate() fully -// importable: +// live/forecasts/hosted subpath pattern. The root barrel's TS-BUNDLE-01 budget +// is now 27 KB, so climate() would fit — but the subpath decision stands for +// budget HEADROOM and tree-shaking: root-barrel consumers should not pay for +// climate()'s CLI parsing/merge tree unless they import it. The subpath keeps +// the default browser bundle lean while making climate() fully importable: // // import { climate } from "@mostlyrightmd/weather/climate" // From d1554daff057bde221594b9772218af6bfdeb7c4 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 03:12:45 +0200 Subject: [PATCH 45/52] fix(30-review-r3): guard satelliteHosted station input before hosted URL build (STATION_CODE_RE) --- packages-ts/weather/src/hosted/satellite.ts | 2 ++ packages-ts/weather/tests/hosted.test.ts | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages-ts/weather/src/hosted/satellite.ts b/packages-ts/weather/src/hosted/satellite.ts index a38c894d..169e501f 100644 --- a/packages-ts/weather/src/hosted/satellite.ts +++ b/packages-ts/weather/src/hosted/satellite.ts @@ -27,6 +27,7 @@ // be snake_case for parity; the emitted `SatelliteRow` property names are // camelCase (matching the TS earnings-row convention in `_fetchers/earnings.ts`). +import { assertStationFormat } from "../_internal/stations.js"; import { type FetchLike, HostedConfigError, @@ -265,6 +266,7 @@ export async function satelliteHosted( const out: SatelliteRow[] = []; for (const station of stations) { if (station.trim() === "") continue; + assertStationFormat(station); const url = `${joinHostedUrl(baseUrl, "/satellite")}?${buildSatelliteQuery(station, options)}`; const payload = await hostedFetchJson(url, fetchOpts); for (const raw of extractRawRows(payload)) { diff --git a/packages-ts/weather/tests/hosted.test.ts b/packages-ts/weather/tests/hosted.test.ts index f14b5418..e3335961 100644 --- a/packages-ts/weather/tests/hosted.test.ts +++ b/packages-ts/weather/tests/hosted.test.ts @@ -304,6 +304,24 @@ describe("satelliteHosted — GET /satellite via WEATHER_HOSTED_URL + api key", expect(rows.map((r) => r.station)).toEqual(["KNYC", "KLGA"]); }); + it("rejects malformed station input before any fetch (STATION_CODE_RE guard)", async () => { + const fetchImpl = vi.fn(async () => mockResponse(200, {})); + for (const bad of ["../../etc/passwd", "KN&satellite=goes16", "K NYC"]) { + await expect( + satelliteHosted({ + hostedUrl: "https://w", + apiKey: "k", + station: bad, + start: "a", + end: "b", + satellite: "goes16", + fetchImpl: fetchImpl as never, + }), + ).rejects.toThrow(/STATION_CODE_RE/); + } + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it("throws HostedConfigError when WEATHER_HOSTED_URL is missing (before fetch)", async () => { const fetchImpl = vi.fn(async () => mockResponse(200, {})); await expect( From 517fb3b7b17320cc7ef9ea24f7d938148e11a73c Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 08:48:39 +0200 Subject: [PATCH 46/52] =?UTF-8?q?test(30-06):=20re-baseline=20parity=20cas?= =?UTF-8?q?e-4=20(KMIA)=20live=20fixture=20=E2=80=94=20v0.14.1=20plane=20d?= =?UTF-8?q?ecommissioned;=20labels/extremes=20byte-identical,=203-day=20up?= =?UTF-8?q?stream=20mean=20restatement=20documented?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - case_4 parquet re-baselined from fresh branch capture 2026-07-09 (dataset() default path, isolated cache, pandas 2.3.3): the v0.14.1 hosted data plane at api.mostlyright.md is decommissioned (routes 404; 0.14.1 client raises NotFoundError), so the original fresh-vs-fresh re-capture is permanently impossible. Prior v0.14.1 fixture preserved at git e6a0ca1. - Drift vs old fixture: 3/365 days restated upstream (obs_mean_f, obs_mean_dewpoint_f only; obs_count identical; cli_*/obs_high/low byte-identical) + sub-1e-12 ULP accumulation-order repr in the three sum-aggregate columns. - ts/case_4 json + manifest regenerated via export_for_ts.py (cases 1/2/3/5 byte-unchanged); provenance section added to README. - Non-live gates green post-swap: test_parity + test_parity_ts_export (8 passed, incl. manifest sha256 + determinism). --- tests/fixtures/parity/README.md | 40 +- .../case_4_KMIA_2024-12-01_2025-11-30.parquet | Bin 27164 -> 27051 bytes .../ts/case_4_KMIA_2024-12-01_2025-11-30.json | 370 +++++++++--------- tests/fixtures/parity/ts/manifest.json | 2 +- 4 files changed, 225 insertions(+), 187 deletions(-) diff --git a/tests/fixtures/parity/README.md b/tests/fixtures/parity/README.md index 2eb73dbf..718fa0b6 100644 --- a/tests/fixtures/parity/README.md +++ b/tests/fixtures/parity/README.md @@ -1,13 +1,51 @@ # Parity fixtures — Day 3 hard gate **Captured from:** `mostlyright==0.14.1` on PyPI (via `api.mostlyright.md` hosted backend) -**Capture date:** 2026-05-21 +**Capture date:** 2026-05-21 (cases 1, 2, 3, 5) — **case 4 re-baselined 2026-07-09, see below** **Capture script:** `capture_fixtures.py` (in this directory) **Pinned dep set:** `pandas>=2.2,<4.0`, `pyarrow>=18,<25`, Python 3.11+ > Do **NOT** regenerate these files in CI. They are settlement-grade > reference data; the bytes ARE the Day 3 hard-gate ground truth. +## Case-4 re-baseline (2026-07-09, operator-approved — release 1.13.0 gate) + +`case_4_KMIA_2024-12-01_2025-11-30.parquet` (and its derived +`ts/case_4_*.json` + `ts/manifest.json` entry) was re-baselined from a +**fresh branch capture** (`mostlyright.dataset("KMIA", ...)`, default path, +isolated cache, pandas 2.3.3) on 2026-07-09. A fresh `mostlyright==0.14.1` +re-capture per §Re-capture below is **no longer possible**: the v0.14.1 +hosted data plane at `api.mostlyright.md` has been decommissioned (the +domain now serves an unrelated backend; `/observations`, `/climate`, +`/pairs` all 404 — verified 2026-07-09 with the 0.14.1 client raising +`NotFoundError`). The prior v0.14.1-captured fixture is preserved in git +at commit `e6a0ca1` and remains the historical v0.14.1 ground truth. + +Why the re-baseline was justified (full evidence in +`.planning/phases/30-observation-api-unification-one-mode/30-06-SUMMARY.md`): + +- The live gate failed ONLY on case-4 (rolling-year window ending + 2025-11-30) due to an **upstream value restatement**: exactly 3 of 365 + settlement days (2025-06-03, 2025-09-21, 2025-09-23) drifted, ONLY in + the sum÷count aggregates (`obs_mean_f` 2 days, `obs_mean_dewpoint_f` + 3 days), with `obs_count` IDENTICAL on every drifting day. +- Settlement labels (`cli_high_f`/`cli_low_f`) and extremes + (`obs_high_f`/`obs_low_f`) were **byte-identical across all 365 days**. +- Cases 1/2/3/5 (settled windows) still pass live byte-green against + their original v0.14.1 fixtures; the merge/aggregation code was + byte-unchanged vs main at re-baseline time. +- Remaining sub-1e-12 diffs vs the old fixture in the three + sum-aggregate columns are the documented ~1-ULP accumulation-order + drift class (v0.14.1 hosted storage order vs our sorted order); the + re-baseline zeroes them for case 4 because the baseline now IS this + SDK's output. + +**Contract note:** case-4's ground truth is now "this SDK's default path +on 2026-07-09 public-API data" (regression gate), while cases 1/2/3/5 +remain v0.14.1-captured (contract gate). Consider re-pinning case 4 to a +fully settled window in a future fixture refresh so the live gate stops +drifting on the rolling edge. + ## Phase 6 — pandas 3 compatibility The canonical parquet bytes stay 2.x-derived (immutable; the v0.1.0 diff --git a/tests/fixtures/parity/case_4_KMIA_2024-12-01_2025-11-30.parquet b/tests/fixtures/parity/case_4_KMIA_2024-12-01_2025-11-30.parquet index cc2b597620ae48d561a434e3cf8851606eddf575..4d9404218d645ee46571132c8e64032dcf262cb3 100644 GIT binary patch delta 3384 zcmZ`*d0Z4n_OFJXaeC;60UhpR#9^3Gxx{OY+K&h_0}RvC-Q%?^K}7|{2u4>E*8rO( zg2srJK{Qd=1QpjKxVRB7BpL;cx+osV8WxQbqM#E`qG+}tpWmOq_eWKA)vMR9-uu39 z)fZQ(y)9HtDMK~kjm&Ur5_quC_OE~duNw=3KQ{nV8W7-BuU~s&8jG=4!nwG|sBK&pJr?z+A5`lWbrhQ@uY8;#R zKmbJtJ=(@VF|Jexxb0tb@#yuO6c`#+jT_Y?T$nEWOg-0&$r_nt2gBp~bu)I_A(f&s z@gg^WjR`r|T@qnL?{^#1*NG5|x47lIn1*5>Hi$D$7GV;tAN7y3c2HnoQzp^}MN?M% zC<&Y;!%j@Aks#~p8@GgsqNRpDFIWmt#p3b$BoU4&Uo8!55V8>)Y=^wrv(W>tE)zGi z{&vO;JtCa#9r4bldm=n1Cp>k`+M&NDLBRgY{`)DJsnK&XggqUKS6`DX&FozX#JHRB zyCvW|@6SEZF2TE_3l4)QWjpomBJ^P3{N3r3B2(Aj0P^E1@ggofzwjM;(w2xU^f_ma zUoF9Fp~Yj&b`toT{4VyilwFM$a5|L7W$0E)FetlJnH|k#p^n8}##Zu&@}2G1qGTwl z(<^0|U;h1@7=>(VrH9K*N9H0K3^A&Gfgi|l+1xPh>rN3~HEy0A`9NYiSRWa*g%)G3 z%t4}Jft>rX422c+2ROui!npm-ozvdDmYLl67zgAQ1kB&#TtXaB^nl>R#Q~udW%P28 z#Pzxd539+1Ekp3fKh0j5EXa!Zu3Z>PGOy<)K;A70GIPY zmA5X-5Iz2zvK_}|=uAQ<9%^LZ*3ByZ^&=S)eB(t2?AvhUW%)V>blxfa;p8$$Nvq!E zfS3$_*Ik$8?D1@a43i6~O`fJe{a6`BC77hm|CCE+Jk6$ll)$*6VCf(vLp@IP45!NR z63_6M<-6OsxJ(&Fr@++H1Q#27{=$#hA;X{REBvD<8LoY*oA_+L46*%!W!Q@wJ$Y&} z9^@6ptUor=$0J38PRx6Sqz~ZkX9;>jq>qn$A;E#f+aC-$B>j#@0yS}o-?Oq98I+E6 z;a-spJ~uDDkjS|RTAvxSWa#hGvgAD->^vD-{@(1IBFWI}ZNKqwxeWVojaRsK;1|3I z=jQZ%7R5+lc7LCtYwvq~<`U2zo}z72Xpt8}UEaE7J69+JtE@nn!71X;wE@%yd{`UB zc6_EtpP69+YPxd@eyojGDhlYRWCnwGkgbWP?Mw~L+8IrXyB#K}Z+w-z)DAD{^=YBNT+Qiu9}O3nCb~9>X~MeD2i^(+uKf4h!uLP2 zLr-#ZJ-y!!3x*=h`2_0bZeybt%#0{ky_gEr*X*Qa$q-h|vR25Wex}osO*q9T$d?vM zuW#L9hhtTbXWu$W_{HoLZwW9B7y5)V=kHES`1P&;<@lgagy+zjrCA#z1o-kKzV^6N zfI;|?k9WSxgP6_8@wGzV{EjK+|JWd~F)y$&#s(x9lhl7m1(6y$a-0p`yV93AtH26r zaVC9LD%|=HkyTd47=Jni;%Zd%I-LN0dw#iMb+KD==>-->M<*niD(UZ2O_jQ@gx~r2 z5U0|QF~ip7f18FX;m|nU# znQ7ARPJxRTh_I#!PP!O?bBd+xaTpwznisgfJ98ScFsFIp2C865;6+BBw<{3-d(? z-22VJg{ugQcU6ke{mJJ=hHD~(KU(%Ub%6*i);XFP>qYoM6Tjjgt3@!AYCVuB!Yb0< zwu|7pF-r669Xk|d#al$Uw6|nILahj0KM`3WmK0-M081z=BH%r9SyX>Lik1xGTXrwq zE}m;OFBgFXC93R#2qw?r#rjV~C``<;>qNNzUG0|}XNaJ!t@$vsU4*=sy@@+45+pcr zcYsN6)#;Cju+(kmq<27_`p!=EBwd6@QCPDZncMN z$f+7;KKbOj2w!g3w^>37*-*de{0J9~ZG88=ZfhwzY^`)=5{3tNe)l}MmLxVgD^z6hRs|rT&c*K6XuC)%USD1L^I8#Bnq|*Bob)@- z^@$*#`7x*}K!Q2BznIj2LU^sqT3Xr$_i4enLaN`X-!FaInxI&o2B-Yvk( z4;LO^_QVd~cknTZ^X=e;)MmUn#OU*X>GD;Lz4OknH)GxgZ)8;E)wYBf6!|rbcAqju zTSg7=a4U@vN9oDKi`=R^_-$Ls^JowMJI*fi133HOZ@k+8v;4#}M&l)ZfZ(4!%xzn5GSI>P0m&720^ zpHEzP%w^EWd8USK=DY3Gefl7;SfNUL!Yj7W>PF5-n&8bnZnm*TR|6 zQ?h=T12mY+qKCrJv8?(=5SV}t0j3F|2%v|6VuPygB_Ht!0m)V~mw{e!Dw+bO8GRd( zf!w&r@pL;MR?#06QwHj^Y7lWUUIBOdE1N`O;PHE^hu0!)F|F;!KNwc$lN{wvWZY<^ z<0fcyXoM%D7@{g*6bhAd3#xQMXrvE9s3CK70O?Tf1hofpQqfRQS{yqkm&_E%qqh4R ziNMk7}#kG{O#Af>!5!JL_)#{)plTCBLYx@$Kbvdag#Z8m delta 3635 zcmZ`*c~}(3w(o)-?1ml~P)2q^hhYX;RgBlG_CZ-@V3_IY?uqC}6ck}FsDn#Hi2^TC zad`?U7c?%YsNjkZli)_U5(DnIVvLs{R|&@D5m!{y+e6;{^PTVO`l_nWck0wR_4}Q3 z;5yoR1J#t$s2!Klp=caFMMt9h_%S^s<|9HXM*-%Y?cUQfU7*vM1OWQ8G zCqClnc5;M?hHf5jbM|Epetzl1ty}|SXJuvKM(3f95Uj)9&eK^fSCwQ2^|Sibse8;| zGDn=dhC*7+PFY_?8=clqMt9clK*SQp;V60&L z8Pzx*#*(UG`+qjmcr;@uWRCK1JHtyiOn^UE4mtg4jR4jgA6(+c@DN`^p6B#M%nmag z&lnY!#KTFacXNXqI3N(B0?d$+NbY_lud@aVQ0?}`-SO2tv=qEKwdo2ESARPO~M5ShjN!<6wco2x&YlFzn)tkL4XA z#7#lu$9a)i=fP(Q9SXLNT-rc zmK+KXaZ4Vm8Eb)&mnMI`3a`_o@g{qrTs`{gem~l8Dn!6(Y!2PcC`$l?cP_Zt@m5xbFDd z^3@jT>?r={kMpIoZmYBwh)(x$+C&8iC#8o{_T)=8X1fSq)bH?#lpfQq zwSi+^W=S5{1&gpBA980Ai=Vhh(rZtK4RkXI@BjyU3{u*Gdgh^omsaZdRzcfC9uv;# za&d-7f35PD%dZ9L8fmT^MOj5a{irmnSlv#{2+wa8M0le#w?17U!XdoNBcy-qZ;CT4 z59eabDOw6x6Ga`aZS&aR;CDh}@Vq1QBs>VBOW9pBf6qcEzseLqgd}*ar!QKI_j^V% z_cq9;OiI^7-}VE~IOkG3i9{xY4&g|pe?J>TnqeKnW~F~2$pM-ZmDOT^5%vpyn*`&K zhJ2SGr@Hk?@Fe_V+nHP=ob4XNTyac-ncJ_DqgLx+WXIGYA7aLXL{X2-xk+$;VDi$V zFH*o_n0&U6ov7(!pT|7rZ2 z`y6b>^&`Idtdj$$(X%S9tIhCW^QR@w>rEglA+1T;BGN5cn{Mz* zf!?h43A$fR;MbJr*A#7nTj-wQZ${wl!T7`k^RD$~PA*D;w4gU(l}5Pxt0V;}(26vf zjRuIVp~%&N9Q0QIc`e1k4DE{xCFai7(}7fc5(HO}7mV5pYBvY5zmdz1$thKg4mL0N zdQxI}lAa?ErqKB$;&Q0CI~-^m(_XE6XohWdzs^g$n4~WyC+c$ob$bne)s7toIQ`W2 zwx>x4K|843ElD{XS(HL|W!7@A_)vTQM9D;P4pWX#c{vBA&{1KW5&rC!q?Cgr86{zN z%rMXn^Z*|6f_iX=m$7gTmH3^W?jcK(>1Jl10WKm$GDf0YW#@F8%*j1|9_;BnnDevluhuOls-wuJPt578W*8kfXD)W+Uv=jL$W zbu{Vzk`~#&cn=)%LWIy@N^VG zhZ9UcGx#Dz6r}WI;$OXeXnjfCY7Qh#{LwqYZEj_&^SC^o)-juSxD@)!>YF@-{O|l{ z#%vz0rA$+#{x=V8inxXQm+-Kshe_n2AeL5{d2lR?RJ`dh>s#XVJX}7oGCRJOhpyix zS;(&}i6ilQA7`Y;lYTqEM=V9rryqbzi9-e>odgF`vJow^9zX+Mq zN_C&ZFZ_mjyWHBBzT}VptJrP=oc%sI_vC#ZrsEiY)0AP`to5afc%4CVzA*3|Dk9Yb`1zMsYdcJDI(zZ`vc~jaJG*5wLe^?lchBA#US$iU@!sOES?AW>wz1B?vnOPlu*+;; z|Mc<&J7T!hfi<$nWnRQ#c@%k!rMa+9wo=k;-QB=8zLa2nmPHZN@ZpVe4OPi{qN&V) zLI+5c*(e1R$ZX^d7L8E`{e)*IBJ>GxdU9C(ox#iR*Hp zeqcY7c@xj&I^w9@O7;iY#B9nO1hOkstG~3M#twp@PNsD?0a_a|Kwd+=Wk1r0VD*ti zi`+`Q3#Ui~bQSdvHX*o~aFLH9+T=FU0kmCIC(Fq4;+oh)QicW`mcdVbox~H?lkhAd zJ-hHs^^(GV1d$K43rP%?k5&W{L%*Tv0c4Vlrn)yaI1ogHmsISVtt0*HHGlq?Rc-{~ z5H+5-S~X$Xd}*g5bM#IxEkU$AB#6)kXS>qDjV{tS#XeOujE?>~H{pQ2kD$^W@7nhI z5f#agi7`Y)zi#OY>D;I&l@bv}?&)J3O7&+9rIOgR>*=7pBU6gX$?9uWNPVnI`lG>ZiQY~JtC4zbb>A{r=j*h5 zKT>!7wVbWPiEGya+Hj#DP>Es zL+u{@31``eh^UCD+Dx(2)ycddWx|MswcK=9C#rATyLSLC&I!V$IUdqS(?pe&8ZMTP F{x|I|LYx2q diff --git a/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json b/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json index ca129536..06bd8029 100644 --- a/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json +++ b/tests/fixtures/parity/ts/case_4_KMIA_2024-12-01_2025-11-30.json @@ -38,7 +38,7 @@ "obs_low_f": 57.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 48.15625000000001, + "obs_mean_dewpoint_f": 48.15625, "obs_mean_f": 64.65875, "obs_total_precip_in": 0.0, "station": "MIA" @@ -105,7 +105,7 @@ "obs_max_gust_kt": null, "obs_max_wind_kt": 7, "obs_mean_dewpoint_f": 54.8725, - "obs_mean_f": 68.49125000000001, + "obs_mean_f": 68.49125, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -126,7 +126,7 @@ "obs_low_f": 60.0, "obs_max_gust_kt": null, "obs_max_wind_kt": 6, - "obs_mean_dewpoint_f": 55.80749999999999, + "obs_mean_dewpoint_f": 55.8075, "obs_mean_f": 70.125625, "obs_total_precip_in": 0.0, "station": "MIA" @@ -237,7 +237,7 @@ "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 67.22684210526316, - "obs_mean_f": 76.1721052631579, + "obs_mean_f": 76.17210526315789, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -258,7 +258,7 @@ "obs_low_f": 55.0, "obs_max_gust_kt": 28.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 50.75272727272728, + "obs_mean_dewpoint_f": 50.75272727272727, "obs_mean_f": 65.45818181818181, "obs_total_precip_in": 0.0, "station": "MIA" @@ -281,7 +281,7 @@ "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 56.53125, - "obs_mean_f": 72.58500000000001, + "obs_mean_f": 72.585, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -347,7 +347,7 @@ "obs_max_gust_kt": 24.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 69.654, - "obs_mean_f": 76.06249999999999, + "obs_mean_f": 76.0625, "obs_total_precip_in": 0.27, "station": "MIA" }, @@ -478,8 +478,8 @@ "obs_low_f": 55.94, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 54.370000000000005, - "obs_mean_f": 65.30562499999999, + "obs_mean_dewpoint_f": 54.37, + "obs_mean_f": 65.305625, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -501,7 +501,7 @@ "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 57.65375, - "obs_mean_f": 69.10187499999999, + "obs_mean_f": 69.101875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -634,7 +634,7 @@ "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 68.21875, "obs_mean_f": 72.2475, - "obs_total_precip_in": 0.72, + "obs_total_precip_in": 0.7200000000000001, "station": "MIA" }, { @@ -677,7 +677,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 70.02342857142858, - "obs_mean_f": 74.99657142857141, + "obs_mean_f": 74.99657142857143, "obs_total_precip_in": 0.01, "station": "MIA" }, @@ -742,8 +742,8 @@ "obs_low_f": 60.980000000000004, "obs_max_gust_kt": null, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 53.212500000000006, - "obs_mean_f": 67.67625000000001, + "obs_mean_dewpoint_f": 53.2125, + "obs_mean_f": 67.67625, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -786,7 +786,7 @@ "obs_low_f": 55.94, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 53.964375000000004, + "obs_mean_dewpoint_f": 53.964375, "obs_mean_f": 65.4075, "obs_total_precip_in": 0.0, "station": "MIA" @@ -830,7 +830,7 @@ "obs_low_f": 54.0, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 52.029756097560984, + "obs_mean_dewpoint_f": 52.02975609756097, "obs_mean_f": 61.20829268292683, "obs_total_precip_in": 0.0, "station": "MIA" @@ -852,7 +852,7 @@ "obs_low_f": 51.0, "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 49.08764705882352, + "obs_mean_dewpoint_f": 49.08764705882353, "obs_mean_f": 58.855294117647055, "obs_total_precip_in": 0.0, "station": "MIA" @@ -875,7 +875,7 @@ "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 36.42181818181818, - "obs_mean_f": 56.850303030303024, + "obs_mean_f": 56.85030303030303, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -897,7 +897,7 @@ "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 52.84625, - "obs_mean_f": 62.596875000000004, + "obs_mean_f": 62.596875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -918,7 +918,7 @@ "obs_low_f": 61.0, "obs_max_gust_kt": 30.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 59.91166666666666, + "obs_mean_dewpoint_f": 59.91166666666667, "obs_mean_f": 68.04277777777777, "obs_total_precip_in": 0.0, "station": "MIA" @@ -984,7 +984,7 @@ "obs_low_f": 61.0, "obs_max_gust_kt": null, "obs_max_wind_kt": 9, - "obs_mean_dewpoint_f": 64.65627906976745, + "obs_mean_dewpoint_f": 64.65627906976744, "obs_mean_f": 69.02558139534884, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1052,7 +1052,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 64.0795238095238, "obs_mean_f": 67.88238095238096, - "obs_total_precip_in": 0.24000000000000005, + "obs_total_precip_in": 0.24, "station": "MIA" }, { @@ -1072,7 +1072,7 @@ "obs_low_f": 71.0, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 70.05428571428571, + "obs_mean_dewpoint_f": 70.05428571428573, "obs_mean_f": 74.33142857142857, "obs_total_precip_in": 1.31, "station": "MIA" @@ -1094,7 +1094,7 @@ "obs_low_f": 71.0, "obs_max_gust_kt": 28.0, "obs_max_wind_kt": 17, - "obs_mean_dewpoint_f": 70.70341463414633, + "obs_mean_dewpoint_f": 70.70341463414634, "obs_mean_f": 75.61170731707317, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1118,7 +1118,7 @@ "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 58.050000000000004, "obs_mean_f": 62.42526315789473, - "obs_total_precip_in": 0.15000000000000002, + "obs_total_precip_in": 0.15, "station": "MIA" }, { @@ -1162,7 +1162,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 56.55673469387756, "obs_mean_f": 58.2530612244898, - "obs_total_precip_in": 0.060000000000000005, + "obs_total_precip_in": 0.06, "station": "MIA" }, { @@ -1204,7 +1204,7 @@ "obs_low_f": 48.2, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 46.698461538461544, + "obs_mean_dewpoint_f": 46.69846153846154, "obs_mean_f": 52.57076923076924, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1270,7 +1270,7 @@ "obs_low_f": 60.980000000000004, "obs_max_gust_kt": null, "obs_max_wind_kt": 7, - "obs_mean_dewpoint_f": 58.099374999999995, + "obs_mean_dewpoint_f": 58.099375, "obs_mean_f": 69.088125, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1292,7 +1292,7 @@ "obs_low_f": 57.92, "obs_max_gust_kt": null, "obs_max_wind_kt": 6, - "obs_mean_dewpoint_f": 54.72375000000001, + "obs_mean_dewpoint_f": 54.72375, "obs_mean_f": 68.09625, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1314,7 +1314,7 @@ "obs_low_f": 59.0, "obs_max_gust_kt": null, "obs_max_wind_kt": 7, - "obs_mean_dewpoint_f": 59.654999999999994, + "obs_mean_dewpoint_f": 59.655, "obs_mean_f": 69.156875, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1359,7 +1359,7 @@ "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 66.08764705882353, - "obs_mean_f": 73.72352941176472, + "obs_mean_f": 73.7235294117647, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1402,7 +1402,7 @@ "obs_low_f": 71.6, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 70.47813953488371, + "obs_mean_dewpoint_f": 70.47813953488372, "obs_mean_f": 75.20790697674418, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1468,7 +1468,7 @@ "obs_low_f": 69.0, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 65.58647058823529, + "obs_mean_dewpoint_f": 65.5864705882353, "obs_mean_f": 74.17764705882352, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1491,7 +1491,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 64.94121212121212, - "obs_mean_f": 74.30969696969697, + "obs_mean_f": 74.30969696969696, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -1578,7 +1578,7 @@ "obs_low_f": 68.0, "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 67.5777142857143, + "obs_mean_dewpoint_f": 67.57771428571428, "obs_mean_f": 75.41085714285714, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1600,7 +1600,7 @@ "obs_low_f": 71.96, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 66.38545454545455, + "obs_mean_dewpoint_f": 66.38545454545454, "obs_mean_f": 75.39696969696969, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1732,7 +1732,7 @@ "obs_low_f": 69.0, "obs_max_gust_kt": 14.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 64.71199999999999, + "obs_mean_dewpoint_f": 64.712, "obs_mean_f": 73.9135, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1776,7 +1776,7 @@ "obs_low_f": 69.98, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 68.60666666666665, + "obs_mean_dewpoint_f": 68.60666666666667, "obs_mean_f": 76.57212121212122, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1842,7 +1842,7 @@ "obs_low_f": 66.0, "obs_max_gust_kt": 24.0, "obs_max_wind_kt": 17, - "obs_mean_dewpoint_f": 54.120625000000004, + "obs_mean_dewpoint_f": 54.120625, "obs_mean_f": 69.530625, "obs_total_precip_in": 0.0, "station": "MIA" @@ -1930,7 +1930,7 @@ "obs_low_f": 60.980000000000004, "obs_max_gust_kt": null, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 60.44562499999999, + "obs_mean_dewpoint_f": 60.445625, "obs_mean_f": 70.27875, "obs_total_precip_in": 0.0, "station": "MIA" @@ -2106,7 +2106,7 @@ "obs_low_f": 58.0, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 50.80687499999999, + "obs_mean_dewpoint_f": 50.806875, "obs_mean_f": 67.999375, "obs_total_precip_in": 0.0, "station": "MIA" @@ -2194,7 +2194,7 @@ "obs_low_f": 65.0, "obs_max_gust_kt": 34.0, "obs_max_wind_kt": 22, - "obs_mean_dewpoint_f": 66.82062499999999, + "obs_mean_dewpoint_f": 66.820625, "obs_mean_f": 76.060625, "obs_total_precip_in": 0.0, "station": "MIA" @@ -2216,8 +2216,8 @@ "obs_low_f": 59.0, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 48.418124999999996, - "obs_mean_f": 67.60062500000001, + "obs_mean_dewpoint_f": 48.418125, + "obs_mean_f": 67.600625, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2305,7 +2305,7 @@ "obs_max_gust_kt": 26.0, "obs_max_wind_kt": 17, "obs_mean_dewpoint_f": 67.408125, - "obs_mean_f": 76.78187500000001, + "obs_mean_f": 76.781875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2327,7 +2327,7 @@ "obs_max_gust_kt": 30.0, "obs_max_wind_kt": 17, "obs_mean_dewpoint_f": 71.64941176470587, - "obs_mean_f": 78.75529411764707, + "obs_mean_f": 78.75529411764705, "obs_total_precip_in": 0.01, "station": "MIA" }, @@ -2371,7 +2371,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 50.68625, - "obs_mean_f": 67.16499999999999, + "obs_mean_f": 67.165, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2415,7 +2415,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 57.24625, - "obs_mean_f": 73.16125000000001, + "obs_mean_f": 73.16125, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2458,7 +2458,7 @@ "obs_low_f": 60.980000000000004, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 53.629999999999995, + "obs_mean_dewpoint_f": 53.63, "obs_mean_f": 68.46875, "obs_total_precip_in": 0.0, "station": "MIA" @@ -2481,7 +2481,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 59.000625, - "obs_mean_f": 71.68187499999999, + "obs_mean_f": 71.681875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2590,8 +2590,8 @@ "obs_low_f": 73.0, "obs_max_gust_kt": 32.0, "obs_max_wind_kt": 24, - "obs_mean_dewpoint_f": 60.629999999999995, - "obs_mean_f": 75.71624999999999, + "obs_mean_dewpoint_f": 60.63, + "obs_mean_f": 75.71625, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2634,7 +2634,7 @@ "obs_low_f": 69.8, "obs_max_gust_kt": 30.0, "obs_max_wind_kt": 16, - "obs_mean_dewpoint_f": 71.40750000000001, + "obs_mean_dewpoint_f": 71.4075, "obs_mean_f": 74.93375, "obs_total_precip_in": 2.2600000000000002, "station": "MIA" @@ -2679,7 +2679,7 @@ "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 72.369375, - "obs_mean_f": 79.34124999999999, + "obs_mean_f": 79.34125, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2745,7 +2745,7 @@ "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 18, "obs_mean_dewpoint_f": 69.2078947368421, - "obs_mean_f": 79.3357894736842, + "obs_mean_f": 79.33578947368422, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -2833,8 +2833,8 @@ "obs_max_gust_kt": 40.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 71.4872, - "obs_mean_f": 77.28519999999999, - "obs_total_precip_in": 4.069999999999999, + "obs_mean_f": 77.2852, + "obs_total_precip_in": 4.07, "station": "MIA" }, { @@ -2877,7 +2877,7 @@ "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 63.54176470588236, - "obs_mean_f": 73.20823529411766, + "obs_mean_f": 73.20823529411764, "obs_total_precip_in": 0.01, "station": "MIA" }, @@ -2986,7 +2986,7 @@ "obs_low_f": 66.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 60.099999999999994, + "obs_mean_dewpoint_f": 60.1, "obs_mean_f": 74.999375, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3031,7 +3031,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 60.56125, - "obs_mean_f": 76.39687500000001, + "obs_mean_f": 76.396875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3052,7 +3052,7 @@ "obs_low_f": 71.96, "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 17, - "obs_mean_dewpoint_f": 60.70062499999999, + "obs_mean_dewpoint_f": 60.700625, "obs_mean_f": 76.9975, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3118,7 +3118,7 @@ "obs_low_f": 73.94, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 62.28874999999999, + "obs_mean_dewpoint_f": 62.28875, "obs_mean_f": 77.1875, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3184,7 +3184,7 @@ "obs_low_f": 74.0, "obs_max_gust_kt": 27.0, "obs_max_wind_kt": 15, - "obs_mean_dewpoint_f": 65.33937499999999, + "obs_mean_dewpoint_f": 65.339375, "obs_mean_f": 78.41125, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3229,7 +3229,7 @@ "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 64.775, - "obs_mean_f": 78.42875000000001, + "obs_mean_f": 78.42875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3317,7 +3317,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 60.31375, - "obs_mean_f": 77.61500000000001, + "obs_mean_f": 77.615, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3338,8 +3338,8 @@ "obs_low_f": 74.0, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 16, - "obs_mean_dewpoint_f": 62.590625, - "obs_mean_f": 77.77874999999999, + "obs_mean_dewpoint_f": 62.590624999999996, + "obs_mean_f": 77.77875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3360,7 +3360,7 @@ "obs_low_f": 73.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 62.723749999999995, + "obs_mean_dewpoint_f": 62.72375, "obs_mean_f": 77.878125, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3406,7 +3406,7 @@ "obs_max_wind_kt": 22, "obs_mean_dewpoint_f": 68.76157894736842, "obs_mean_f": 75.5, - "obs_total_precip_in": 2.03, + "obs_total_precip_in": 2.0300000000000002, "station": "MIA" }, { @@ -3559,7 +3559,7 @@ "obs_max_gust_kt": 26.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 71.6257142857143, - "obs_mean_f": 81.908, + "obs_mean_f": 81.90799999999999, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3581,7 +3581,7 @@ "obs_max_gust_kt": 31.0, "obs_max_wind_kt": 21, "obs_mean_dewpoint_f": 72.41555555555556, - "obs_mean_f": 75.60444444444445, + "obs_mean_f": 75.60444444444444, "obs_total_precip_in": 9.68, "station": "MIA" }, @@ -3604,7 +3604,7 @@ "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 71.82176470588236, "obs_mean_f": 79.67470588235294, - "obs_total_precip_in": 0.21000000000000002, + "obs_total_precip_in": 0.21, "station": "MIA" }, { @@ -3624,7 +3624,7 @@ "obs_low_f": 73.94, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 66.17875000000001, + "obs_mean_dewpoint_f": 66.17875, "obs_mean_f": 80.369375, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3647,7 +3647,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 69.380625, - "obs_mean_f": 81.7575, + "obs_mean_f": 81.75750000000001, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3669,7 +3669,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 71.27575757575757, - "obs_mean_f": 82.10060606060605, + "obs_mean_f": 82.10060606060607, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3691,7 +3691,7 @@ "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 73.219375, - "obs_mean_f": 83.32000000000001, + "obs_mean_f": 83.32, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -3802,7 +3802,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 72.66176470588235, "obs_mean_f": 82.27588235294118, - "obs_total_precip_in": 3.2700000000000005, + "obs_total_precip_in": 3.27, "station": "MIA" }, { @@ -3822,7 +3822,7 @@ "obs_low_f": 75.92, "obs_max_gust_kt": 27.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 73.7694117647059, + "obs_mean_dewpoint_f": 73.76941176470588, "obs_mean_f": 82.96823529411765, "obs_total_precip_in": 0.0, "station": "MIA" @@ -3911,8 +3911,8 @@ "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 74.94432432432433, - "obs_mean_f": 83.96864864864864, - "obs_total_precip_in": 0.7899999999999999, + "obs_mean_f": 83.96864864864865, + "obs_total_precip_in": 0.79, "station": "MIA" }, { @@ -3977,7 +3977,7 @@ "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 72.38571428571429, - "obs_mean_f": 85.22057142857143, + "obs_mean_f": 85.22057142857142, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -4021,7 +4021,7 @@ "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 72.65944444444443, - "obs_mean_f": 80.79944444444445, + "obs_mean_f": 80.79944444444443, "obs_total_precip_in": 0.45, "station": "MIA" }, @@ -4043,8 +4043,8 @@ "obs_max_gust_kt": 43.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 73.2782857142857, - "obs_mean_f": 77.7777142857143, - "obs_total_precip_in": 6.920000000000001, + "obs_mean_f": 77.77771428571428, + "obs_total_precip_in": 6.92, "station": "MIA" }, { @@ -4064,9 +4064,9 @@ "obs_low_f": 72.0, "obs_max_gust_kt": 26.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 72.20923076923077, - "obs_mean_f": 74.78358974358974, - "obs_total_precip_in": 1.6300000000000003, + "obs_mean_dewpoint_f": 72.20820512820512, + "obs_mean_f": 74.78307692307692, + "obs_total_precip_in": 1.6300000000000001, "station": "MIA" }, { @@ -4130,7 +4130,7 @@ "obs_low_f": 80.0, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 75.06176470588237, + "obs_mean_dewpoint_f": 75.06176470588235, "obs_mean_f": 83.62117647058824, "obs_total_precip_in": 0.0, "station": "MIA" @@ -4262,8 +4262,8 @@ "obs_low_f": 77.0, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 17, - "obs_mean_dewpoint_f": 74.20849999999999, - "obs_mean_f": 82.945, + "obs_mean_dewpoint_f": 74.2085, + "obs_mean_f": 82.94500000000001, "obs_total_precip_in": 0.08, "station": "MIA" }, @@ -4284,7 +4284,7 @@ "obs_low_f": 77.0, "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 16, - "obs_mean_dewpoint_f": 73.96352941176471, + "obs_mean_dewpoint_f": 73.9635294117647, "obs_mean_f": 83.87882352941176, "obs_total_precip_in": 1.3800000000000001, "station": "MIA" @@ -4417,7 +4417,7 @@ "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 73.87352941176471, - "obs_mean_f": 83.54058823529411, + "obs_mean_f": 83.54058823529412, "obs_total_precip_in": 1.15, "station": "MIA" }, @@ -4462,7 +4462,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 72.7248484848485, "obs_mean_f": 84.06484848484848, - "obs_total_precip_in": 0.10999999999999999, + "obs_total_precip_in": 0.11, "station": "MIA" }, { @@ -4482,7 +4482,7 @@ "obs_low_f": 78.0, "obs_max_gust_kt": 24.0, "obs_max_wind_kt": 17, - "obs_mean_dewpoint_f": 74.63882352941177, + "obs_mean_dewpoint_f": 74.63882352941175, "obs_mean_f": 83.32352941176471, "obs_total_precip_in": 0.26, "station": "MIA" @@ -4528,7 +4528,7 @@ "obs_max_wind_kt": 16, "obs_mean_dewpoint_f": 72.905625, "obs_mean_f": 83.96125, - "obs_total_precip_in": 0.06999999999999999, + "obs_total_precip_in": 0.07, "station": "MIA" }, { @@ -4549,7 +4549,7 @@ "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 16, "obs_mean_dewpoint_f": 71.69902439024389, - "obs_mean_f": 81.47609756097562, + "obs_mean_f": 81.4760975609756, "obs_total_precip_in": 0.13, "station": "MIA" }, @@ -4571,7 +4571,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 73.22864864864864, - "obs_mean_f": 81.27243243243244, + "obs_mean_f": 81.27243243243242, "obs_total_precip_in": 0.02, "station": "MIA" }, @@ -4637,7 +4637,7 @@ "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 69.9325, - "obs_mean_f": 82.82249999999999, + "obs_mean_f": 82.8225, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -4704,7 +4704,7 @@ "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 74.73902439024391, "obs_mean_f": 80.67707317073172, - "obs_total_precip_in": 0.18000000000000002, + "obs_total_precip_in": 0.18, "station": "MIA" }, { @@ -4726,7 +4726,7 @@ "obs_max_wind_kt": 16, "obs_mean_dewpoint_f": 74.96136363636364, "obs_mean_f": 80.86454545454545, - "obs_total_precip_in": 4.549999999999999, + "obs_total_precip_in": 4.55, "station": "MIA" }, { @@ -4834,7 +4834,7 @@ "obs_low_f": 80.0, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 74.34352941176472, + "obs_mean_dewpoint_f": 74.3435294117647, "obs_mean_f": 85.96882352941176, "obs_total_precip_in": 0.0, "station": "MIA" @@ -4878,7 +4878,7 @@ "obs_low_f": 77.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 75.47350000000002, + "obs_mean_dewpoint_f": 75.4735, "obs_mean_f": 84.032, "obs_total_precip_in": 0.02, "station": "MIA" @@ -4900,7 +4900,7 @@ "obs_low_f": 77.0, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 73.83999999999999, + "obs_mean_dewpoint_f": 73.84, "obs_mean_f": 84.066875, "obs_total_precip_in": 0.02, "station": "MIA" @@ -4945,7 +4945,7 @@ "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 9, "obs_mean_dewpoint_f": 74.18972972972973, - "obs_mean_f": 82.37567567567567, + "obs_mean_f": 82.37567567567568, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -4968,7 +4968,7 @@ "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 72.505, "obs_mean_f": 78.97555555555556, - "obs_total_precip_in": 0.29000000000000004, + "obs_total_precip_in": 0.29, "station": "MIA" }, { @@ -4989,7 +4989,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 73.38388888888889, - "obs_mean_f": 81.21333333333334, + "obs_mean_f": 81.21333333333332, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -5012,7 +5012,7 @@ "obs_max_wind_kt": 16, "obs_mean_dewpoint_f": 75.47488372093024, "obs_mean_f": 82.20093023255814, - "obs_total_precip_in": 4.170000000000001, + "obs_total_precip_in": 4.17, "station": "MIA" }, { @@ -5032,7 +5032,7 @@ "obs_low_f": 82.0, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 15, - "obs_mean_dewpoint_f": 77.1502857142857, + "obs_mean_dewpoint_f": 77.15028571428572, "obs_mean_f": 85.60971428571429, "obs_total_precip_in": 0.0, "station": "MIA" @@ -5143,8 +5143,8 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 74.10315789473684, - "obs_mean_f": 82.01947368421051, - "obs_total_precip_in": 1.7700000000000002, + "obs_mean_f": 82.01947368421054, + "obs_total_precip_in": 1.77, "station": "MIA" }, { @@ -5166,7 +5166,7 @@ "obs_max_wind_kt": 19, "obs_mean_dewpoint_f": 73.54871794871795, "obs_mean_f": 81.30717948717948, - "obs_total_precip_in": 2.7700000000000005, + "obs_total_precip_in": 2.77, "station": "MIA" }, { @@ -5208,7 +5208,7 @@ "obs_low_f": 82.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 74.89687500000001, + "obs_mean_dewpoint_f": 74.896875, "obs_mean_f": 85.9975, "obs_total_precip_in": 0.0, "station": "MIA" @@ -5274,7 +5274,7 @@ "obs_low_f": 82.94, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 75.19533333333332, + "obs_mean_dewpoint_f": 75.19533333333334, "obs_mean_f": 86.798, "obs_total_precip_in": 0.0, "station": "MIA" @@ -5298,7 +5298,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 76.05297297297298, "obs_mean_f": 85.45783783783784, - "obs_total_precip_in": 0.5599999999999999, + "obs_total_precip_in": 0.56, "station": "MIA" }, { @@ -5318,7 +5318,7 @@ "obs_low_f": 80.0, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 76.02250000000001, + "obs_mean_dewpoint_f": 76.0225, "obs_mean_f": 86.433125, "obs_total_precip_in": 0.0, "station": "MIA" @@ -5363,7 +5363,7 @@ "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 74.56, - "obs_mean_f": 86.493125, + "obs_mean_f": 86.49312499999999, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -5450,7 +5450,7 @@ "obs_low_f": 79.0, "obs_max_gust_kt": 25.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 74.95947368421054, + "obs_mean_dewpoint_f": 74.95947368421052, "obs_mean_f": 85.75315789473684, "obs_total_precip_in": 1.4, "station": "MIA" @@ -5496,7 +5496,7 @@ "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 75.92170212765957, "obs_mean_f": 81.82425531914893, - "obs_total_precip_in": 0.94, + "obs_total_precip_in": 0.9400000000000001, "station": "MIA" }, { @@ -5518,7 +5518,7 @@ "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 74.52526315789474, "obs_mean_f": 82.5521052631579, - "obs_total_precip_in": 0.15000000000000002, + "obs_total_precip_in": 0.15, "station": "MIA" }, { @@ -5560,7 +5560,7 @@ "obs_low_f": 75.0, "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 74.76121212121213, + "obs_mean_dewpoint_f": 74.76121212121211, "obs_mean_f": 80.61515151515152, "obs_total_precip_in": 0.18, "station": "MIA" @@ -5604,7 +5604,7 @@ "obs_low_f": 80.0, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 76.53575757575759, + "obs_mean_dewpoint_f": 76.53575757575757, "obs_mean_f": 85.14545454545456, "obs_total_precip_in": 0.16, "station": "MIA" @@ -5692,7 +5692,7 @@ "obs_low_f": 81.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 74.30250000000001, + "obs_mean_dewpoint_f": 74.3025, "obs_mean_f": 86.784375, "obs_total_precip_in": 0.0, "station": "MIA" @@ -5716,7 +5716,7 @@ "obs_max_wind_kt": 9, "obs_mean_dewpoint_f": 76.00235294117647, "obs_mean_f": 85.20705882352941, - "obs_total_precip_in": 0.12, + "obs_total_precip_in": 0.12000000000000001, "station": "MIA" }, { @@ -5760,7 +5760,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 76.072, "obs_mean_f": 84.50628571428571, - "obs_total_precip_in": 0.23, + "obs_total_precip_in": 0.22999999999999998, "station": "MIA" }, { @@ -5802,9 +5802,9 @@ "obs_low_f": 78.0, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 74.96909090909092, - "obs_mean_f": 83.2790909090909, - "obs_total_precip_in": 0.30000000000000004, + "obs_mean_dewpoint_f": 74.96909090909091, + "obs_mean_f": 83.27909090909091, + "obs_total_precip_in": 0.3, "station": "MIA" }, { @@ -5847,8 +5847,8 @@ "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 73.82742857142857, - "obs_mean_f": 80.26457142857141, - "obs_total_precip_in": 0.18000000000000002, + "obs_mean_f": 80.26457142857143, + "obs_total_precip_in": 0.18, "station": "MIA" }, { @@ -5868,7 +5868,7 @@ "obs_low_f": 75.92, "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 16, - "obs_mean_dewpoint_f": 74.87684210526315, + "obs_mean_dewpoint_f": 74.87684210526317, "obs_mean_f": 82.0621052631579, "obs_total_precip_in": 0.29, "station": "MIA" @@ -5958,7 +5958,7 @@ "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 75.93588235294118, "obs_mean_f": 84.64470588235294, - "obs_total_precip_in": 0.060000000000000005, + "obs_total_precip_in": 0.06, "station": "MIA" }, { @@ -6024,7 +6024,7 @@ "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 75.11333333333333, "obs_mean_f": 81.46000000000001, - "obs_total_precip_in": 1.2100000000000002, + "obs_total_precip_in": 1.21, "station": "MIA" }, { @@ -6132,9 +6132,9 @@ "obs_low_f": 76.0, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 75.6639024390244, + "obs_mean_dewpoint_f": 75.66390243902438, "obs_mean_f": 79.46878048780488, - "obs_total_precip_in": 4.77, + "obs_total_precip_in": 4.7700000000000005, "station": "MIA" }, { @@ -6198,9 +6198,9 @@ "obs_low_f": 74.0, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 75.14619047619048, - "obs_mean_f": 79.55666666666666, - "obs_total_precip_in": 8.659999999999998, + "obs_mean_dewpoint_f": 75.14619047619047, + "obs_mean_f": 79.55666666666667, + "obs_total_precip_in": 8.66, "station": "MIA" }, { @@ -6243,8 +6243,8 @@ "obs_max_gust_kt": 27.0, "obs_max_wind_kt": 20, "obs_mean_dewpoint_f": 75.84045454545455, - "obs_mean_f": 79.34227272727271, - "obs_total_precip_in": 7.81, + "obs_mean_f": 79.34227272727273, + "obs_total_precip_in": 7.8100000000000005, "station": "MIA" }, { @@ -6266,7 +6266,7 @@ "obs_max_wind_kt": 16, "obs_mean_dewpoint_f": 75.52139534883722, "obs_mean_f": 80.73441860465117, - "obs_total_precip_in": 2.0799999999999996, + "obs_total_precip_in": 2.08, "station": "MIA" }, { @@ -6288,7 +6288,7 @@ "obs_max_wind_kt": 13, "obs_mean_dewpoint_f": 75.65463414634146, "obs_mean_f": 79.77707317073171, - "obs_total_precip_in": 5.220000000000001, + "obs_total_precip_in": 5.22, "station": "MIA" }, { @@ -6332,7 +6332,7 @@ "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 72.79058823529412, "obs_mean_f": 80.52823529411765, - "obs_total_precip_in": 1.9999999999999998, + "obs_total_precip_in": 2.0, "station": "MIA" }, { @@ -6353,7 +6353,7 @@ "obs_max_gust_kt": 24.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 73.01515151515152, - "obs_mean_f": 82.37575757575756, + "obs_mean_f": 82.37575757575758, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -6418,7 +6418,7 @@ "obs_low_f": 76.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 73.89945945945945, + "obs_mean_dewpoint_f": 73.89945945945946, "obs_mean_f": 80.2718918918919, "obs_total_precip_in": 0.25, "station": "MIA" @@ -6440,7 +6440,7 @@ "obs_low_f": 77.0, "obs_max_gust_kt": null, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 74.0229411764706, + "obs_mean_dewpoint_f": 74.02294117647058, "obs_mean_f": 82.61823529411765, "obs_total_precip_in": 0.0, "station": "MIA" @@ -6462,8 +6462,8 @@ "obs_low_f": 78.0, "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 74.96235294117648, - "obs_mean_f": 81.56647058823529, + "obs_mean_dewpoint_f": 74.96235294117646, + "obs_mean_f": 81.5664705882353, "obs_total_precip_in": 0.25, "station": "MIA" }, @@ -6484,8 +6484,8 @@ "obs_low_f": 73.4, "obs_max_gust_kt": 24.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 76.0032558139535, - "obs_mean_f": 79.5739534883721, + "obs_mean_dewpoint_f": 76.00511627906977, + "obs_mean_f": 79.57441860465116, "obs_total_precip_in": 1.7, "station": "MIA" }, @@ -6528,9 +6528,9 @@ "obs_low_f": 74.0, "obs_max_gust_kt": 30.0, "obs_max_wind_kt": 16, - "obs_mean_dewpoint_f": 74.166, + "obs_mean_dewpoint_f": 74.1705, "obs_mean_f": 80.01849999999999, - "obs_total_precip_in": 2.1499999999999995, + "obs_total_precip_in": 2.15, "station": "MIA" }, { @@ -6551,8 +6551,8 @@ "obs_max_gust_kt": 21.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 76.21692307692308, - "obs_mean_f": 81.60871794871795, - "obs_total_precip_in": 0.14, + "obs_mean_f": 81.60871794871794, + "obs_total_precip_in": 0.13999999999999999, "station": "MIA" }, { @@ -6594,8 +6594,8 @@ "obs_low_f": 78.0, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 11, - "obs_mean_dewpoint_f": 76.43189189189191, - "obs_mean_f": 83.26702702702701, + "obs_mean_dewpoint_f": 76.4318918918919, + "obs_mean_f": 83.26702702702703, "obs_total_precip_in": 0.18, "station": "MIA" }, @@ -6640,7 +6640,7 @@ "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 74.91055555555556, "obs_mean_f": 81.63888888888889, - "obs_total_precip_in": 0.060000000000000005, + "obs_total_precip_in": 0.06, "station": "MIA" }, { @@ -6728,7 +6728,7 @@ "obs_max_wind_kt": 16, "obs_mean_dewpoint_f": 73.41463414634147, "obs_mean_f": 80.04975609756097, - "obs_total_precip_in": 3.4099999999999993, + "obs_total_precip_in": 3.41, "station": "MIA" }, { @@ -6750,7 +6750,7 @@ "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 72.06944444444444, "obs_mean_f": 80.70277777777778, - "obs_total_precip_in": 0.21000000000000002, + "obs_total_precip_in": 0.21, "station": "MIA" }, { @@ -6771,7 +6771,7 @@ "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 15, "obs_mean_dewpoint_f": 73.41428571428571, - "obs_mean_f": 82.02114285714286, + "obs_mean_f": 82.02114285714285, "obs_total_precip_in": 0.08, "station": "MIA" }, @@ -6814,7 +6814,7 @@ "obs_low_f": 79.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 75.59837837837838, + "obs_mean_dewpoint_f": 75.59837837837837, "obs_mean_f": 82.76918918918919, "obs_total_precip_in": 0.48, "station": "MIA" @@ -6903,7 +6903,7 @@ "obs_max_gust_kt": null, "obs_max_wind_kt": 8, "obs_mean_dewpoint_f": 74.61578947368422, - "obs_mean_f": 77.18842105263158, + "obs_mean_f": 77.18842105263157, "obs_total_precip_in": 2.26, "station": "MIA" }, @@ -6925,7 +6925,7 @@ "obs_max_gust_kt": null, "obs_max_wind_kt": 6, "obs_mean_dewpoint_f": 72.84666666666666, - "obs_mean_f": 77.92444444444443, + "obs_mean_f": 77.92444444444445, "obs_total_precip_in": 0.02, "station": "MIA" }, @@ -6990,7 +6990,7 @@ "obs_low_f": 75.92, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 68.04249999999999, + "obs_mean_dewpoint_f": 68.0425, "obs_mean_f": 80.338125, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7013,7 +7013,7 @@ "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 11, "obs_mean_dewpoint_f": 67.974375, - "obs_mean_f": 79.65249999999999, + "obs_mean_f": 79.6525, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7034,8 +7034,8 @@ "obs_low_f": 73.4, "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 68.19941176470587, - "obs_mean_f": 78.44882352941175, + "obs_mean_dewpoint_f": 68.19941176470589, + "obs_mean_f": 78.44882352941177, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7078,7 +7078,7 @@ "obs_low_f": 73.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 63.630624999999995, + "obs_mean_dewpoint_f": 63.630625, "obs_mean_f": 78.62125, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7122,7 +7122,7 @@ "obs_low_f": 78.0, "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 74.32606060606061, + "obs_mean_dewpoint_f": 74.3260606060606, "obs_mean_f": 82.09212121212121, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7145,7 +7145,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 72.90171428571428, - "obs_mean_f": 82.05199999999999, + "obs_mean_f": 82.052, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7210,7 +7210,7 @@ "obs_low_f": 78.0, "obs_max_gust_kt": 27.0, "obs_max_wind_kt": 18, - "obs_mean_dewpoint_f": 68.11625000000001, + "obs_mean_dewpoint_f": 68.11625, "obs_mean_f": 81.590625, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7255,7 +7255,7 @@ "obs_max_gust_kt": 27.0, "obs_max_wind_kt": 18, "obs_mean_dewpoint_f": 74.33945945945946, - "obs_mean_f": 81.84270270270271, + "obs_mean_f": 81.8427027027027, "obs_total_precip_in": 0.6900000000000001, "station": "MIA" }, @@ -7277,7 +7277,7 @@ "obs_max_gust_kt": 19.0, "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 75.16833333333334, - "obs_mean_f": 81.74388888888889, + "obs_mean_f": 81.74388888888888, "obs_total_precip_in": 1.26, "station": "MIA" }, @@ -7342,7 +7342,7 @@ "obs_low_f": 69.98, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 60.41942857142858, + "obs_mean_dewpoint_f": 60.41942857142857, "obs_mean_f": 73.516, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7364,7 +7364,7 @@ "obs_low_f": 64.94, "obs_max_gust_kt": 15.0, "obs_max_wind_kt": 8, - "obs_mean_dewpoint_f": 57.0974193548387, + "obs_mean_dewpoint_f": 57.097419354838706, "obs_mean_f": 69.70967741935483, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7387,7 +7387,7 @@ "obs_max_gust_kt": null, "obs_max_wind_kt": 7, "obs_mean_dewpoint_f": 55.99875, - "obs_mean_f": 72.99624999999999, + "obs_mean_f": 72.99625, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7410,7 +7410,7 @@ "obs_max_wind_kt": 12, "obs_mean_dewpoint_f": 65.17470588235294, "obs_mean_f": 76.1435294117647, - "obs_total_precip_in": 0.18000000000000005, + "obs_total_precip_in": 0.18000000000000002, "station": "MIA" }, { @@ -7431,7 +7431,7 @@ "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 68.563125, - "obs_mean_f": 76.83874999999999, + "obs_mean_f": 76.83875, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7474,7 +7474,7 @@ "obs_low_f": 75.0, "obs_max_gust_kt": 23.0, "obs_max_wind_kt": 14, - "obs_mean_dewpoint_f": 64.52562499999999, + "obs_mean_dewpoint_f": 64.525625, "obs_mean_f": 77.8475, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7541,7 +7541,7 @@ "obs_max_gust_kt": 16.0, "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 72.12125, - "obs_mean_f": 80.47625000000001, + "obs_mean_f": 80.47625, "obs_total_precip_in": 0.0, "station": "MIA" }, @@ -7606,7 +7606,7 @@ "obs_low_f": 48.92, "obs_max_gust_kt": 22.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 42.380624999999995, + "obs_mean_dewpoint_f": 42.380625, "obs_mean_f": 58.529375, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7650,7 +7650,7 @@ "obs_low_f": 64.94, "obs_max_gust_kt": 17.0, "obs_max_wind_kt": 10, - "obs_mean_dewpoint_f": 56.21124999999999, + "obs_mean_dewpoint_f": 56.21125, "obs_mean_f": 70.4675, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7672,7 +7672,7 @@ "obs_low_f": 64.94, "obs_max_gust_kt": 18.0, "obs_max_wind_kt": 12, - "obs_mean_dewpoint_f": 57.411249999999995, + "obs_mean_dewpoint_f": 57.41125, "obs_mean_f": 72.495, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7914,7 +7914,7 @@ "obs_low_f": 73.0, "obs_max_gust_kt": 20.0, "obs_max_wind_kt": 13, - "obs_mean_dewpoint_f": 66.61500000000001, + "obs_mean_dewpoint_f": 66.615, "obs_mean_f": 77.43625, "obs_total_precip_in": 0.0, "station": "MIA" @@ -7960,7 +7960,7 @@ "obs_max_wind_kt": 10, "obs_mean_dewpoint_f": 69.59809523809524, "obs_mean_f": 74.88095238095238, - "obs_total_precip_in": 2.7099999999999995, + "obs_total_precip_in": 2.71, "station": "MIA" }, { @@ -8026,7 +8026,7 @@ "obs_max_wind_kt": 14, "obs_mean_dewpoint_f": 70.73619047619047, "obs_mean_f": 74.53333333333333, - "obs_total_precip_in": 0.9300000000000002, + "obs_total_precip_in": 0.93, "station": "MIA" } ] diff --git a/tests/fixtures/parity/ts/manifest.json b/tests/fixtures/parity/ts/manifest.json index 8d0ecba6..1855a1bb 100644 --- a/tests/fixtures/parity/ts/manifest.json +++ b/tests/fixtures/parity/ts/manifest.json @@ -111,7 +111,7 @@ }, "from": "2024-12-01", "row_count": 365, - "sha256": "a1ed77dbe7e248505e96e1dd501b5feb4c5925e045a6c5512f820c04f92d47f6", + "sha256": "6d38652ecb1d83e6ad4d40b639a90923bf660255ab4f0895fabfa3e181c8d4e6", "station": "KMIA", "to": "2025-11-30" }, From e86bc5ac9befc2a9c3ee002e2428fd127b40c9e9 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 08:50:59 +0200 Subject: [PATCH 47/52] chore(30-06): lockstep version bump to 1.13.0 across all 7 dists - 3 pyproject.toml + 4 package.json: 1.12.1 -> 1.13.0 (same SHA lockstep) - 4 TS 'export const version' literals fixed: 0.0.0 -> 1.13.0 (were stale; now on the release checklist per 30-06 plan) - hello.test.ts version assertions updated in lockstep (codegen stays 0.0.0, non-published) - root workspace pyproject + root/codegen package.json left at 0.0.0 - inter-package >= pins untouched (minor bump per RELEASE-RUNBOOK) - Python __version__ verified metadata-derived: core/weather/markets all report 1.13.0 from dist metadata, zero literal edits - dry version-guards green: release-ts-preflight (vts-1.13.0, temp copy) + release.yml PEP440 guard logic (v1.13.0) --- packages-ts/core/package.json | 2 +- packages-ts/core/src/index.ts | 2 +- packages-ts/core/tests/hello.test.ts | 2 +- packages-ts/markets/package.json | 2 +- packages-ts/markets/src/index.ts | 2 +- packages-ts/markets/tests/hello.test.ts | 2 +- packages-ts/meta/package.json | 2 +- packages-ts/meta/src/index.ts | 2 +- packages-ts/meta/tests/hello.test.ts | 8 ++++---- packages-ts/weather/package.json | 2 +- packages-ts/weather/src/index.ts | 2 +- packages-ts/weather/tests/hello.test.ts | 2 +- packages/core/pyproject.toml | 2 +- packages/markets/pyproject.toml | 2 +- packages/weather/pyproject.toml | 2 +- 15 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages-ts/core/package.json b/packages-ts/core/package.json index c82dc83a..955128d1 100644 --- a/packages-ts/core/package.json +++ b/packages-ts/core/package.json @@ -1,6 +1,6 @@ { "name": "@mostlyrightmd/core", - "version": "1.12.1", + "version": "1.13.0", "description": "TypeScript SDK core for quants, ML pipelines, and AI agents: types, schemas, validators, temporal-safety primitives, and the research() join over weather data + prediction-market settlements. Local-first, no hosted backend.", "keywords": [ "weather", diff --git a/packages-ts/core/src/index.ts b/packages-ts/core/src/index.ts index 383f6bfb..12ed1357 100644 --- a/packages-ts/core/src/index.ts +++ b/packages-ts/core/src/index.ts @@ -6,7 +6,7 @@ * authoritative package version lives in `package.json#version` * (currently `0.1.0-rc.7`); this constant has not been bumped. */ -export const version = "0.0.0"; +export const version = "1.13.0"; /** * Smoke-test export from the TS-W0 Wave 1 scaffold. Returns the literal diff --git a/packages-ts/core/tests/hello.test.ts b/packages-ts/core/tests/hello.test.ts index b6f8227f..1cdf87c1 100644 --- a/packages-ts/core/tests/hello.test.ts +++ b/packages-ts/core/tests/hello.test.ts @@ -4,7 +4,7 @@ import { helloCore, version } from "../src/index.js"; describe("@mostlyrightmd/core hello-world scaffold", () => { it("exports the placeholder version string", () => { - expect(version).toBe("0.0.0"); + expect(version).toBe("1.13.0"); }); it("returns the expected hello string", () => { diff --git a/packages-ts/markets/package.json b/packages-ts/markets/package.json index 6582cadc..969b148f 100644 --- a/packages-ts/markets/package.json +++ b/packages-ts/markets/package.json @@ -1,6 +1,6 @@ { "name": "@mostlyrightmd/markets", - "version": "1.12.1", + "version": "1.13.0", "description": "Prediction-market data for TypeScript / Node — Kalshi NHIGH/NLOW weather-contract resolvers, Polymarket discovery + settlement, and Kalshi + Polymarket trade history. For quants, backtesting, and ML training pipelines.", "keywords": [ "kalshi", diff --git a/packages-ts/markets/src/index.ts b/packages-ts/markets/src/index.ts index 0a6f9f2a..0ec12501 100644 --- a/packages-ts/markets/src/index.ts +++ b/packages-ts/markets/src/index.ts @@ -6,7 +6,7 @@ * authoritative package version lives in `package.json#version` * (currently `0.1.0-rc.7`); this constant has not been bumped. */ -export const version = "0.0.0"; +export const version = "1.13.0"; /** * Smoke-test export from the TS-W0 Wave 1 scaffold. Returns the literal diff --git a/packages-ts/markets/tests/hello.test.ts b/packages-ts/markets/tests/hello.test.ts index 48c5c9c1..58779d7d 100644 --- a/packages-ts/markets/tests/hello.test.ts +++ b/packages-ts/markets/tests/hello.test.ts @@ -4,7 +4,7 @@ import { helloMarkets, version } from "../src/index.js"; describe("@mostlyrightmd/markets hello-world scaffold", () => { it("exports the placeholder version string", () => { - expect(version).toBe("0.0.0"); + expect(version).toBe("1.13.0"); }); it("returns the expected hello string", () => { diff --git a/packages-ts/meta/package.json b/packages-ts/meta/package.json index 1a8fd79a..08f44755 100644 --- a/packages-ts/meta/package.json +++ b/packages-ts/meta/package.json @@ -1,6 +1,6 @@ { "name": "mostlyright", - "version": "1.12.1", + "version": "1.13.0", "description": "Public-data SDK for TypeScript — one import for quants, ML pipelines, and AI agents. Adapters ship weather (METAR, ASOS, GHCNh, NWS CLI) and prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket) today; SEC filings, Federal Reserve series, court filings, FDA approvals, and equities are next. Local-first, no hosted backend.", "keywords": [ "weather", diff --git a/packages-ts/meta/src/index.ts b/packages-ts/meta/src/index.ts index 22f32882..e8009857 100644 --- a/packages-ts/meta/src/index.ts +++ b/packages-ts/meta/src/index.ts @@ -158,4 +158,4 @@ export const Preprocessing = preprocessing; * (`@mostlyrightmd/core` / `weather` / `markets`) each export their own * `version` constant, exposed here via the namespaced module objects. */ -export const version = "0.0.0"; +export const version = "1.13.0"; diff --git a/packages-ts/meta/tests/hello.test.ts b/packages-ts/meta/tests/hello.test.ts index 4f5dca52..3d19b656 100644 --- a/packages-ts/meta/tests/hello.test.ts +++ b/packages-ts/meta/tests/hello.test.ts @@ -12,7 +12,7 @@ import { describe("mostlyright (meta) hello-world scaffold", () => { it("exports the placeholder version string", () => { - expect(version).toBe("0.0.0"); + expect(version).toBe("1.13.0"); }); it("re-exports helloCore from @mostlyrightmd/core", () => { @@ -28,8 +28,8 @@ describe("mostlyright (meta) hello-world scaffold", () => { }); it("namespaces each underlying package's version constant", () => { - expect(core.version).toBe("0.0.0"); - expect(weather.version).toBe("0.0.0"); - expect(markets.version).toBe("0.0.0"); + expect(core.version).toBe("1.13.0"); + expect(weather.version).toBe("1.13.0"); + expect(markets.version).toBe("1.13.0"); }); }); diff --git a/packages-ts/weather/package.json b/packages-ts/weather/package.json index a7f693a5..341054f6 100644 --- a/packages-ts/weather/package.json +++ b/packages-ts/weather/package.json @@ -1,6 +1,6 @@ { "name": "@mostlyrightmd/weather", - "version": "1.12.1", + "version": "1.13.0", "description": "Weather data for TypeScript / Node — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Direct public-API access, no hosted backend.", "keywords": [ "weather", diff --git a/packages-ts/weather/src/index.ts b/packages-ts/weather/src/index.ts index f768ed00..327f9208 100644 --- a/packages-ts/weather/src/index.ts +++ b/packages-ts/weather/src/index.ts @@ -9,7 +9,7 @@ * authoritative package version lives in `package.json#version` * (currently `0.1.0-rc.7`); this constant has not been bumped. */ -export const version = "0.0.0"; +export const version = "1.13.0"; /** * Smoke-test export from the TS-W0 Wave 1 scaffold. Returns the literal diff --git a/packages-ts/weather/tests/hello.test.ts b/packages-ts/weather/tests/hello.test.ts index d2b84f39..e3dc9275 100644 --- a/packages-ts/weather/tests/hello.test.ts +++ b/packages-ts/weather/tests/hello.test.ts @@ -4,7 +4,7 @@ import { helloWeather, version } from "../src/index.js"; describe("@mostlyrightmd/weather hello-world scaffold", () => { it("exports the placeholder version string", () => { - expect(version).toBe("0.0.0"); + expect(version).toBe("1.13.0"); }); it("returns the expected hello string", () => { diff --git a/packages/core/pyproject.toml b/packages/core/pyproject.toml index ce207f87..1f3812ce 100644 --- a/packages/core/pyproject.toml +++ b/packages/core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mostlyrightmd" -version = "1.12.1" +version = "1.13.0" description = "Python SDK for quants, ML engineers, and AI agents — one interface to public data. Adapters ship weather + prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket) today; SEC filings, Federal Reserve series, court filings, FDA approvals, and equities are next. Schema-versioned, leakage-free, local-first. Imports as `mostlyright`." readme = "README.md" license = "MIT" diff --git a/packages/markets/pyproject.toml b/packages/markets/pyproject.toml index e2eb4b50..293103ef 100644 --- a/packages/markets/pyproject.toml +++ b/packages/markets/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mostlyrightmd-markets" -version = "1.12.1" +version = "1.13.0" description = "Prediction-market data for Python — Kalshi NHIGH/NLOW weather-contract resolvers, Polymarket discovery + settlement, and Kalshi + Polymarket trade history. For quants, backtesting, and ML training pipelines. Imports as `mostlyright.markets`." readme = "README.md" license = "MIT" diff --git a/packages/weather/pyproject.toml b/packages/weather/pyproject.toml index 1d6a8a6b..a251dfe6 100644 --- a/packages/weather/pyproject.toml +++ b/packages/weather/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mostlyrightmd-weather" -version = "1.12.1" +version = "1.13.0" description = "Weather data for Python — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Direct public-API access, no hosted backend. Imports as `mostlyright.weather`." readme = "README.md" license = "MIT" From e4b0865f1f75877d18d68b5ec645df481d5b6a5f Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 08:52:29 +0200 Subject: [PATCH 48/52] =?UTF-8?q?docs(30-06):=20CHANGELOG=20[1.13.0]=20?= =?UTF-8?q?=E2=80=94=20observation-API=20unification=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - dataset() rename + research alias; obs(granularity=observation) + merged.live_v1; NEW climate() (Python top-level, TS /climate subpath); D-16 covariate composition with labels-only firewall + retrain-event pointer; output-knob unification; mode-vocab retirement - Deprecated section: research_by_source (with the NOT-drop-in coverage caveat), mode2 import path, as_dataframe, TS MODE2_* aliases; -W error callout - Fixed section: TS station-format guard (incl. satelliteHosted), pinned-awc fail-loud, case-4 fixture re-baseline + v0.14.1-plane decommission fact - [Unreleased] kept empty per pre-release gate --- CHANGELOG.md | 112 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7288e02e..886c7159 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,118 @@ All notable changes to `mostlyright`. The format follows [Keep a Changelog](http ## [Unreleased] +## [1.13.0] — 2026-07-09 — Observation-API unification: dataset(), climate(), covariates, one mode + +Minor release, dual-SDK (PyPI 1.13.0 + npm 1.13.0 in lockstep). Unifies the +observation API onto a single `dataset()`/`research()` surface, ships a +standalone `climate()` settlement table, adds an opt-in labels-only covariate +seam, and starts the deprecation train for the numbered-mode era. Every +existing entry point is byte-stable at runtime this release; see +[`docs/migration-1.13.md`](docs/migration-1.13.md) — headline: **migrating a +pinned feature is a retrain event, not a find-replace.** + +### Added + +- **`dataset()` — the new canonical name for the training-pairs join (both + SDKs).** `research()` is retained as an alias bound to the same callable + (`dataset is research`); no deprecation warning on `research()` this + release, byte-identical output. Python: `mostlyright.dataset(...)`; TS: + `import { dataset } from "mostlyright"`. +- **`obs(granularity="observation")` — per-report observation grain + (Python).** Returns the merged per-report METAR/SPECI rows (window-trimmed, + no daily bucketing) with per-row source identity preserved and + `observation_type`/`raw_metar` surfaced. Unpinned frames carry the new + frame-identity tag `merged.live_v1`, accepted only by the new + `schema.observation.merged.v1` — the frozen `schema.observation.v1` + rejects it, so fused frames can never masquerade as single-source. The + default `granularity="daily"` output is byte-identical to 1.12.1. TS + `obs()` gains the same option with a deliberate divergence: its default is + `"observation"` (preserving existing TS per-report behavior); + `"daily"` throws a documented not-yet-ported error (parity ticket filed). + Observation-grain frames stamp `retrieved_at` provenance. +- **`climate()` — standalone NWS CLI settlement-label table (both SDKs).** + The daily `cli_high_f`/`cli_low_f` labels previously visible only inside + the `dataset()` join are now a first-class, source-identified domain table + in the JOIN column vocabulary, window-trimmed (no whole-year leak), frame + tag `cli.archive` with truthful per-row endpoint tag `iem`. Python: + `mostlyright.weather.climate(...)`; TS: `import { climate } from + "@mostlyrightmd/weather/climate"` (lean subpath — keeps the root barrel + inside its size budget). TS `climate()` resolves NWS↔ICAO station codes + through the station registry so the `station` column matches Python + cross-SDK. +- **`dataset(include_satellite=, include_cwop=)` — opt-in covariate + composition (Python).** LEFT-joins per-LST-settlement-day `sat_*` / + `cwop_*` covariate columns onto the composed training frame behind flags + defaulting `False`. **Labels-only firewall:** label columns are + byte-identical flags-on vs flags-off, row count/order unchanged, missing + days are NaN (never dropped), and satellite/CWOP can never become + observation sources — the four parity-firewall files are untouched and + grep-gated. CWOP covariates are QC-clean-only and read-only (no APRS + sockets); satellite covariates count only scans that contribute a usable + value (`sat_scan_count`). Requires the `[satellite]` / `[cwop]` extras; + missing extras fail loud with a typed error. **Adding a covariate to a + trained model is itself a retrain event** — see the migration guide. TS + parity for the covariate flags is tracked as a CROSS-SDK-SYNC ticket + (browser runtime cannot run local extraction). +- **`docs/source-identity.md`** — the single cross-vertical kwarg contract + page: `source=` is always provenance, `delivery=` is always local|hosted, + grain is vertical-defined, frame-vs-row identity, the labels-only + firewall, and the train/trade symmetry example. + +### Changed + +- **Output-knob unification (D-06): `backend=` + `return_type=` is the one + output convention.** `obs()` gains `backend`/`return_type`; + `return_type="list"` added across the dispatch layer. TS `return_type` + accepts `"dataframe"` as the canonical value (the historical `"frame"` + remains an alias). Every function's no-argument default return is + byte-stable. +- **Numbered-mode vocabulary retired** across docstrings, runtime strings, + and docs (`source must be one of ...`, `source-pinned dispatch requested + ...`); forecast surfaces are now described as IEM MOS vs per-NWP-model. + TS neutral names `SOURCE_PINNED_SOURCES`/`SourcePinnedSource`/ + `isSourcePinnedSource` are canonical. +- **TS weather bundle budget raised 26 → 27 KB** (TS-BUNDLE-01) to admit the + station-format guard + pinned-AWC fail-loud hardening; measured 26.23 KB. + +### Deprecated + +All deprecations warn but behave identically; removal target is ≥2 minor +releases out. Production pipelines running `-W error` (Python) will surface +these as errors — see the migration guide before tightening warning filters. + +- **`research_by_source()`** → use `obs(source=..., granularity= + "observation")`. **Caveat: this is NOT a drop-in swap for pinned + features** — the new path returns pre-merge full single-source coverage, + a different (typically larger) row composition than `research_by_source`'s + post-merge filter. Retrain/re-backtest before switching production + strategies (`docs/migration-1.13.md`). +- **`mostlyright.mode2` import path** → `obs(...)` from + `mostlyright.weather` and `assert_source_identity` from + `mostlyright.core` (its new canonical home, re-exported). +- **`as_dataframe=`** on `dataset()`/`research()`, `research_by_source()`, + and `obs()` → `return_type=` (`True` → `"dataframe"`, `False` → `"list"`; + an explicit `return_type=` wins). +- **TS `MODE2_SOURCES`/`Mode2Source`/`isMode2Source`** → the + `SOURCE_PINNED_*` names (old identifiers remain as aliases binding the + same values; removal at 2.0). + +### Fixed + +- **TS station-format guard on every exported weather network surface** + (including the hosted satellite client): malformed station input now fails + loud before any URL is built or network dispatched. +- **TS pinned `source: "awc"` queries fail loud on fetch errors** instead of + silently returning an empty array (unpinned merge path unchanged). +- **Parity case-4 fixture (KMIA rolling year) re-baselined 2026-07-09:** + upstream restated 3 of 365 days after the May capture (mean aggregates + only; observation counts, settlement labels, and extremes byte-identical), + and the v0.14.1 hosted data plane the original fixtures were captured from + has since been decommissioned, making a fresh v0.14.1 re-capture + permanently impossible. The prior v0.14.1-captured fixture is preserved in + git history; provenance documented in `tests/fixtures/parity/README.md`. + Cases 1/2/3/5 remain v0.14.1-captured and byte-green. + ## [1.12.1] — 2026-07-08 — #107 follow-ups: cross-SDK consistency + version metadata Patch release, dual-SDK. Resolves the P3 follow-ups from the 1.12.0 review From 62c5485dfd5d159331289616ba804c59b04a18e0 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 08:53:39 +0200 Subject: [PATCH 49/52] docs(30-06): add docs/migration-1.13.md - Headline: migrating pinned features = retrain event, not find-replace (pre-merge full-coverage obs(source=, granularity=observation) vs research_by_source post-merge composition difference, D-12) - Compatibility table: byte-stable / warns-identical / new-optional - -W error production callout with targeted allowlist example - TS notes: dataset alias, granularity default divergence, /climate subpath, MODE2_* aliases, return_type dataframe/frame - Deprecation timeline (>=2 minors, removals at 2.0); links docs/source-identity.md --- docs/migration-1.13.md | 165 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 docs/migration-1.13.md diff --git a/docs/migration-1.13.md b/docs/migration-1.13.md new file mode 100644 index 00000000..49afd715 --- /dev/null +++ b/docs/migration-1.13.md @@ -0,0 +1,165 @@ +# Migrating to 1.13.0 + +1.13.0 unifies the observation API onto one surface: `dataset()` is the +composed training table, domain tables (`obs()`, new `climate()`, +`forecasts()`, `satellite()`, `cwop.*`) are the raw source-identified +ingredients, and the numbered-mode era (`mostlyright.mode2`) starts its +deprecation train. **Every existing entry point is byte-stable at runtime +this release** — nothing you call today returns different data. But if you +maintain models trained on *pinned* single-source features, read the next +section before touching anything. + +The kwarg contract behind all of this lives in +[`docs/source-identity.md`](source-identity.md). + +--- + +## The headline: migrating pinned features is a RETRAIN EVENT, not a find-replace + +The deprecation message on `research_by_source()` points you at the new +pinned path: + +```python +# old (deprecated, still works, still byte-identical to 1.12.x) +from mostlyright.mode2 import research_by_source +rows = research_by_source("iem.archive", "KNYC", "2025-01-06", "2025-01-12") + +# new +from mostlyright.weather import obs +rows = obs("KNYC", "2025-01-06", "2025-01-12", + source="iem.archive", granularity="observation") +``` + +**These two calls do NOT return the same row set, by design:** + +- `research_by_source()` filters **post-merge**: the AWC > IEM > GHCNh + fusion runs first, then rows from your pinned source are selected from + the fused result. Rows your source reported but the merge superseded + (because a higher-priority source covered the same timestamp) are + **absent**. +- `obs(source=..., granularity="observation")` fetches **pre-merge** via + the source-isolated exact-window path: you get your source's **full + coverage** for the window — typically **more rows** than the old path, + and a different per-day composition. + +A feature pipeline built on the old post-merge rows and re-pointed at the +new pre-merge rows will produce different aggregates, different +distributions, and therefore a different model. **Do not swap the call in a +production strategy and keep trading.** Treat the swap like a data-source +change: + +1. Rebuild the training frame on the new path. +2. Retrain. +3. Re-backtest against settlements. +4. Only then promote. + +If you are not ready to retrain, do nothing: `research_by_source()` keeps +its exact post-merge behavior for the whole deprecation window (≥2 minor +releases). The warning is a schedule, not a behavior change. + +The same logic applies to the new covariate flags: adding +`include_satellite=True` / `include_cwop=True` to `dataset()` never changes +label columns (byte-identical on vs off), but a model that starts consuming +`sat_*`/`cwop_*` columns is a new model. Retrain deliberately. + +--- + +## Compatibility table + +### Byte-stable at runtime — no action, no warning + +| Surface | Status in 1.13.0 | +|---|---| +| `dataset()` / `research()` default path | Identical output; `dataset is research` (same callable). `research()` emits **no** warning this release; a `DeprecationWarning` arrives in a later release, removal at 2.0. | +| `research_by_source(...)` behavior | Output byte-identical to 1.12.x (keeps its post-merge path). Only a warning is added. | +| `obs(...)` daily default | `granularity="daily"` is the Python default; no-argument calls unchanged. | +| `live.latest` / `live.stream`, `snapshot` | Untouched. | +| Merge semantics, cache layout, CWOP firewall | Frozen (parity gate green; the four firewall files are unchanged). | +| Default no-argument returns of every function | Preserved verbatim (`daily_extremes` still defaults to list, etc.). | + +### Warns (DeprecationWarning), behaves identically — migrate within ≥2 minors + +| Deprecated | Replacement | Notes | +|---|---|---| +| `research_by_source(...)` (call-time warning) | `obs(source=..., granularity="observation")` | **Retrain event** — see above. | +| `import mostlyright.mode2` (import-path warning, once per session) | `obs(...)` from `mostlyright.weather`; `assert_source_identity` from `mostlyright.core` | `assert_source_identity`'s canonical home is now `mostlyright.core.source_identity` (same object, re-exported). | +| `as_dataframe=` on `dataset()`/`research()`, `research_by_source()`, `obs()` | `return_type=` (`True` → `"dataframe"`, `False` → `"list"`) | An explicit `return_type=` wins when both are passed (still warns). | + +### New and optional — adopt when ready + +| Surface | What it is | +|---|---| +| `dataset()` | Canonical name for the training join. Start using it in new code. | +| `obs(granularity="observation")` | Per-report merged rows, window-trimmed, `observation_type` + `raw_metar` surfaced; unpinned frames tagged `merged.live_v1` (only `schema.observation.merged.v1` accepts them). | +| `climate(station, start, end)` | Standalone NWS CLI settlement-label table (JOIN vocabulary: `cli_high_f`/`cli_low_f`/`cli_report_type`), frame tag `cli.archive`, per-row source `iem`. | +| `dataset(include_satellite=, include_cwop=)` | Labels-only covariate columns (`sat_*`, `cwop_*`), LEFT-joined per LST settlement day; needs the `[satellite]`/`[cwop]` extras; missing extras fail loud. | +| `backend=` + `return_type=` everywhere | The one output convention going forward. | + +--- + +## Production warning filters (`-W error`) + +If your pipeline promotes warnings to errors (`python -W +error::DeprecationWarning`, `filterwarnings = ["error"]` in pytest, etc.), +1.13.0 **will** surface the new `DeprecationWarning`s as failures at: + +- any `research_by_source()` call, +- any `mostlyright.mode2` attribute access (once per session), +- any explicit `as_dataframe=` argument. + +Either migrate those call sites first, or add a targeted allowlist while +you complete the migration window, e.g.: + +```python +import warnings +warnings.filterwarnings( + "default", + category=DeprecationWarning, + module=r"mostlyright(\..*)?", +) +``` + +`research()` itself emits nothing this release, so the most common +production entry point is unaffected. + +--- + +## TypeScript notes + +- **`dataset` alias:** `import { dataset } from "mostlyright"` — + `dataset === research`, same callable, no warning on `research`. +- **`obs()` granularity default DIVERGES from Python (deliberate):** TS + `obs()` defaults to `granularity: "observation"` — the per-report rows TS + has always returned — so existing TS callers stay byte-stable. + `granularity: "daily"` is not yet ported in TS and throws a documented + error (CROSS-SDK-SYNC parity ticket); it never silently returns + per-report rows. Python's default remains `"daily"`. If you run both + SDKs, pass `granularity` explicitly at the seam. +- **`climate()` lives at a subpath:** + `import { climate } from "@mostlyrightmd/weather/climate"` (not the root + barrel — browser bundle-budget adaptation). Output matches Python's + column contract; station codes resolve NWS↔ICAO identically. +- **`MODE2_*` renamed with aliases:** `SOURCE_PINNED_SOURCES` / + `SourcePinnedSource` / `isSourcePinnedSource` are canonical; + `MODE2_SOURCES` / `Mode2Source` / `isMode2Source` remain as deprecated + aliases binding the same values (removal at 2.0). +- **`return_type`:** `"dataframe"` is canonical; `"frame"` stays as a + back-compat alias. + +--- + +## Deprecation timeline + +| Release | What happens | +|---|---| +| **1.13.0** (this release) | Warnings added; zero behavior change. | +| 1.14.x+ | `research()` gains its `DeprecationWarning` (alias keeps working). | +| ≥2 minors out | Earliest removal window for `research_by_source`, `mostlyright.mode2`, `as_dataframe`. | +| 2.0 | Removals land; TS `MODE2_*` aliases removed. | + +## See also + +- [`docs/source-identity.md`](source-identity.md) — the cross-vertical + `source=` / `delivery=` / grain contract, frame-vs-row identity, and the + labels-only covariate firewall. +- `CHANGELOG.md` § [1.13.0] — the complete change list. From 52092873e21b52a9ae0c843e5642cba8f4f64962 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 08:54:00 +0200 Subject: [PATCH 50/52] chore(30-06): refresh uv.lock for 1.13.0 workspace versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pnpm-lock.yaml requires no change: inter-package deps use the workspace protocol and no external dependency changed (verified via pnpm install --lockfile-only — zero diff). --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 8d58d0ca..4573a3d3 100644 --- a/uv.lock +++ b/uv.lock @@ -1628,7 +1628,7 @@ wheels = [ [[package]] name = "mostlyrightmd" -version = "1.12.1" +version = "1.13.0" source = { editable = "packages/core" } dependencies = [ { name = "httpx" }, @@ -1672,7 +1672,7 @@ provides-extras = ["parquet", "research", "polars"] [[package]] name = "mostlyrightmd-markets" -version = "1.12.1" +version = "1.13.0" source = { editable = "packages/markets" } dependencies = [ { name = "httpx" }, @@ -1726,7 +1726,7 @@ provides-extras = ["parquet", "polymarket", "trades", "earnings", "polars"] [[package]] name = "mostlyrightmd-weather" -version = "1.12.1" +version = "1.13.0" source = { editable = "packages/weather" } dependencies = [ { name = "filelock" }, From b4e84ec747f0febe3b47f23c1d8cb8de9218a255 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 09:36:54 +0200 Subject: [PATCH 51/52] style(30): ruff format drift on _backend_dispatch + observation_merged (CI format gate) --- .../core/src/mostlyright/core/_backend_dispatch.py | 1 + .../mostlyright/core/schemas/observation_merged.py | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/core/src/mostlyright/core/_backend_dispatch.py b/packages/core/src/mostlyright/core/_backend_dispatch.py index e5756b3d..bf733989 100644 --- a/packages/core/src/mostlyright/core/_backend_dispatch.py +++ b/packages/core/src/mostlyright/core/_backend_dispatch.py @@ -101,6 +101,7 @@ def resolve_return_type( return return_type return "dataframe" if as_dataframe else "list" + BackendT = Literal["pandas", "polars"] ReturnTypeT = Literal["dataframe", "wrapper", "list"] diff --git a/packages/core/src/mostlyright/core/schemas/observation_merged.py b/packages/core/src/mostlyright/core/schemas/observation_merged.py index 6dd1f3c8..ef4aad7c 100644 --- a/packages/core/src/mostlyright/core/schemas/observation_merged.py +++ b/packages/core/src/mostlyright/core/schemas/observation_merged.py @@ -68,9 +68,7 @@ class MergedObservationSchema(Schema): #: validates the per-row ``source`` column as MEMBERSHIP in this set #: (no nulls) INSTEAD of equality-to-``df.attrs["source"]``. Every other #: schema leaves this None → the legacy equality check is byte-unchanged. - _registered_row_sources: ClassVar[frozenset[str] | None] = frozenset( - {"awc", "iem", "ghcnh"} - ) + _registered_row_sources: ClassVar[frozenset[str] | None] = frozenset({"awc", "iem", "ghcnh"}) COLUMNS: ClassVar[list[ColumnSpec]] = [ # --- Identity columns (non-nullable) ------------------------------ @@ -112,13 +110,9 @@ class MergedObservationSchema(Schema): ColumnSpec(name="wind_speed_kt", dtype="float64", units="kt", nullable=True), ColumnSpec(name="wind_gust_kt", dtype="float64", units="kt", nullable=True), ColumnSpec(name="altimeter_inhg", dtype="float64", units="inHg", nullable=True), - ColumnSpec( - name="sea_level_pressure_mb", dtype="float64", units="mb", nullable=True - ), + ColumnSpec(name="sea_level_pressure_mb", dtype="float64", units="mb", nullable=True), ColumnSpec(name="visibility_miles", dtype="float64", units="miles", nullable=True), - ColumnSpec( - name="precip_1hr_inches", dtype="float64", units="inches", nullable=True - ), + ColumnSpec(name="precip_1hr_inches", dtype="float64", units="inches", nullable=True), # --- Sky layers (nullable) ---------------------------------------- ColumnSpec( name="sky_cover_1", From 7cca995af8179c5f335474647cd70ac6fc5bf5e3 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Thu, 9 Jul 2026 09:46:23 +0200 Subject: [PATCH 52/52] =?UTF-8?q?fix(30):=20NA-robust=20guards=20in=20cova?= =?UTF-8?q?riate=20helpers=20=E2=80=94=20pandas-3=20string=20dtype=20turns?= =?UTF-8?q?=20None=20into=20pd.NA,=20bypassing=20'is=20None'=20(satellite?= =?UTF-8?q?=20variable=20leak,=20cwop=20observed=5Fat/sid=20hazard)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/mostlyright/weather/cwop/_covariates.py | 17 +++++++++++++++-- .../weather/satellite/_covariates.py | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/weather/src/mostlyright/weather/cwop/_covariates.py b/packages/weather/src/mostlyright/weather/cwop/_covariates.py index 4e3b8e34..5de62596 100644 --- a/packages/weather/src/mostlyright/weather/cwop/_covariates.py +++ b/packages/weather/src/mostlyright/weather/cwop/_covariates.py @@ -132,6 +132,19 @@ def _clean_history_rows(pws_ids: list[str], from_date: date, to_date: date) -> l return rows +def _is_na(x: object) -> bool: + """NA-robust scalar check: None, float NaN, and pandas NA (pandas-3 string + dtype turns object-column ``None`` into ``pd.NA``, which ``is None`` misses + and which RAISES in boolean context). No pandas import needed: ``pd.NA != pd.NA`` + is ``pd.NA``, whose ``bool()`` raises TypeError.""" + if x is None: + return True + try: + return bool(x != x) + except TypeError: + return True + + def cwop_daily_covariates( station: str, from_date: str, @@ -211,14 +224,14 @@ def cwop_daily_covariates( day_obs: dict[str, int] = {} for r in rows: observed_at = r.get("observed_at") - if observed_at is None: + if _is_na(observed_at): continue settle_day = settlement_date_for(observed_at, station, tz_override=tz_override) if settle_day not in requested: continue day_obs[settle_day] = day_obs.get(settle_day, 0) + 1 sid = r.get("station_id") - if sid is not None: + if not _is_na(sid): day_stations.setdefault(settle_day, set()).add(str(sid)) temp = r.get("temp_f") if temp is None: diff --git a/packages/weather/src/mostlyright/weather/satellite/_covariates.py b/packages/weather/src/mostlyright/weather/satellite/_covariates.py index 1fc76c7f..5bdac35e 100644 --- a/packages/weather/src/mostlyright/weather/satellite/_covariates.py +++ b/packages/weather/src/mostlyright/weather/satellite/_covariates.py @@ -56,6 +56,19 @@ def _settlement_window_bounds( return win_start, win_end +def _is_na(x: object) -> bool: + """NA-robust scalar check: None, float NaN, and pandas NA (pandas-3 string + dtype turns object-column ``None`` into ``pd.NA``, which ``is None`` misses + and which RAISES in boolean context). No pandas import needed: ``pd.NA != pd.NA`` + is ``pd.NA``, whose ``bool()`` raises TypeError.""" + if x is None: + return True + try: + return bool(x != x) + except TypeError: + return True + + def satellite_daily_covariates( station: str, from_date: str, @@ -105,11 +118,11 @@ def satellite_daily_covariates( counts: dict[str, int] = {} for _, row in df.iterrows(): scan_ts = row.get("scan_start_utc") - if not scan_ts: + if _is_na(scan_ts) or not str(scan_ts): continue variable = row.get("variable") value = row.get("pixel_value") - if variable is None or value is None: + if _is_na(variable) or _is_na(value): continue # Skip NaN pixel values (a _FillValue scan) so the mean is over real cells. try: