diff --git a/CHANGELOG.md b/CHANGELOG.md index 19746b41..5ed53a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,44 @@ All notable changes to `mostlyright`. The format follows [Keep a Changelog](http ## [Unreleased] +## [1.12.0] — 2026-07-08 — research(source=) pin + Mode 2 window fix + +Minor release, dual-SDK (PyPI 1.12.0 + npm 1.12.0 in lockstep). + +### Added + +- **`research(source=...)` single-source pin (both SDKs).** The singular + `source=` kwarg is now functional (was `NotImplementedError` since Phase 10): + observation rows are filtered to the requested source's alias-accept set + after the multi-source merge and before daily `obs_*` aggregation. Accepts + the full Mode 2 vocabulary (`iem`, `iem.archive`, `iem.live`, `awc`, + `awc.live`, `ghcnh`, `ghcnh.archive`) — byte-identical alias tables across + Python (`mostlyright.mode2._SOURCE_ALIASES`) and TS + (`RESEARCH_SOURCE_ALIASES`). Unknown sources raise before any network + fetch. `cli_*` settlement columns are unaffected (NWS CLI is independent + of the observation pin); the pinned DataFrame carries + `df.attrs["source"]`. Inherited v0.1 limitation documented: the + AWC > IEM > GHCNh merge runs before the pin filter, so pinning a + lower-priority source yields only rows the merge did not supersede. + `sources=[...]` (plural, multi-source subset) remains v0.3; its error + message no longer claims `source=` is unwired. The `source=None` default + path is byte-identical to 1.11.0 (parity gate green). TS `source: null` + follows the Phase 21 wire contract (null-as-absent). Follow-up nits: #107. + ### Fixed +- **`mode2.research_by_source` returned whole calendar months instead of the + requested window.** The month-granular parquet cache flows through + `_fetch_observations_range` unfiltered, so a `KNYC` Jan 6–12 query + returned all 865 January rows (a Jan 28–Feb 3 query returned two full + months) — silent out-of-window leakage into backtest frames, including + the README's own `train = research_by_source(...)` example. Rows are now + trimmed to `[from_date, to_date]` by UTC date-part, inclusive, matching + the TS port's shipped filter exactly (cross-SDK lockstep); the +1-day + fetch extension remains as a fetch-width knob only. Every return shape + is fixed (DataFrame, `as_dataframe=False`, wrapper/backends). Callers + needing the last LST day's pre-midnight UTC tail should extend + `to_date` by one day. - **Earnings venue-rule compound-word under-count (D-30).** Hyphenated compounds (e.g. `supply-chain` for `chain`) were dropped from BOTH venue counts by an inverted guard; they now count on both venues per each issuer's actual rule. diff --git a/package.json b/package.json index ff769b68..053c8beb 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ { "name": "@mostlyrightmd/weather (W1: AWC+CLI only, gzipped ESM)", "path": "packages-ts/weather/dist/index.mjs", - "limit": "25 KB" + "limit": "26 KB" }, { "name": "@mostlyrightmd/markets (W1: Kalshi NHIGH/NLOW, gzipped ESM)", diff --git a/packages-ts/core/package.json b/packages-ts/core/package.json index 16e1b5e2..df013116 100644 --- a/packages-ts/core/package.json +++ b/packages-ts/core/package.json @@ -1,6 +1,6 @@ { "name": "@mostlyrightmd/core", - "version": "1.11.0", + "version": "1.12.0", "description": "TypeScript SDK core for quants, ML pipelines, and AI agents: types, schemas, validators, temporal-safety primitives, and the research() join over weather data + prediction-market settlements. Local-first, no hosted backend.", "keywords": [ "weather", diff --git a/packages-ts/markets/package.json b/packages-ts/markets/package.json index 3c229576..dbe5561e 100644 --- a/packages-ts/markets/package.json +++ b/packages-ts/markets/package.json @@ -1,6 +1,6 @@ { "name": "@mostlyrightmd/markets", - "version": "1.11.0", + "version": "1.12.0", "description": "Prediction-market data for TypeScript / Node — Kalshi NHIGH/NLOW weather-contract resolvers, Polymarket discovery + settlement, and Kalshi + Polymarket trade history. For quants, backtesting, and ML training pipelines.", "keywords": [ "kalshi", diff --git a/packages-ts/meta/package.json b/packages-ts/meta/package.json index 5058a748..0ae058d3 100644 --- a/packages-ts/meta/package.json +++ b/packages-ts/meta/package.json @@ -1,6 +1,6 @@ { "name": "mostlyright", - "version": "1.11.0", + "version": "1.12.0", "description": "Public-data SDK for TypeScript — one import for quants, ML pipelines, and AI agents. Adapters ship weather (METAR, ASOS, GHCNh, NWS CLI) and prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket) today; SEC filings, Federal Reserve series, court filings, FDA approvals, and equities are next. Local-first, no hosted backend.", "keywords": [ "weather", diff --git a/packages-ts/meta/src/mode2.ts b/packages-ts/meta/src/mode2.ts index be400431..2d950016 100644 --- a/packages-ts/meta/src/mode2.ts +++ b/packages-ts/meta/src/mode2.ts @@ -80,6 +80,51 @@ export function isMode2Source(value: unknown): value is Mode2Source { return typeof value === "string" && (MODE2_SOURCES as readonly string[]).includes(value); } +// ── research({ source }) filter vocabulary — the WIDENED superset ────────── +// `researchBySource` (Mode 2 dispatch) narrows to the four dotted forms +// because a live-IEM fetcher gap means bare/live inputs can't be dispatched. +// `research({ source })` is a FILTER over already-fetched, already-merged +// rows, so it accepts Python's full vocabulary. The accepted names and +// alias-accept sets below are byte-identical to Python +// `mostlyright.mode2._VALID_OBSERVATION_SOURCES` / `_SOURCE_ALIASES` +// (packages/core/src/mostlyright/mode2.py:50-63). + +/** + * research()-filter alias table — the Python-parity superset of + * {@link SOURCE_ALIASES}. Extends the narrow dispatch table (the four dotted + * forms) with the three bare parser tags (`iem`, `awc`, `ghcnh`) rather than + * re-declaring the shared dotted rows, so the dotted aliases have one source + * of truth. Mirrors Python `_SOURCE_ALIASES` (7 entries) verbatim. + */ +export const RESEARCH_SOURCE_ALIASES: ReadonlyMap> = new Map< + string, + ReadonlySet +>([ + ...SOURCE_ALIASES, + ["iem", new Set(["iem", "iem.archive", "iem.live"])], + ["awc", new Set(["awc", "awc.live"])], + ["ghcnh", new Set(["ghcnh", "ghcnh.archive"])], +]); + +/** + * Accepted `research({ source })` names — the KEYS of + * {@link RESEARCH_SOURCE_ALIASES}. Byte-identical to Python + * `_VALID_OBSERVATION_SOURCES`: `iem`, `iem.archive`, `iem.live`, `awc`, + * `awc.live`, `ghcnh`, `ghcnh.archive`. + */ +export const RESEARCH_OBSERVATION_SOURCES: ReadonlySet = new Set( + RESEARCH_SOURCE_ALIASES.keys(), +); + +/** + * Type-guard: true iff `value` is an accepted `research({ source })` name. + * WIDER than {@link isMode2Source} — accepts bare parser tags too, matching + * Python's `source in _VALID_OBSERVATION_SOURCES` membership check. + */ +export function isResearchObservationSource(value: unknown): boolean { + return typeof value === "string" && RESEARCH_OBSERVATION_SOURCES.has(value); +} + /** * Throw {@link SourceMismatchError} if any row's `source` field * disagrees with the expected source vocabulary. Rows missing the diff --git a/packages-ts/meta/src/research.ts b/packages-ts/meta/src/research.ts index c42ab1a6..5543a561 100644 --- a/packages-ts/meta/src/research.ts +++ b/packages-ts/meta/src/research.ts @@ -52,6 +52,11 @@ import { parseGhcnhPsv, parseIemCsv, } from "@mostlyrightmd/weather"; +import { + RESEARCH_OBSERVATION_SOURCES, + RESEARCH_SOURCE_ALIASES, + isResearchObservationSource, +} from "./mode2.js"; import { type ResearchKwargsExtension, validateResearchKwargs } from "./research.types.js"; // Re-export PairsRow so callers can `import { research, type PairsRow } from "mostlyright"`. @@ -109,9 +114,15 @@ export interface ResearchOptions extends ResearchKwargsExtension { * StationOverrideWarning via `onWarning?`; output row carries * `settlementMismatch: true`. Only valid with `contract` selector. */ stationOverride?: string; - /** Mode 1 source subset — dedupe within. Mutually exclusive with `source`. */ + /** Mode 1 multi-source SUBSET selector — dedupe within. Validated in v0.2; + * its dedupe-within-subset wiring lands in v0.3 (passing it throws). + * Mutually exclusive with `source`. */ sources?: ReadonlyArray; - /** Mode 2 single-source pin — error on mismatch. Mutually exclusive with `sources`. */ + /** Mode 2 single-source pin. FUNCTIONAL: filters the merged observation + * rows to the requested source's alias set before daily obs_* aggregation + * (cli_* unaffected). Accepts any name in `RESEARCH_OBSERVATION_SOURCES` + * (the four dotted forms plus bare `iem`/`awc`/`ghcnh`). Mutually + * exclusive with `sources`. */ source?: string; /** Attach per-issuer trade timeseries via @mostlyrightmd/markets/trades. * Requires `contract` or `contracts`. */ @@ -785,7 +796,11 @@ async function fetchGhcnhWithCache( * in `[fromDate, toDate]`. Each row carries: * - `cli_*` populated from IEM CLI (final preferred per `mergeClimate`). * - `obs_*` daily aggregates over the 3-source merged observations - * (AWC > IEM > GHCNh per `mergeObservations`). + * (AWC > IEM > GHCNh per `mergeObservations`). When `opts.source` is set + * (Mode 2 single-source pin), obs rows are filtered to that source's + * alias set AFTER the merge — so pinning a lower-priority source yields + * only the rows the merge did not supersede (see the inline note); + * `cli_*` is unaffected. * - `fcst_*` unconditionally null (Mode 1). * - `market_close_utc` formatted `YYYY-MM-DDTHH:MM:SSZ`. * @@ -854,18 +869,35 @@ export async function research( if (hasContracts) names.push("contracts"); throw new Error(`research(): selectors are mutually exclusive; got ${JSON.stringify(names)}`); } - if (opts.sources !== undefined && opts.source !== undefined) { + if (opts.sources !== undefined && opts.source != null) { throw new Error("research(): sources and source are mutually exclusive"); } - // Iter-1 codex HIGH: sources / source validation is shipped in Phase 10 - // v0.2 but the data-selection wiring lands in v0.3. Without this guard - // the station-path runs the full multi-source merge regardless — silent - // data-selection corruption. - if (opts.sources !== undefined || opts.source !== undefined) { + // `sources` (plural) — the Mode 1 multi-source SUBSET selector — is + // validated in Phase 10 v0.2 but its dedupe-within-subset wiring lands in + // v0.3; passing it throws. NOTE: `source` (singular) is now FUNCTIONAL + // (single-source pin, wired below) — only the multi-source subset remains + // deferred. + if (opts.sources !== undefined) { + throw new Error( + "research(): the `sources` multi-source subset selector is validated in " + + "Phase 10 v0.2 but its dedupe-within-subset wiring lands in v0.3. For " + + "single-source pinning today pass `source=` (functional now), or use " + + "`researchBySource(station, source, ...)` from @mostlyrightmd/meta.", + ); + } + // `source` (singular) — Mode 2 single-source pin. Validate against the + // widened Mode 2 vocabulary BEFORE any network fetch so invalid input + // never burns API quota. Mirrors Python `research_by_source`'s ValueError + // guard (packages/core/src/mostlyright/mode2.py:112-115); the accepted + // names are byte-identical across SDKs (RESEARCH_OBSERVATION_SOURCES). + // Presence is nullish (`!= null`), NOT `!== undefined`: per the Phase 21 + // wire contract (research.types.ts `present()`), a JSON-round-tripped + // Python `source=None` arrives as `null` and must behave as ABSENT. + if (opts.source != null && !isResearchObservationSource(opts.source)) { throw new Error( - "research(): sources / source validation surface is shipped in Phase 10 v0.2 " + - "but the data-selection wiring lands in v0.3. For Mode 2 single-source pinning " + - "today, use `researchBySource(station, source, ...)` from @mostlyrightmd/meta.", + `Mode 2 source must be one of ${JSON.stringify( + [...RESEARCH_OBSERVATION_SOURCES].sort(), + )}; got ${JSON.stringify(opts.source)}`, ); } if (opts.stationOverride !== undefined && !hasContract) { @@ -977,12 +1009,32 @@ export async function research( const sorted = sortByObservedAtThenSource(combinedRaw); const merged = mergeObservations(sorted); + // ── Mode 2 single-source pin (opts.source) ──────────────────────────── + // When `source` is set, keep only merged observation rows whose + // parser-emitted `source` tag is in the alias-accept set for the requested + // source — applied BEFORE daily obs_* aggregation. The cli_* settlement + // columns are UNAFFECTED (CLI is independent of the observation pin). + // Mirrors Python `mode2.research_by_source`'s post-merge source filter. + // + // Inherited limitation (Python mode2.py:85-93 parity): the AWC > IEM > + // GHCNh merge above runs on the full multi-source set, so pinning e.g. + // `iem` returns only the IEM rows AWC did NOT supersede on a colliding + // (station_code, observed_at, observation_type) key — not the full + // upstream IEM coverage. Callers needing pre-merge isolation should call + // the per-source fetchers in @mostlyrightmd/weather directly. + let observationsForBuild: ReadonlyArray = merged; + if (opts.source != null) { + const accept = RESEARCH_SOURCE_ALIASES.get(opts.source); + // `accept` is guaranteed defined — `opts.source` validated above. + observationsForBuild = merged.filter((r) => accept?.has(r.source) ?? false); + } + const observationsByDate: Record = {}; // dates is guaranteed non-empty by buildDateList contract (throws on // fromDate > toDate; both validated above). const dateLo = dates[0] ?? ""; const dateHi = dates[dates.length - 1] ?? ""; - for (const obs of merged) { + for (const obs of observationsForBuild) { const settleDate = observedSettlementDate(obs.observed_at, resolved.code); if (settleDate === null) continue; if (settleDate < dateLo || settleDate > dateHi) continue; diff --git a/packages-ts/meta/tests/compose.test.ts b/packages-ts/meta/tests/compose.test.ts index 56ee536f..44b93665 100644 --- a/packages-ts/meta/tests/compose.test.ts +++ b/packages-ts/meta/tests/compose.test.ts @@ -239,10 +239,15 @@ describe("research() Phase 10 signature validation", () => { ).rejects.toThrow(/v0.3/); }); - it("source alone raises v0.3 deferred (iter-1 codex HIGH)", async () => { + it("source alone is now FUNCTIONAL — an unknown source is rejected before any fetch", async () => { + // `source=` (single-source pin) is wired in this PR; it no longer defers + // to v0.3. Validation of an unknown source fires BEFORE any network call + // (mirrors Python `research_by_source`'s ValueError guard), so this stays + // network-free. The functional filter path over mocked fetchers is + // covered in research.test.ts. await expect( - research("KNYC", "2025-01-06", "2025-01-12", { source: "iem.archive" }), - ).rejects.toThrow(/researchBySource/); + research("KNYC", "2025-01-06", "2025-01-12", { source: "not-a-real-source" }), + ).rejects.toThrow(/Mode 2 source must be one of/); }); it("stationOverride requires contract", async () => { diff --git a/packages-ts/meta/tests/mode2.test.ts b/packages-ts/meta/tests/mode2.test.ts index 93774ee4..7d3fb61d 100644 --- a/packages-ts/meta/tests/mode2.test.ts +++ b/packages-ts/meta/tests/mode2.test.ts @@ -246,6 +246,26 @@ describe("researchBySource — iem.archive dispatch", () => { }); expect(rows).toEqual([]); }); + + it("window boundary: keeps rows ON fromDate + ON toDate; drops fromDate-1 + toDate+1", async () => { + // The window filter is inclusive on both ends (observed_at.slice(0,10) + // >= fromDate && <= toDate). Pin the exact boundary rows. + const csv = iemCsv([ + makeIemRow({ valid: "2024-06-09 12:51" }), // fromDate-1 → DROP + makeIemRow({ valid: "2024-06-10 12:51" }), // fromDate → KEEP + makeIemRow({ valid: "2024-06-20 12:51" }), // toDate → KEEP + makeIemRow({ valid: "2024-06-21 12:51" }), // toDate+1 → DROP + ]); + installFetchMock({ iemAsosCsv: csv }); + const rows = await researchBySource("NYC", "iem.archive", "2024-06-10", "2024-06-20", { + iemPolitenessMs: 0, + }); + const days = new Set(rows.map((r) => r.observed_at.slice(0, 10))); + expect(days.has("2024-06-10")).toBe(true); // ON fromDate kept + expect(days.has("2024-06-20")).toBe(true); // ON toDate kept + expect(days.has("2024-06-09")).toBe(false); // fromDate-1 dropped + expect(days.has("2024-06-21")).toBe(false); // toDate+1 dropped + }); }); describe("researchBySource — awc.live dispatch", () => { @@ -315,6 +335,26 @@ describe("researchBySource — awc.live dispatch", () => { const outDates = rows.map((r) => r.observed_at.slice(0, 10)); expect(outDates).not.toContain(outOfWindowFuture); }); + + it("window boundary: keeps rows ON fromDate + ON toDate; drops fromDate-1 + toDate+1", async () => { + // Multi-day window pins fromDate and toDate as DISTINCT boundaries (the + // sibling test above uses a degenerate single-day window). The AWC branch + // filters on observed_at.slice(0,10) inclusive on both ends. The mock + // returns records verbatim regardless of `hours`, so fixed dates are safe. + const awc: AwcMockMetar[] = [ + { icaoId: "KNYC", obsTime: epochOfUtc("2024-06-09T12:00:00Z"), temp: 20 }, // fromDate-1 DROP + { icaoId: "KNYC", obsTime: epochOfUtc("2024-06-10T12:00:00Z"), temp: 21 }, // fromDate KEEP + { icaoId: "KNYC", obsTime: epochOfUtc("2024-06-20T12:00:00Z"), temp: 22 }, // toDate KEEP + { icaoId: "KNYC", obsTime: epochOfUtc("2024-06-21T12:00:00Z"), temp: 23 }, // toDate+1 DROP + ]; + installFetchMock({ awc }); + const rows = await researchBySource("NYC", "awc.live", "2024-06-10", "2024-06-20"); + const days = new Set(rows.map((r) => r.observed_at.slice(0, 10))); + expect(days.has("2024-06-10")).toBe(true); // ON fromDate kept + expect(days.has("2024-06-20")).toBe(true); // ON toDate kept + expect(days.has("2024-06-09")).toBe(false); // fromDate-1 dropped + expect(days.has("2024-06-21")).toBe(false); // toDate+1 dropped + }); }); describe("researchBySource — ghcnh.archive dispatch", () => { diff --git a/packages-ts/meta/tests/research.test.ts b/packages-ts/meta/tests/research.test.ts index 383edd54..2466a097 100644 --- a/packages-ts/meta/tests/research.test.ts +++ b/packages-ts/meta/tests/research.test.ts @@ -360,3 +360,191 @@ describe("research() — TS-W1 Wave 6 (AWC + CLI)", () => { expect(rows[0]?.obs_high_f).not.toBeNull(); }); }); + +// --------------------------------------------------------------------------- +// Mode 2 single-source pin — research({ source }) +// --------------------------------------------------------------------------- +// +// `source=` filters the merged observation rows to the requested source's +// alias set BEFORE daily obs_* aggregation (cli_* is unaffected). The fixture +// below carries one AWC row (40°C → 104°F, the MAX), one IEM CSV row +// (tmpf=75 → 75°F, parsed as both METAR + SPECI = 2 rows), and one GHCNh row +// (20°C → 68°F, the MIN). Distinct observed_at values keep all four rows past +// `mergeObservations`, so pinning provably drops the non-matching sources. + +const PIN_DAY = "2025-06-10"; +const PIN_NOW = new Date("2025-06-10T20:00:00Z"); // inside AWC's 168h window + +const IEM_HEADER = + "station,valid,tmpf,dwpf,drct,sknt,gust,alti,mslp,vsby,skyc1,skyl1,skyc2,skyl2,skyc3,skyl3,skyc4,skyl4,wxcodes,p01i,snowdepth,peak_wind_gust,peak_wind_drct,peak_wind_time,metar"; + +function makeIemRow(overrides: Record = {}): string { + const defaults: Record = { + station: "NYC", + valid: `${PIN_DAY} 12:51`, + tmpf: "75", + dwpf: "55", + drct: "180", + sknt: "8", + gust: "", + alti: "29.92", + mslp: "1013.2", + vsby: "10", + skyc1: "FEW", + skyl1: "2500", + skyc2: "", + skyl2: "", + skyc3: "", + skyl3: "", + skyc4: "", + skyl4: "", + wxcodes: "", + p01i: "", + snowdepth: "", + peak_wind_gust: "", + peak_wind_drct: "", + peak_wind_time: "", + metar: "KNYC 101251Z 18008KT 10SM FEW025 24/13 A2992 RMK AO2", + ...overrides, + }; + return IEM_HEADER.split(",") + .map((c) => defaults[c] ?? "") + .join(","); +} + +function iemCsv(rows: string[]): string { + return `${[IEM_HEADER, ...rows].join("\n")}\n`; +} + +const GHCNH_HEADER = + "temperature_Source_Station_ID|DATE|temperature|temperature_Quality_Code|dew_point_temperature|dew_point_temperature_Quality_Code"; + +function ghcnhPsv(date: string, tempC: string): string { + return `${GHCNH_HEADER}\nICAO-KNYC|${date}|${tempC}|0|20.0|0\n`; +} + +// Mixed-source fixture: AWC (104°F, max) + IEM (75°F ×2) + GHCNh (68°F, min). +function installMixedSourceMock() { + return installFetchMock({ + awc: [ + { + icaoId: "KNYC", + obsTime: epochOfUtc(`${PIN_DAY}T18:00:00Z`), + temp: 40, // 104°F — MAX; excluded whenever the pin isn't awc + dewp: 10, + wspd: 5, + }, + ], + iemAsosCsv: iemCsv([makeIemRow({ valid: `${PIN_DAY} 12:51`, tmpf: "75" })]), + ghcnhPsv: ghcnhPsv(`${PIN_DAY}T06:00:00Z`, "20.0"), // 68°F — MIN + ghcnhStatus: 200, + }); +} + +describe("research() — Mode 2 single-source pin (opts.source)", () => { + beforeEach(() => vi.restoreAllMocks()); + afterEach(() => vi.restoreAllMocks()); + + it("source:'iem' computes obs_* from IEM rows only (AWC max temp excluded)", async () => { + installMixedSourceMock(); + const rows = await research("NYC", PIN_DAY, PIN_DAY, { + source: "iem", + now: PIN_NOW, + cache: null, + }); + expect(rows).toHaveLength(1); + const row = rows[0]; + // IEM contributes 2 rows (METAR + SPECI off the same CSV), both 75°F. + expect(row?.obs_count).toBe(2); + expect(row?.obs_high_f).toBe(75); // NOT 104 (AWC) → AWC provably excluded + expect(row?.obs_low_f).toBe(75); // NOT 68 (GHCNh) → GHCNh provably excluded + // cli_* is independent of the obs pin (no CLI fixture → null). + expect(row?.cli_high_f).toBeNull(); + }); + + it("source:'awc.live' accepts bare 'awc'-tagged rows via the alias set", async () => { + installMixedSourceMock(); + const rows = await research("NYC", PIN_DAY, PIN_DAY, { + source: "awc.live", + now: PIN_NOW, + cache: null, + }); + expect(rows).toHaveLength(1); + const row = rows[0]; + // Only the single bare-'awc' row survives ('awc' ∈ alias set of 'awc.live'). + expect(row?.obs_count).toBe(1); + expect(row?.obs_high_f).toBe(104); + expect(row?.obs_low_f).toBe(104); + }); + + it("unknown source throws BEFORE any fetch", async () => { + const fetchSpy = installMixedSourceMock(); + await expect( + research("NYC", PIN_DAY, PIN_DAY, { source: "banana", now: PIN_NOW, cache: null }), + ).rejects.toThrow(/Mode 2 source must be one of/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("source + sources together still throws the mutual-exclusion error", async () => { + const fetchSpy = installFetchMock({}); + await expect( + research("NYC", PIN_DAY, PIN_DAY, { + source: "iem", + sources: ["awc"], + now: PIN_NOW, + cache: null, + }), + ).rejects.toThrow(/mutually exclusive/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("sources:['iem'] alone still throws — rewritten message (no longer claims `source` is unwired)", async () => { + const fetchSpy = installFetchMock({}); + let err: unknown = null; + try { + await research("NYC", PIN_DAY, PIN_DAY, { sources: ["iem"], now: PIN_NOW, cache: null }); + } catch (e) { + err = e; + } + expect(err).toBeInstanceOf(Error); + const msg = (err as Error).message; + expect(msg).toMatch(/sources/); + expect(msg).toMatch(/v0\.3/); + // The rewritten message points at the now-functional `source=`. + expect(msg).toMatch(/functional now/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("no `source` → full merge unchanged (golden: AWC max + all 4 rows present)", async () => { + installMixedSourceMock(); + const rows = await research("NYC", PIN_DAY, PIN_DAY, { now: PIN_NOW, cache: null }); + expect(rows).toHaveLength(1); + const row = rows[0]; + // Golden for the unpinned path: full 3-source merge — AWC 104°F max, + // GHCNh 68°F min, all four merged rows counted, mean over all four. + expect(row?.obs_count).toBe(4); + expect(row?.obs_high_f).toBe(104); + expect(row?.obs_low_f).toBe(68); + expect(row?.obs_mean_f).toBe(80.5); + }); + + it("source: null (JSON round-trip of Python None) behaves as ABSENT — full merge, no throw", async () => { + // Phase 21 wire contract (research.types.ts `present()`): a Python + // `source=None` crosses the JSON wire as `null` and must be treated as + // not-provided, identically to omitting the key. Codex P2 regression + // guard: `!== undefined` presence tests would throw the unknown-source + // error here. + installMixedSourceMock(); + const rows = await research("NYC", PIN_DAY, PIN_DAY, { + source: null as unknown as string, + now: PIN_NOW, + cache: null, + }); + expect(rows).toHaveLength(1); + const row = rows[0]; + expect(row?.obs_count).toBe(4); + expect(row?.obs_high_f).toBe(104); + expect(row?.obs_low_f).toBe(68); + expect(row?.obs_mean_f).toBe(80.5); + }); +}); diff --git a/packages-ts/weather/package.json b/packages-ts/weather/package.json index 85244385..eae22685 100644 --- a/packages-ts/weather/package.json +++ b/packages-ts/weather/package.json @@ -1,6 +1,6 @@ { "name": "@mostlyrightmd/weather", - "version": "1.11.0", + "version": "1.12.0", "description": "Weather data for TypeScript / Node — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Direct public-API access, no hosted backend.", "keywords": [ "weather", diff --git a/packages/core/pyproject.toml b/packages/core/pyproject.toml index 6812e940..3096be2f 100644 --- a/packages/core/pyproject.toml +++ b/packages/core/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mostlyrightmd" -version = "1.11.0" +version = "1.12.0" description = "Python SDK for quants, ML engineers, and AI agents — one interface to public data. Adapters ship weather + prediction-market settlements (Kalshi NHIGH/NLOW, Polymarket) today; SEC filings, Federal Reserve series, court filings, FDA approvals, and equities are next. Schema-versioned, leakage-free, local-first. Imports as `mostlyright`." readme = "README.md" license = "MIT" diff --git a/packages/core/src/mostlyright/mode2.py b/packages/core/src/mostlyright/mode2.py index 5111d827..e4f3b4e8 100644 --- a/packages/core/src/mostlyright/mode2.py +++ b/packages/core/src/mostlyright/mode2.py @@ -63,6 +63,22 @@ } +def _in_query_window(observed_at: Any, from_date: str, to_date: str) -> bool: + """True iff ``observed_at``'s UTC date-part is in ``[from_date, to_date]``. + + Compares the ISO date-part slice (``observed_at[:10]``) inclusively at + both ends — a pure string comparison, no timezone math. The slice + handles both emitted forms (``"2025-01-01T00:51:00Z"`` and + ``"2025-01-06T12:00:00+00:00"``) identically. Rows with missing/empty + ``observed_at`` return False: they cannot be placed in any window. + Must match the TS port (``packages-ts/meta/src/mode2.ts``) for + cross-SDK lockstep. + """ + if not observed_at: + return False + return from_date <= observed_at[:10] <= to_date + + def research_by_source( station: str, source: str, @@ -102,7 +118,13 @@ def research_by_source( Returns: DataFrame (or list[dict]) with one row per observation matching - the requested source. + the requested source, bounded to ``[from_date, to_date]`` by UTC + date-part (``observed_at[:10]``), inclusive at both ends. The fetch + extends one calendar day past ``to_date`` to capture the last LST + day's pre-midnight UTC tail, but those extension rows are trimmed + out of the result. Callers doing LST settlement math who need that + pre-midnight UTC tail of the final LST day should pass a ``to_date`` + one day later than their settlement window. Raises: ValueError: ``source`` not in the supported set. @@ -131,8 +153,10 @@ def research_by_source( ) info = _resolve_station(station) - # Mirror research(): extend the fetch by one day so the last LST - # day's pre-midnight UTC tail observations are captured. + # Mirror research(): extend the fetch by one day so the last LST day's + # pre-midnight UTC tail observations are captured. The returned rows + # are trimmed back to the strict [from_date, to_date] window below — + # the extension is only a fetch-width knob, not a result-width one. extended_to = (_date.fromisoformat(to_date) + timedelta(days=1)).isoformat() awc_rows: list[dict[str, Any]] | None = None @@ -152,8 +176,22 @@ def research_by_source( # mostlyright' canonical vocab is dotted; the alias table bridges # both at the boundary without rewriting the per-row source — # downstream Validator sees the truthful parser-emitted tag). + # + # Trim to the strict [from_date, to_date] window on the same pass. + # `_fetch_observations_range` returns whole cached calendar months + # (month-granular parquet — research.py extends cached months verbatim) + # plus the +1-day fetch extension above, so raw_obs leaks rows outside + # the caller's window. This single filter fixes EVERY return shape + # (DataFrame, as_dataframe=False list, wrapper/backends) because it runs + # before any of them. Rows with missing/empty observed_at are dropped + # by `_in_query_window` (they cannot be placed in any window). accepted_sources = _SOURCE_ALIASES.get(source, {source}) - filtered = [r for r in raw_obs if r.get("source") in accepted_sources] + filtered = [ + r + for r in raw_obs + if r.get("source") in accepted_sources + and _in_query_window(r.get("observed_at"), from_date, to_date) + ] if not as_dataframe: return filtered diff --git a/packages/core/src/mostlyright/research.py b/packages/core/src/mostlyright/research.py index 53e7bd60..b073b424 100644 --- a/packages/core/src/mostlyright/research.py +++ b/packages/core/src/mostlyright/research.py @@ -1612,6 +1612,24 @@ def research( :data:`mostlyright.snapshot._STATION_TZ`. Passed through to settlement-date math; rarely needed for the US registry (all entries are covered). + source: Pin observations to a single named source (Mode 2). One of + :data:`mostlyright.mode2._VALID_OBSERVATION_SOURCES` (bare + ``"iem"``/``"awc"``/``"ghcnh"`` or the dotted canonical forms); + validated early, so an unknown value raises ``ValueError`` + before any fetch. When set, observation rows are filtered to the + requested source's alias-accept set before aggregation, and + ``df.attrs["source"]`` records the pin. The ``cli_*`` columns are + UNAFFECTED — NWS CLI is the settlement source, independent of the + observation-source pin. **v0.1.0 limitation:** the Mode 1 merge + policy (AWC > IEM > GHCNh on key collision) runs BEFORE the pin + filter, so pinning ``"iem"`` over a window where AWC also + reported yields only the IEM rows AWC did not supersede — not the + full upstream IEM coverage. ``source=None`` (default) leaves the + path byte-identical to the v0.14.1 parity baseline. + sources: Multi-source subset selector (plural). Validated at the + mutual-exclusion boundary but the data-selection wiring lands in + v0.3 — passing it raises ``NotImplementedError``. Mutually + exclusive with ``source=``. Returns: DataFrame (or ``list[dict]`` when ``as_dataframe=False``) with **one @@ -1704,22 +1722,38 @@ def research( if sources is not None and source is not None: raise ValueError("research(): sources= and source= are mutually exclusive") - # Iter-1 codex HIGH: sources= / source= are validated at the - # mutual-exclusion boundary but the actual data-selection wiring lands - # in v0.3. The station-path silently runs the full multi-source merge - # regardless of these kwargs, which would be silent data-selection - # corruption. Surface a clear NotImplementedError pointing callers at - # `mostlyright.mode2.research_by_source` (the Mode-2-pin path) until - # the kwargs are wired into the station-path dispatch. - if sources is not None or source is not None: + # `sources=` (plural, multi-source subset) is validated at the + # mutual-exclusion boundary but its data-selection wiring lands in + # v0.3. The singular `source=` pin IS wired below (a post-fetch + # single-source row subset), so only the plural form still raises. + if sources is not None: raise NotImplementedError( - "research(): sources= and source= validation surface is shipped in " - "Phase 10 v0.2 but the data-selection wiring lands in v0.3. For " - "Mode 2 single-source pinning today, use " - "`mostlyright.mode2.research_by_source(station, source, ...)` " - "directly. Mode 1 multi-source subset (sources=[...]) ships in v0.3." + "research(): the sources= multi-source subset selector is validated " + "(mutual exclusion) but its data-selection wiring lands in v0.3. Pin " + "a single observation source with source=... today; the multi-source " + "subset (sources=[...]) is the v0.3 deliverable." ) + # Mode 2 single-source pin. Validate source= EARLY — before any network + # or cache work — against Mode 2's source vocabulary, then compute the + # alias-accept set used to subset the raw observation rows after the + # fetch. Reuse mode2's tables rather than duplicating them. The import + # is function-local (not module-level): mode2 already imports research + # function-locally, so a top-level `from mostlyright.mode2 import ...` + # here would add a research->mode2 module edge and make the coupling + # bidirectional; a call-time import keeps it one-directional and is + # provably non-circular (mode2 has no top-level research import). + accepted_sources: frozenset[str] | None = None + if source is not None: + from mostlyright.mode2 import _SOURCE_ALIASES, _VALID_OBSERVATION_SOURCES + + if source not in _VALID_OBSERVATION_SOURCES: + raise ValueError( + f"research(): source must be one of " + f"{sorted(_VALID_OBSERVATION_SOURCES)}; got {source!r}" + ) + accepted_sources = _SOURCE_ALIASES.get(source, frozenset({source})) + # `station_override=` only makes sense when paired with `contract=`. if station_override is not None and not _has_contract: raise ValueError( @@ -1846,6 +1880,25 @@ def research( extended_to, prefetched_awc_rows=awc_rows, ) + + # Mode 2 single-source pin (source=): keep only rows whose parser- + # emitted `source` is in the requested alias-accept set. Applied as a + # pure post-fetch row subset BEFORE the settlement-date bucketing below, + # which already trims the window — so the pinned frame keeps the same + # columns/shape as unpinned research(). PARITY FIREWALL: this runs only + # when source is not None (accepted_sources stays None otherwise), so + # the default path is byte-identical. cli_* (NWS CLI settlement) columns + # are unaffected — CLI is the settlement source, independent of the + # observation-source pin. + # + # v0.1.0 limitation (inherited from Mode 2): _fetch_observations_range + # applies the Mode 1 merge policy (AWC > IEM > GHCNh on key collision) + # BEFORE this filter, so pinning "iem" over a window where AWC also + # reported yields only the IEM rows AWC did NOT supersede — not the full + # upstream IEM coverage. A pre-merge source-isolated path is v0.2 work. + if accepted_sources is not None: + raw_obs = [r for r in raw_obs if r.get("source") in accepted_sources] + raw_climate = _fetch_climate_range(info, from_date, to_date) # Phase 3.4: opt-in QC. Runs the QCEngine + IEM-vs-GHCNh crosscheck @@ -1912,6 +1965,16 @@ def research( with contextlib.suppress(AttributeError): result.attrs["qc"] = qc_summary + # Mode 2 provenance contract: stamp the requested source on the + # returned DataFrame when pinned (mirrors mode2.research_by_source's + # df.attrs["source"]). No attrs change when source is None — the + # parity firewall keeps the default path untouched. + if source is not None and as_dataframe: + import contextlib + + with contextlib.suppress(AttributeError): + result.attrs["source"] = source + # Phase 6 W3-T2: when as_dataframe=False the caller wants raw list[dict] — # backend/return_type kwargs do not apply. Same when backend kwarg is the # default (pandas + dataframe) — return unchanged for zero-overhead. diff --git a/packages/core/tests/test_compose.py b/packages/core/tests/test_compose.py index 97fc49c6..9e5ced2b 100644 --- a/packages/core/tests/test_compose.py +++ b/packages/core/tests/test_compose.py @@ -289,27 +289,34 @@ def test_sources_and_source_mutually_exclusive(self): def test_sources_only_raises_notimplemented_v03(self): """Iter-1 codex HIGH: sources= silently bypassed the data-fetch - wiring. v0.2 now raises NotImplementedError directing callers at - researchBySource for Mode-2 today + v0.3 for sources=[].""" + wiring. The plural multi-source subset stays deferred to v0.3; + the rewritten message no longer redirects source= at + research_by_source (source= is now wired — see Fix B).""" from mostlyright import research - with pytest.raises(NotImplementedError, match=r"v0\.3"): + with pytest.raises(NotImplementedError, match=r"v0\.3") as exc: research( station="NYC", from_date="2025-01-06", to_date="2025-01-12", sources=["iem.archive"], ) - - def test_source_only_raises_notimplemented_v03(self): + assert "research_by_source" not in str(exc.value) + + def test_source_only_is_wired_not_notimplemented(self): + """Fix B: source= is now wired (post-fetch single-source pin), so it + no longer raises NotImplementedError. It validates its vocabulary + early — an unknown source raises ValueError before any fetch; a + valid source runs the join. Full pin behaviour lives in + test_research_source_pin.py.""" from mostlyright import research - with pytest.raises(NotImplementedError, match="research_by_source"): + with pytest.raises(ValueError, match="source must be one of"): research( station="NYC", from_date="2025-01-06", to_date="2025-01-12", - source="iem.archive", + source="bogus.source", ) def test_station_override_requires_contract(self): diff --git a/packages/core/tests/test_mode2.py b/packages/core/tests/test_mode2.py index 6b87cadd..c12fa1bf 100644 --- a/packages/core/tests/test_mode2.py +++ b/packages/core/tests/test_mode2.py @@ -180,3 +180,108 @@ def test_assert_source_identity_no_source_column_is_noop(): def test_assert_source_identity_empty_df(): df = pd.DataFrame({"source": []}) assert_source_identity(df, "iem.archive") + + +# ---------------------------------------------------------------------- +# Fix A — window trim. `_fetch_observations_range` returns whole cached +# calendar months (month-granular parquet) plus the +1-day fetch +# extension, so raw_obs leaks rows outside the caller's window. Mode 2 +# must trim to [from_date, to_date] inclusive by UTC date-part. The +# pre-fix tests above only ever fed in-window rows, so they never caught +# this — these feed out-of-window rows on purpose. +# ---------------------------------------------------------------------- +def _patch_fetch(monkeypatch, tmp_path, rows: list[dict]) -> None: + """Stub the research-module fetch seam mode2 imports function-locally.""" + import importlib + + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + research_module = importlib.import_module("mostlyright.research") + monkeypatch.setattr(research_module, "_fetch_observations_range", lambda *a, **kw: rows) + monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) + + +def test_window_trim_dataframe_full_month(tmp_path, monkeypatch) -> None: + """A full month of rows is trimmed to just the queried [from, to] window. + + Boundary rows ON from_date (Jan 6) and ON to_date (Jan 12) are kept; + the row on to_date+1 (Jan 13, the +1-day fetch extension) is dropped. + """ + rows = [ + { + "station_code": "KNYC", + "observed_at": f"2025-01-{d:02d}T12:00:00Z", + "source": "iem", + "temp_c": 5.0, + } + for d in (1, 3, 5, 6, 10, 12, 13, 20, 31) + ] + _patch_fetch(monkeypatch, tmp_path, rows) + df = research_by_source("KNYC", "iem", "2025-01-06", "2025-01-12") + assert set(df["observed_at"]) == { + "2025-01-06T12:00:00Z", # ON from_date — kept + "2025-01-10T12:00:00Z", + "2025-01-12T12:00:00Z", # ON to_date — kept + } + + +def test_window_trim_list_full_month(tmp_path, monkeypatch) -> None: + """The same trim applies on the as_dataframe=False raw-rows path.""" + rows = [ + { + "station_code": "KNYC", + "observed_at": f"2025-01-{d:02d}T12:00:00Z", + "source": "iem", + "temp_c": 5.0, + } + for d in (1, 3, 5, 6, 10, 12, 13, 20, 31) + ] + _patch_fetch(monkeypatch, tmp_path, rows) + out = research_by_source("KNYC", "iem", "2025-01-06", "2025-01-12", as_dataframe=False) + assert isinstance(out, list) + assert {r["observed_at"] for r in out} == { + "2025-01-06T12:00:00Z", + "2025-01-10T12:00:00Z", + "2025-01-12T12:00:00Z", + } + + +def test_window_trim_month_straddling(tmp_path, monkeypatch) -> None: + """A Jan 28 to Feb 3 window keeps in-window rows from BOTH months only.""" + rows = [ + {"station_code": "KNYC", "observed_at": ts, "source": "iem", "temp_c": 5.0} + for ts in ( + "2025-01-27T12:00:00Z", # before window — dropped + "2025-01-28T12:00:00Z", # ON from_date — kept + "2025-01-31T12:00:00Z", # kept + "2025-02-01T12:00:00Z", # kept + "2025-02-03T12:00:00Z", # ON to_date — kept + "2025-02-04T12:00:00Z", # to_date+1 extension — dropped + ) + ] + _patch_fetch(monkeypatch, tmp_path, rows) + df = research_by_source("KNYC", "iem", "2025-01-28", "2025-02-03") + assert set(df["observed_at"]) == { + "2025-01-28T12:00:00Z", + "2025-01-31T12:00:00Z", + "2025-02-01T12:00:00Z", + "2025-02-03T12:00:00Z", + } + + +def test_window_trim_mixed_timestamp_formats_and_missing(tmp_path, monkeypatch) -> None: + """Both the "Z" and "+00:00" forms trim correctly via the [:10] slice; + rows with missing/empty observed_at are dropped (cannot be placed).""" + rows = [ + {"station_code": "KNYC", "observed_at": "2025-01-06T00:51:00Z", "source": "iem"}, + {"station_code": "KNYC", "observed_at": "2025-01-08T12:00:00+00:00", "source": "iem"}, + {"station_code": "KNYC", "observed_at": "2025-01-13T00:00:00+00:00", "source": "iem"}, + {"station_code": "KNYC", "observed_at": "2025-01-05T23:59:00Z", "source": "iem"}, + {"station_code": "KNYC", "observed_at": "", "source": "iem"}, + {"station_code": "KNYC", "source": "iem"}, # no observed_at key + ] + _patch_fetch(monkeypatch, tmp_path, rows) + df = research_by_source("KNYC", "iem", "2025-01-06", "2025-01-12") + assert set(df["observed_at"]) == { + "2025-01-06T00:51:00Z", # ON from_date, Z form — kept + "2025-01-08T12:00:00+00:00", # in-window, +00:00 form — kept + } diff --git a/packages/core/tests/test_research_source_pin.py b/packages/core/tests/test_research_source_pin.py new file mode 100644 index 00000000..232db6fa --- /dev/null +++ b/packages/core/tests/test_research_source_pin.py @@ -0,0 +1,190 @@ +"""Fix B — ``research(source=...)`` single-source pinning. + +Wires the singular ``source=`` pin (previously a ``NotImplementedError`` +stub) as a pure post-fetch row subset applied only when ``source`` is not +None. The plural ``sources=`` multi-source subset stays deferred to v0.3. + +The fetcher is monkeypatched (the research module's function-local seam, +per the established ``test_qc_wired.py`` pattern) so no test hits the +network. Observation rows carry ``temp_f`` because the pairs aggregation +(``_obs_aggregates``) reads that field for ``obs_high_f``/``obs_low_f``. +""" + +from __future__ import annotations + +import importlib + +import pandas as pd +import pytest + + +def _load(): + return importlib.import_module("mostlyright.research") + + +def _patch_fetchers(research_module, monkeypatch, tmp_path, obs) -> None: + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + monkeypatch.setattr(research_module, "_fetch_observations_range", lambda *a, **kw: list(obs)) + monkeypatch.setattr(research_module, "_fetch_climate_range", lambda *a, **kw: []) + monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) + + +def test_source_pin_filters_obs_aggregates(tmp_path, monkeypatch) -> None: + """With source="iem", obs_* reflect ONLY iem rows. + + All four rows bucket into the Jan 8 settlement date for KNYC (EST → + the mid-day UTC stamps share the same local date). The AWC row carries + the window max (99F) and the GHCNh row the min (20F); both must be + excluded so the pinned Jan 8 row shows the iem-only 45/40 spread. + """ + research_module = _load() + obs = [ + { + "station_code": "KNYC", + "observed_at": "2025-01-08T15:00:00Z", + "source": "iem", + "temp_f": 40.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-08T18:00:00Z", + "source": "iem", + "temp_f": 45.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-08T16:00:00Z", + "source": "awc", + "temp_f": 99.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-08T17:00:00Z", + "source": "ghcnh", + "temp_f": 20.0, + }, + ] + _patch_fetchers(research_module, monkeypatch, tmp_path, obs) + + df = research_module.research("KNYC", "2025-01-06", "2025-01-12", source="iem") + row = df.loc[pd.Timestamp("2025-01-08")] + assert row["obs_high_f"] == 45.0 + assert row["obs_low_f"] == 40.0 + assert row["obs_count"] == 2 + assert df.attrs["source"] == "iem" + + +def test_source_pin_alias_awc_live_accepts_bare_awc(tmp_path, monkeypatch) -> None: + """source="awc.live" accepts alias-tagged bare "awc" rows (alias table).""" + research_module = _load() + obs = [ + { + "station_code": "KNYC", + "observed_at": "2025-01-08T15:00:00Z", + "source": "awc", + "temp_f": 55.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-08T16:00:00Z", + "source": "iem", + "temp_f": 99.0, + }, + ] + _patch_fetchers(research_module, monkeypatch, tmp_path, obs) + + df = research_module.research("KNYC", "2025-01-06", "2025-01-12", source="awc.live") + row = df.loc[pd.Timestamp("2025-01-08")] + assert row["obs_high_f"] == 55.0 + assert row["obs_low_f"] == 55.0 + assert row["obs_count"] == 1 + assert df.attrs["source"] == "awc.live" + + +def test_unknown_source_raises_before_fetch(tmp_path, monkeypatch) -> None: + """An unknown source raises ValueError BEFORE any fetch/cache work.""" + research_module = _load() + monkeypatch.setenv("MOSTLYRIGHT_CACHE_DIR", str(tmp_path)) + + called = {"fetch": False} + + def _boom(*a, **kw): + called["fetch"] = True + raise AssertionError("fetch must not run for an invalid source") + + monkeypatch.setattr(research_module, "_fetch_observations_range", _boom) + monkeypatch.setattr(research_module, "_all_caches_warm", lambda *a, **kw: True) + + with pytest.raises(ValueError, match="source must be one of"): + research_module.research("KNYC", "2025-01-06", "2025-01-12", source="bogus.source") + assert called["fetch"] is False + + +def test_source_and_sources_together_raises_typeerror() -> None: + """The existing mutual-exclusion guard is untouched (source= + sources=).""" + research_module = _load() + with pytest.raises(TypeError, match="mutually exclusive"): + research_module.research("KNYC", "2025-01-06", "2025-01-12", source="iem", sources=["iem"]) + + +def test_sources_plural_still_not_implemented() -> None: + """sources=[...] alone stays NotImplementedError with the rewritten + message: points at v0.3 for the multi-source subset only, and no + longer redirects source= at research_by_source.""" + research_module = _load() + with pytest.raises(NotImplementedError, match=r"v0\.3") as exc: + research_module.research("KNYC", "2025-01-06", "2025-01-12", sources=["iem"]) + assert "research_by_source" not in str(exc.value) + + +def test_source_none_identical_to_unpinned(tmp_path, monkeypatch) -> None: + """source=None is a no-op: byte-identical frame + no attrs["source"].""" + research_module = _load() + obs = [ + { + "station_code": "KNYC", + "observed_at": "2025-01-08T15:00:00Z", + "source": "iem", + "temp_f": 40.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-08T16:00:00Z", + "source": "awc", + "temp_f": 50.0, + }, + ] + _patch_fetchers(research_module, monkeypatch, tmp_path, obs) + + df_default = research_module.research("KNYC", "2025-01-06", "2025-01-12") + df_none = research_module.research("KNYC", "2025-01-06", "2025-01-12", source=None) + pd.testing.assert_frame_equal(df_default, df_none) + assert "source" not in df_default.attrs + + +def test_source_pin_noop_when_all_rows_already_match(tmp_path, monkeypatch) -> None: + """Pinning "iem" over an all-iem window yields the same obs frame as + unpinned — the filter is a pure subset, not a perturbation.""" + research_module = _load() + obs = [ + { + "station_code": "KNYC", + "observed_at": "2025-01-08T15:00:00Z", + "source": "iem", + "temp_f": 40.0, + }, + { + "station_code": "KNYC", + "observed_at": "2025-01-08T18:00:00Z", + "source": "iem", + "temp_f": 45.0, + }, + ] + _patch_fetchers(research_module, monkeypatch, tmp_path, obs) + + df_pinned = research_module.research("KNYC", "2025-01-06", "2025-01-12", source="iem") + df_unpinned = research_module.research("KNYC", "2025-01-06", "2025-01-12") + pd.testing.assert_frame_equal( + df_pinned[["obs_high_f", "obs_low_f", "obs_count"]], + df_unpinned[["obs_high_f", "obs_low_f", "obs_count"]], + ) diff --git a/packages/markets/pyproject.toml b/packages/markets/pyproject.toml index 85a2670c..b56ee0db 100644 --- a/packages/markets/pyproject.toml +++ b/packages/markets/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mostlyrightmd-markets" -version = "1.11.0" +version = "1.12.0" description = "Prediction-market data for Python — Kalshi NHIGH/NLOW weather-contract resolvers, Polymarket discovery + settlement, and Kalshi + Polymarket trade history. For quants, backtesting, and ML training pipelines. Imports as `mostlyright.markets`." readme = "README.md" license = "MIT" diff --git a/packages/weather/pyproject.toml b/packages/weather/pyproject.toml index 52581f3e..2189c1a8 100644 --- a/packages/weather/pyproject.toml +++ b/packages/weather/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "mostlyrightmd-weather" -version = "1.11.0" +version = "1.12.0" description = "Weather data for Python — live METAR (AWC), ASOS archive (IEM), historical observations (GHCNh), and NWS climate text products (CLI). For quants, ML training pipelines, and weather-bot agents. Direct public-API access, no hosted backend. Imports as `mostlyright.weather`." readme = "README.md" license = "MIT" diff --git a/uv.lock b/uv.lock index 10768b27..44414400 100644 --- a/uv.lock +++ b/uv.lock @@ -1628,7 +1628,7 @@ wheels = [ [[package]] name = "mostlyrightmd" -version = "1.11.0" +version = "1.12.0" source = { editable = "packages/core" } dependencies = [ { name = "httpx" }, @@ -1672,7 +1672,7 @@ provides-extras = ["parquet", "research", "polars"] [[package]] name = "mostlyrightmd-markets" -version = "1.11.0" +version = "1.12.0" source = { editable = "packages/markets" } dependencies = [ { name = "httpx" }, @@ -1726,7 +1726,7 @@ provides-extras = ["parquet", "polymarket", "trades", "earnings", "polars"] [[package]] name = "mostlyrightmd-weather" -version = "1.11.0" +version = "1.12.0" source = { editable = "packages/weather" } dependencies = [ { name = "filelock" }, @@ -1764,6 +1764,7 @@ polars = [ satellite = [ { name = "boto3" }, { name = "eumdac" }, + { name = "fsspec" }, { name = "gcsfs" }, { name = "h5netcdf" }, { name = "h5py" }, @@ -1781,7 +1782,8 @@ requires-dist = [ { name = "eumdac", marker = "extra == 'satellite'", specifier = ">=3.1,<4.0" }, { name = "faster-whisper", marker = "extra == 'earnings'", specifier = ">=1.0,<2.0" }, { name = "filelock", specifier = ">=3.12" }, - { name = "gcsfs", marker = "extra == 'satellite'", specifier = ">=2024.0" }, + { name = "fsspec", marker = "extra == 'satellite'", specifier = "==2026.6.0" }, + { name = "gcsfs", marker = "extra == 'satellite'", specifier = "==2026.6.0" }, { name = "h5netcdf", marker = "extra == 'satellite'", specifier = ">=1.3" }, { name = "h5py", marker = "extra == 'satellite'", specifier = ">=3.0" }, { name = "httpx", specifier = ">=0.27" }, @@ -1798,7 +1800,7 @@ requires-dist = [ { name = "polars", marker = "extra == 'polars'", specifier = ">=1.0,<2.0" }, { name = "pyarrow", specifier = ">=17.0,<24.0" }, { name = "pyarrow", marker = "extra == 'polars'", specifier = ">=17.0,<24.0" }, - { name = "s3fs", marker = "extra == 'satellite'", specifier = ">=2024.0" }, + { name = "s3fs", marker = "extra == 'satellite'", specifier = "==2026.6.0" }, { name = "scikit-learn", marker = "extra == 'nwp'", specifier = ">=1.3,<2.0" }, { name = "tzdata", marker = "sys_platform == 'win32'" }, { name = "xarray", marker = "extra == 'nwp'", specifier = ">=2024.0" },