From 4cbcdf453d161af4fb1575303f4c11a424136a6b Mon Sep 17 00:00:00 2001 From: Gabor Szabo Date: Tue, 26 May 2026 07:08:14 +0200 Subject: [PATCH] feat(forecast): add feature frame v2 (#299) Lands V2 feature-frame contract as additive, opt-in surface alongside frozen V1. Training + scenarios + shared builders complete; backtesting V2 dispatch deferred to follow-up tracked in #299. V1 callers unchanged. - Shared layer: V2 manifest (38 default / 53 max columns), sidecars, row builders - Training: TrainRequest gains feature_frame_version + feature_groups (opt-in) - Scenarios: build_future_frame dispatches V1/V2 via bundle metadata - 3 LOAD-BEARING leakage specs land alongside the V1 spec - No Alembic migration (V2 reads existing tables, writes nothing) - V1 bundles load/predict/scenario-simulate/backtest unchanged --- PR-BODY-DRAFT.md | 147 +++ app/features/forecasting/routes.py | 2 + app/features/forecasting/schemas.py | 70 +- app/features/forecasting/service.py | 318 ++++- .../test_regression_features_v2_leakage.py | 124 ++ app/features/forecasting/v2_loaders.py | 349 ++++++ app/features/scenarios/feature_frame.py | 193 ++- app/features/scenarios/service.py | 25 + .../tests/test_future_frame_v2_leakage.py | 158 +++ app/shared/feature_frames/__init__.py | 79 +- app/shared/feature_frames/contract_v2.py | 370 ++++++ app/shared/feature_frames/rows_v2.py | 1034 +++++++++++++++++ app/shared/feature_frames/sidecar.py | 116 ++ .../feature_frames/tests/test_contract_v2.py | 288 +++++ .../feature_frames/tests/test_leakage_v2.py | 339 ++++++ .../10-baseforecaster-feature-contract.md | 40 + .../forecasting/feature_frame_v2_preview.py | 111 ++ 17 files changed, 3736 insertions(+), 27 deletions(-) create mode 100644 PR-BODY-DRAFT.md create mode 100644 app/features/forecasting/tests/test_regression_features_v2_leakage.py create mode 100644 app/features/forecasting/v2_loaders.py create mode 100644 app/features/scenarios/tests/test_future_frame_v2_leakage.py create mode 100644 app/shared/feature_frames/contract_v2.py create mode 100644 app/shared/feature_frames/rows_v2.py create mode 100644 app/shared/feature_frames/sidecar.py create mode 100644 app/shared/feature_frames/tests/test_contract_v2.py create mode 100644 app/shared/feature_frames/tests/test_leakage_v2.py create mode 100644 examples/forecasting/feature_frame_v2_preview.py diff --git a/PR-BODY-DRAFT.md b/PR-BODY-DRAFT.md new file mode 100644 index 00000000..077b5c64 --- /dev/null +++ b/PR-BODY-DRAFT.md @@ -0,0 +1,147 @@ +# feat(forecast): add feature frame v2 + +Tracking issue: **#299** (under Forecast Intelligence roadmap epic **#295**). +PRP: `PRPs/PRP-35-forecast-intelligence-A-feature-frame-v2.md`. + +## Summary + +Lands the **V2 feature-frame contract** as an **additive, opt-in** surface +alongside the frozen V1 contract: + +- **Shared layer** — V2 column manifest (38 default / 53 max columns across 11 + `FeatureGroup`s), `V2HistoricalSidecar` / `V2FutureSidecar` data carriers, + and `build_historical_feature_rows_v2` / `build_future_feature_rows_v2` + pure row builders. `app/shared/feature_frames/` stays leaf-level. +- **Training path** — `POST /forecasting/train` accepts optional + `feature_frame_version: int = 1` and `feature_groups: list[str] | None = None`. + V2 bundles persist `feature_frame_version`, `feature_columns`, + `feature_groups`, `feature_safety_classes`, and `feature_pinned_constants` + in bundle metadata. +- **Scenarios path** — `POST /scenarios/simulate` reads `feature_frame_version` + from the loaded bundle metadata and dispatches V1 vs V2 future-frame + assembly transparently. +- **LOAD-BEARING leakage specs** — three new specs land alongside the V1 + spec; never to be weakened: + - `app/shared/feature_frames/tests/test_leakage_v2.py` + - `app/features/forecasting/tests/test_regression_features_v2_leakage.py` + - `app/features/scenarios/tests/test_future_frame_v2_leakage.py` + +## V1 compatibility (back-compat invariant) + +- Every V1 export keeps its current signature, return type, and behaviour. +- The load-bearing V1 leakage spec + (`app/shared/feature_frames/tests/test_leakage.py`) and 22 sibling V1 + contract tests remain green **without modification**. +- V1 bundles trained before this PR load, predict, scenario-simulate, and + backtest unchanged. +- `feature_frame_version=1` is the default everywhere; legacy bundles that + predate the metadata field are treated as V1 via + `bundle.metadata.get("feature_frame_version", 1)`. +- `feature_frame_version` lives on `TrainRequest`, **not** on + `ModelConfigBase` — adding it to the config would mutate every existing V1 + `config_hash()` and orphan registry rows / aliases. Persisted to bundle + metadata instead. + +## V2 opt-in behaviour + +- A `TrainRequest` with `feature_frame_version=2` (optionally `feature_groups=[…]`) + triggers the V2 path; otherwise V1 runs unchanged. +- Validator gates: + - V1 + `feature_groups` supplied → 422. + - V2 + unknown `FeatureGroup` name → 422. +- Default V2 groups: `TARGET_HISTORY`, `CALENDAR`, `ROLLING`, `TREND`, + `PRICE_PROMO`, `LIFECYCLE` (38 columns). Phase-2 sidecar groups + (`INVENTORY`, `REPLENISHMENT`, `RETURNS`, `EXOGENOUS_WEATHER`, + `EXOGENOUS_MACRO`) are off by default so the MVP stays green on smaller + seeded DBs (max 53 columns when all enabled). +- Pinned V2 constants: `EXOGENOUS_LAGS_V2=(1,7,14,28,56,364)`, + `ROLLING_WINDOWS_V2=(7,28,90)`, `TREND_WINDOWS_V2=(30,90)`, + `HISTORY_TAIL_DAYS_V2=400`. + +## Validation + +All four mandatory gates green locally on `Python 3.12`: + +``` +✅ uv run ruff check . All checks passed +✅ uv run ruff format --check . 327 files already formatted +✅ uv run mypy app/ 0 PRP-35 errors (3 pre-existing xgboost noise on dev) +✅ uv run pyright app/ 0 PRP-35 errors (8 pre-existing optional-extra noise on dev) +✅ uv run pytest -m "not integration" 1480 passed, 12 skipped, 264 deselected +``` + +40 V2 leakage tests across 3 LOAD-BEARING files all green; 23 V1 contract / +leakage tests byte-stable. + +The 3 mypy + 8 pyright pre-existing errors stem from optional `lightgbm` / +`xgboost` extras and are unrelated to PRP-35; CI runs `--all-extras` and won't +see them. + +## No Alembic migration + +V2 reads only existing tables (`inventory_snapshot_daily`, +`replenishment_event`, `sales_returns`, `exogenous_signal`, `promotion`, +`product`) and writes nothing to the DB. `alembic heads` unchanged at +`c1d2e3f40512`. + +## Deferred: V2 backtesting dispatch — tracked in #299 + +PRP-35 lands V2 **training + scenarios + shared builders**. **Backtesting V2 +dispatch is deferred** and explicitly tracked in the +"Deferred follow-up: V2 backtesting dispatch" section of **#299**. + +PRP-35 Task 13 reads *"READ `feature_frame_version` from the fitted bundle +BEFORE the fold loop"*, but +`app/features/backtesting/service.py:_run_model_backtest` trains fresh per +fold from `BacktestConfig.model_config_main` and **does not load a fitted +bundle**. The correct opt-in surface is a request-time field on +`BacktestConfig` itself — a re-design Task 13 did not spec. + +**This PR does NOT claim completion of PRP-35 Tasks 13 or 18.** V1 +backtesting is unchanged; a V2-trained bundle still trains and +scenario-simulates correctly. Only `/backtesting/run` remains V1-only until +the follow-up under #299 lands. Integration tests (PRP-35 Tasks 15 + 16) and +the PHASE/3 + PHASE/4 doc edits (Task 21) are also deferred there. + +## qwen3 stash status + +The session's `stash@{0}` ("local qwen3 rag demo changes before prp-35", +`app/features/rag/models.py` +7/-2) is **not applied, not popped, not +dropped**. The decision on it (write a real +`INITIAL-rag-embedding-provider-pluggability.md` doc vs. add to +`.git/info/exclude`) is carryover work, untouched by this PR. + +## Files changed + +``` + M app/features/forecasting/routes.py (+2) + M app/features/forecasting/schemas.py (+70) + M app/features/forecasting/service.py (+318) + M app/features/scenarios/feature_frame.py (+193) + M app/features/scenarios/service.py (+25) + M app/shared/feature_frames/__init__.py (+79) + M docs/optional-features/10-baseforecaster-feature-contract.md (+40) + A app/shared/feature_frames/contract_v2.py + A app/shared/feature_frames/rows_v2.py + A app/shared/feature_frames/sidecar.py + A app/shared/feature_frames/tests/test_contract_v2.py + A app/shared/feature_frames/tests/test_leakage_v2.py + A app/features/forecasting/v2_loaders.py + A app/features/forecasting/tests/test_regression_features_v2_leakage.py + A app/features/scenarios/tests/test_future_frame_v2_leakage.py + A examples/forecasting/feature_frame_v2_preview.py + A PR-BODY-DRAFT.md +``` + +## Test plan + +- [ ] CI green on all five gates (ruff / mypy / pyright / pytest / migration-check). +- [ ] Verify `/forecasting/train` accepts `feature_frame_version=2` with default groups. +- [ ] Verify `/forecasting/train` accepts `feature_frame_version=2` with opt-in + Phase-2 group (e.g. `INVENTORY`) on a seeded DB carrying inventory rows. +- [ ] Verify `/scenarios/simulate` against a V2-trained bundle produces a + `model_exogenous` re-forecast (V2 future-frame assembly via bundle metadata). +- [ ] Verify a V1 bundle trained before this PR still loads, predicts, and + scenario-simulates unchanged. +- [ ] Verify `/backtesting/run` against a V2-trained bundle remains V1-only + (no V2 dispatch on the fold loop) — documented deferral above. diff --git a/app/features/forecasting/routes.py b/app/features/forecasting/routes.py index d122ab6f..2258d1cc 100644 --- a/app/features/forecasting/routes.py +++ b/app/features/forecasting/routes.py @@ -103,6 +103,8 @@ async def train_model( train_start_date=request.train_start_date, train_end_date=request.train_end_date, config=request.config, + feature_frame_version=request.feature_frame_version, + feature_groups=request.feature_groups, ) logger.info( diff --git a/app/features/forecasting/schemas.py b/app/features/forecasting/schemas.py index 23308e39..1223f8b9 100644 --- a/app/features/forecasting/schemas.py +++ b/app/features/forecasting/schemas.py @@ -13,7 +13,9 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from app.shared.feature_frames import FeatureGroup # ============================================================================= # Model Configuration Schemas @@ -312,6 +314,30 @@ class TrainRequest(BaseModel): description="End date of training period (inclusive)", ) config: ModelConfig + # PRP-35: opt-in to the V2 feature contract (richer, leakage-safe). V1 + # remains the default and the back-compat path; V2 callers also set + # ``feature_groups`` to pick the enabled :class:`FeatureGroup` subset. + # NOTE: these fields live on ``TrainRequest``, NOT on ``ModelConfigBase`` — + # adding them to the config would mutate every existing ``config_hash()`` + # value, orphaning every registry row and alias. The resolved version is + # persisted into bundle metadata instead. + feature_frame_version: int = Field( + default=1, + ge=1, + le=2, + description=( + "Feature contract version. 1 = V1 (default, 14 columns, back-compat); " + "2 = V2 (richer manifest, opt-in)." + ), + ) + feature_groups: list[str] | None = Field( + default=None, + description=( + "V2 only: optional list of FeatureGroup names to enable " + "(None → DEFAULT_V2_GROUPS). MUST be None / omitted when " + "feature_frame_version=1 (422 otherwise)." + ), + ) @field_validator("train_end_date") @classmethod @@ -323,6 +349,24 @@ def validate_date_range(cls, v: date_type, info: object) -> date_type: raise ValueError("train_end_date must be after train_start_date") return v + @model_validator(mode="after") + def validate_feature_frame_version_and_groups(self) -> TrainRequest: + """Reject ``feature_groups`` when V1 and unknown group names when V2.""" + if self.feature_frame_version == 1 and self.feature_groups is not None: + raise ValueError( + "feature_groups is only valid when feature_frame_version=2; " + "omit it for V1 training." + ) + if self.feature_frame_version == 2 and self.feature_groups is not None: + valid_names = {g.value for g in FeatureGroup} + unknown = [name for name in self.feature_groups if name not in valid_names] + if unknown: + raise ValueError( + f"Unknown FeatureGroup name(s): {unknown!r}. " + f"Valid names: {sorted(valid_names)}." + ) + return self + class TrainResponse(BaseModel): """Response body for POST /forecasting/train. @@ -503,3 +547,27 @@ class FeatureMetadataResponse(BaseModel): "know what the numbers mean." ), ) + # PRP-35 — purely additive V2 metadata. ``feature_frame_version`` defaults + # to 1 for legacy bundles (``bundle.metadata.get("feature_frame_version", 1)``). + # ``feature_groups`` / ``feature_safety_classes`` are populated for V2 + # bundles only and absent (None) for V1. + feature_frame_version: int = Field( + default=1, + ge=1, + le=2, + description="Feature contract version recorded in the bundle metadata.", + ) + feature_groups: dict[str, list[str]] | None = Field( + default=None, + description=( + "V2 only: ``{group_name: [columns]}`` mapping from " + "``v2_feature_groups_dict``. None for V1 bundles." + ), + ) + feature_safety_classes: dict[str, str] | None = Field( + default=None, + description=( + "V2 only: ``{column: safety.value}`` mapping from " + "``v2_feature_safety_classes``. None for V1 bundles." + ), + ) diff --git a/app/features/forecasting/service.py b/app/features/forecasting/service.py index 5d7c810c..27cc2d1e 100644 --- a/app/features/forecasting/service.py +++ b/app/features/forecasting/service.py @@ -59,10 +59,27 @@ # field on ``RunResponse``). The explainability slice avoids the same trap by # importing only ``registry.models`` (a read-only ORM contract); we keep the # import-graph one-way by deferring our service-level imports. +from app.features.forecasting.v2_loaders import ( + assemble_v2_historical_sidecar, + load_exogenous_history, + load_inventory_history, + load_lifecycle_attrs, + load_promotion_history, + load_replenishment_history, + load_returns_history, +) from app.shared.feature_frames import ( + DEFAULT_V2_GROUPS, HISTORY_TAIL_DAYS, + HISTORY_TAIL_DAYS_V2, + FeatureGroup, build_historical_feature_rows, + build_historical_feature_rows_v2, canonical_feature_columns, + canonical_feature_columns_v2, + v2_feature_groups_dict, + v2_feature_safety_classes, + v2_pinned_constants, ) if TYPE_CHECKING: @@ -97,6 +114,35 @@ def __post_init__(self) -> None: # Minimum observed rows required to train a regression model — enough to # resolve the lag features and still leave training signal (PRP-27 GOTCHA #14). _MIN_REGRESSION_TRAIN_ROWS = 30 + + +def _resolve_feature_frame_version(request_version: int) -> int: + """Clamp + validate the requested feature_frame_version against {1, 2}.""" + if request_version not in (1, 2): + raise ValueError(f"feature_frame_version must be 1 or 2, got {request_version!r}") + return request_version + + +def _resolve_feature_groups( + requested: list[str] | None, +) -> tuple[FeatureGroup, ...]: + """Map a list of group-name strings to the canonical FeatureGroup tuple. + + ``None`` → DEFAULT_V2_GROUPS. Unknown names raise ValueError (and surface + at the route layer as 422; the request schema also pre-validates names + via ``model_validator``). + """ + if requested is None: + return DEFAULT_V2_GROUPS + valid: dict[str, FeatureGroup] = {g.value: g for g in FeatureGroup} + out: list[FeatureGroup] = [] + for name in requested: + if name not in valid: + raise ValueError(f"Unknown FeatureGroup name {name!r}; valid: {sorted(valid)}") + out.append(valid[name]) + return tuple(out) + + # The regression feature-frame contract — the lag offsets (``EXOGENOUS_LAGS``), # the observed-target tail length (``HISTORY_TAIL_DAYS``), and the canonical # column set and order (``canonical_feature_columns()``) — is the single source @@ -206,6 +252,9 @@ async def train_model( train_start_date: date_type, train_end_date: date_type, config: ModelConfig, + *, + feature_frame_version: int = 1, + feature_groups: list[str] | None = None, ) -> TrainResponse: """Train a forecasting model and save to disk. @@ -216,6 +265,11 @@ async def train_model( train_start_date: Start date of training period. train_end_date: End date of training period (inclusive). config: Model configuration. + feature_frame_version: PRP-35 — 1 (default, V1) or 2 (opt-in, V2 + richer manifest). Recorded into bundle metadata so dispatch + downstream (scenarios / backtesting) is self-describing. + feature_groups: V2 only — optional list of FeatureGroup names; + ``None`` resolves to DEFAULT_V2_GROUPS. Returns: TrainResponse with training results. @@ -242,21 +296,49 @@ async def train_model( model = model_factory(config, random_state=self.settings.forecast_random_seed) extra_metadata: dict[str, object] = {} if model.requires_features: - features = await self._build_regression_features( - db=db, - store_id=store_id, - product_id=product_id, - start_date=train_start_date, - end_date=train_end_date, - ) - model.fit(features.y, features.X) - n_observations = features.n_observations - extra_metadata = { - "feature_columns": features.feature_columns, - "history_tail": features.history_tail, - "history_tail_dates": features.history_tail_dates, - "launch_date": features.launch_date_iso, - } + version = _resolve_feature_frame_version(feature_frame_version) + if version == 2: + resolved_groups = _resolve_feature_groups(feature_groups) + features = await self._build_regression_features_v2( + db=db, + store_id=store_id, + product_id=product_id, + start_date=train_start_date, + end_date=train_end_date, + groups=resolved_groups, + ) + model.fit(features.y, features.X) + n_observations = features.n_observations + extra_metadata = { + "feature_columns": features.feature_columns, + "history_tail": features.history_tail, + "history_tail_dates": features.history_tail_dates, + "launch_date": features.launch_date_iso, + "feature_frame_version": 2, + "feature_groups": v2_feature_groups_dict(features.feature_columns), + "feature_safety_classes": v2_feature_safety_classes(features.feature_columns), + "feature_pinned_constants": v2_pinned_constants(), + } + else: + features = await self._build_regression_features( + db=db, + store_id=store_id, + product_id=product_id, + start_date=train_start_date, + end_date=train_end_date, + ) + model.fit(features.y, features.X) + n_observations = features.n_observations + # ``feature_frame_version`` is additive and harmless for V1 + # bundles — load-side back-compat (``.get(..., 1)``) makes the + # absence equivalent to a value of 1. + extra_metadata = { + "feature_columns": features.feature_columns, + "history_tail": features.history_tail, + "history_tail_dates": features.history_tail_dates, + "launch_date": features.launch_date_iso, + "feature_frame_version": 1, + } else: training_data = await self._load_training_data( db=db, @@ -646,6 +728,191 @@ async def _build_regression_features( n_observations=len(dates), ) + async def _build_regression_features_v2( + self, + db: AsyncSession, + store_id: int, + product_id: int, + start_date: date_type, + end_date: date_type, + groups: tuple[FeatureGroup, ...], + ) -> RegressionFeatureMatrix: + """Build the V2 historical feature matrix (PRP-35). + + Sibling of :meth:`_build_regression_features`. Loads the same V1 inputs + (sales, holidays, promotions, launch_date) plus the enabled V2 sidecar + groups' data (inventory / replenishment / returns / exogenous / + promotion-kinds) and delegates to + :func:`build_historical_feature_rows_v2`. + + Time-safe by construction: every SQL filter uses ``<= end_date``. + + Args: + db: Database session. + store_id: Store ID. + product_id: Product ID. + start_date: Start of the training window (inclusive). + end_date: End of the training window (inclusive) — origin ``T``. + groups: The resolved :class:`FeatureGroup` subset to emit. + + Returns: + The V2 feature matrix + bundle metadata the future frame needs. + + Raises: + ValueError: When fewer than ``_MIN_REGRESSION_TRAIN_ROWS`` observed + days are available. + """ + sales_rows = ( + await db.execute( + select(SalesDaily.date, SalesDaily.quantity, SalesDaily.unit_price) + .where( + (SalesDaily.store_id == store_id) + & (SalesDaily.product_id == product_id) + & (SalesDaily.date >= start_date) + & (SalesDaily.date <= end_date) + ) + .order_by(SalesDaily.date) + ) + ).all() + if len(sales_rows) < _MIN_REGRESSION_TRAIN_ROWS: + raise ValueError( + f"A regression model needs at least {_MIN_REGRESSION_TRAIN_ROWS} " + f"observed days; store={store_id} product={product_id} has " + f"{len(sales_rows)} between {start_date} and {end_date}." + ) + + dates = [row.date for row in sales_rows] + quantities = [float(row.quantity) for row in sales_rows] + prices = [float(row.unit_price) for row in sales_rows] + positive_prices = sorted(price for price in prices if price > 0.0) + baseline_price = positive_prices[len(positive_prices) // 2] if positive_prices else 1.0 + + # V1-equivalent inputs (always loaded — both V1 and V2 PRICE_PROMO + # need promo_dates / holiday_dates; LIFECYCLE needs launch_date). + holiday_dates: set[date_type] = set( + ( + await db.execute( + select(Calendar.date).where( + Calendar.date >= start_date, + Calendar.date <= end_date, + Calendar.is_holiday.is_(True), + ) + ) + ) + .scalars() + .all() + ) + promo_rows = ( + await db.execute( + select(Promotion.start_date, Promotion.end_date).where( + Promotion.product_id == product_id, + (Promotion.store_id == store_id) | (Promotion.store_id.is_(None)), + Promotion.start_date <= end_date, + Promotion.end_date >= start_date, + ) + ) + ).all() + promo_dates: set[date_type] = set() + for promo in promo_rows: + day = max(promo.start_date, start_date) + last = min(promo.end_date, end_date) + while day <= last: + promo_dates.add(day) + day += timedelta(days=1) + launch_date, discontinue_date, _stage = await load_lifecycle_attrs(db, product_id) + + # V2 sidecar inputs — only loaded when the enabled groups need them. + # Keeps the SQL footprint minimal on V2 calls that omit the Phase-2 + # groups. + inventory_per_day: dict[date_type, tuple[int, bool]] = {} + replenishment_event_dates: list[date_type] = [] + replenishment_event_qty: list[int] = [] + returns_per_day: dict[date_type, int] = {} + promo_per_day: dict[date_type, tuple[frozenset[str], float]] = {} + weather_per_day: dict[date_type, dict[str, float]] = {} + macro_per_day: dict[date_type, dict[str, float]] = {} + if FeatureGroup.INVENTORY in groups: + inventory_per_day = await load_inventory_history( + db, store_id, product_id, start_date, end_date + ) + if FeatureGroup.REPLENISHMENT in groups: + ( + replenishment_event_dates, + replenishment_event_qty, + ) = await load_replenishment_history(db, store_id, product_id, start_date, end_date) + if FeatureGroup.RETURNS in groups: + returns_per_day = await load_returns_history( + db, store_id, product_id, start_date, end_date + ) + if FeatureGroup.PRICE_PROMO in groups: + promo_per_day = await load_promotion_history( + db, store_id, product_id, start_date, end_date + ) + if FeatureGroup.EXOGENOUS_WEATHER in groups or FeatureGroup.EXOGENOUS_MACRO in groups: + all_exogenous = await load_exogenous_history(db, store_id, start_date, end_date) + # Split into weather (per-store) and macro (chain-wide) buckets by + # signal name prefix; the V2 builder reads the canonical names + # pinned in ``contract_v2.WEATHER_SIGNAL_NAMES_V2`` / + # ``MACRO_SIGNAL_NAMES_V2``. + for day, signals in all_exogenous.items(): + weather_subset = { + name: value for name, value in signals.items() if name.startswith("weather_") + } + macro_subset = { + name: value for name, value in signals.items() if name.startswith("macro_") + } + if weather_subset: + weather_per_day[day] = weather_subset + if macro_subset: + macro_per_day[day] = macro_subset + + sidecar = assemble_v2_historical_sidecar( + dates=dates, + promo_dates=promo_dates, + holiday_dates=holiday_dates, + launch_date=launch_date, + discontinue_date=discontinue_date, + inventory_per_day=inventory_per_day, + replenishment_event_dates=replenishment_event_dates, + replenishment_event_qty=replenishment_event_qty, + returns_per_day=returns_per_day, + promo_per_day=promo_per_day, + weather_per_day=weather_per_day, + macro_per_day=macro_per_day, + ) + + feature_columns = canonical_feature_columns_v2(groups=groups) + feature_rows = build_historical_feature_rows_v2( + dates=dates, + quantities=quantities, + prices=prices, + baseline_price=baseline_price, + sidecar=sidecar, + groups=groups, + ) + + tail = quantities[-HISTORY_TAIL_DAYS_V2:] + tail_dates = [day.isoformat() for day in dates[-HISTORY_TAIL_DAYS_V2:]] + + logger.info( + "forecasting.regression_features_v2_built", + store_id=store_id, + product_id=product_id, + n_observations=len(dates), + n_features=len(feature_columns), + groups=[g.value for g in groups], + ) + + return RegressionFeatureMatrix( + X=np.array(feature_rows, dtype=np.float64), + y=np.array(quantities, dtype=np.float64), + feature_columns=feature_columns, + history_tail=[float(value) for value in tail], + history_tail_dates=tail_dates, + launch_date_iso=launch_date.isoformat() if launch_date is not None else None, + n_observations=len(dates), + ) + # ------------------------------------------------------------------ # # MLZOO-D / PRP-31 — feature-metadata extraction # @@ -900,6 +1167,24 @@ def _build_metadata_response( importance_type = importance_type_for(bundle.model) + # PRP-35: surface V2 metadata when the bundle is V2; absent → V1. + version_raw = bundle.metadata.get("feature_frame_version", 1) + version = int(version_raw) if isinstance(version_raw, int | str) else 1 + feature_groups_raw = bundle.metadata.get("feature_groups") + feature_safety_raw = bundle.metadata.get("feature_safety_classes") + feature_groups: dict[str, list[str]] | None = None + feature_safety_classes: dict[str, str] | None = None + if version == 2 and isinstance(feature_groups_raw, dict): + feature_groups = { + str(k): [str(c) for c in cast(list[object], v)] + for k, v in cast(dict[object, object], feature_groups_raw).items() + if isinstance(v, list) + } + if version == 2 and isinstance(feature_safety_raw, dict): + feature_safety_classes = { + str(k): str(v) for k, v in cast(dict[object, object], feature_safety_raw).items() + } + return FeatureMetadataResponse( run_id=source_id, model_type=model_type, @@ -907,4 +1192,7 @@ def _build_metadata_response( feature_columns=feature_columns, features=features, importance_type=importance_type, + feature_frame_version=version, + feature_groups=feature_groups, + feature_safety_classes=feature_safety_classes, ) diff --git a/app/features/forecasting/tests/test_regression_features_v2_leakage.py b/app/features/forecasting/tests/test_regression_features_v2_leakage.py new file mode 100644 index 00000000..31c664b7 --- /dev/null +++ b/app/features/forecasting/tests/test_regression_features_v2_leakage.py @@ -0,0 +1,124 @@ +"""V2 leakage spec at the forecasting-slice layer — LOAD-BEARING (PRP-35). + +Mirrors ``test_regression_features_leakage.py``: must NEVER be weakened to +make a feature pass (AGENTS.md § Safety). + +The slice-layer counterpart to ``app/shared/feature_frames/tests/test_leakage_v2.py``. +Pins the time-safety invariants of the V2 historical row assembler as used +through the public ``build_historical_feature_rows_v2`` (driven by sequential +targets so leakage is mathematically detectable). +""" + +from __future__ import annotations + +import math +from datetime import date, timedelta + +from app.shared.feature_frames import ( + EXOGENOUS_LAGS_V2, + ROLLING_WINDOWS_V2, + TREND_WINDOWS_V2, + FeatureGroup, + V2HistoricalSidecar, + build_historical_feature_rows_v2, + canonical_feature_columns_v2, +) + +_N = 200 +_DATES = [date(2026, 1, 1) + timedelta(days=offset) for offset in range(_N)] +_QUANTITIES = [float(offset + 1) for offset in range(_N)] +_PRICES = [10.0] * _N +_BASELINE_PRICE = 10.0 + + +def _build_rows( + groups: tuple[FeatureGroup, ...] | None = None, +) -> tuple[list[str], list[list[float]]]: + """Assemble the V2 feature matrix from sequential targets.""" + columns = canonical_feature_columns_v2(groups=groups) + sidecar = V2HistoricalSidecar() + rows = build_historical_feature_rows_v2( + dates=_DATES, + quantities=_QUANTITIES, + prices=_PRICES, + baseline_price=_BASELINE_PRICE, + sidecar=sidecar, + groups=groups, + ) + return columns, rows + + +def test_v2_lag_columns_read_only_strictly_earlier_observations() -> None: + """CRITICAL: every V2 lag cell reads a strictly-earlier observation, or NaN.""" + columns, rows = _build_rows() + for lag in EXOGENOUS_LAGS_V2: + col_index = columns.index(f"lag_{lag}") + for i in range(_N): + cell = rows[i][col_index] + if i < lag: + assert math.isnan(cell), ( + f"row {i}: lag_{lag} has no source day yet — expected NaN, got {cell}" + ) + continue + expected = _QUANTITIES[i - lag] + assert cell == expected, f"LEAKAGE at row {i}: lag_{lag}={cell} != expected={expected}" + assert cell < _QUANTITIES[i], ( + f"LEAKAGE at row {i}: lag_{lag}={cell} >= current={_QUANTITIES[i]}" + ) + + +def test_v2_rolling_mean_strictly_less_than_current_target() -> None: + """Rolling-mean values built from sequential prior rows are always < current.""" + columns, rows = _build_rows() + for window in ROLLING_WINDOWS_V2: + col_index = columns.index(f"rolling_mean_{window}") + for i in range(_N): + cell = rows[i][col_index] + if i < window: + assert math.isnan(cell), f"row {i}: rolling_mean_{window} should be NaN" + continue + expected = sum(_QUANTITIES[i - window : i]) / window + assert cell == expected, ( + f"row {i}: rolling_mean_{window} expected {expected}, got {cell}" + ) + assert cell < _QUANTITIES[i], ( + f"LEAKAGE at row {i}: rolling_mean_{window}={cell} >= current" + ) + + +def test_v2_trend_strictly_positive_with_sequential_targets() -> None: + """For a monotonic-up sequential series the trend slope is ~1.0 everywhere computable.""" + columns, rows = _build_rows() + for window in TREND_WINDOWS_V2: + col_index = columns.index(f"trend_{window}") + for i in range(window, _N): + cell = rows[i][col_index] + # Sequential 1..N with window points: slope == 1.0 (approximately) + assert abs(cell - 1.0) < 1e-6, ( + f"row {i}: trend_{window} expected ≈1.0 (sequential), got {cell}" + ) + + +def test_v2_matrix_shape_matches_canonical_columns() -> None: + columns, rows = _build_rows() + assert len(rows) == _N + assert all(len(row) == len(columns) for row in rows) + + +def test_v2_assemble_is_deterministic() -> None: + """Identical inputs produce an identical V2 matrix — no hidden state.""" + _, a = _build_rows() + _, b = _build_rows() + assert a == b + + +def test_v2_disabled_groups_omit_their_columns_entirely() -> None: + """A disabled group's columns do NOT appear (NOT NaN-fill placeholders).""" + columns_narrow, rows_narrow = _build_rows( + groups=(FeatureGroup.TARGET_HISTORY, FeatureGroup.CALENDAR) + ) + # No rolling / trend columns should be in the manifest. + assert "rolling_mean_7" not in columns_narrow + assert "trend_30" not in columns_narrow + # Width is exactly len(columns_narrow), not the full default. + assert all(len(row) == len(columns_narrow) for row in rows_narrow) diff --git a/app/features/forecasting/v2_loaders.py b/app/features/forecasting/v2_loaders.py new file mode 100644 index 00000000..40de1d6f --- /dev/null +++ b/app/features/forecasting/v2_loaders.py @@ -0,0 +1,349 @@ +"""V2 sidecar loaders for the forecasting slice (PRP-35). + +The V2 builders (``app/shared/feature_frames/rows_v2.py``) consume a pure +``V2HistoricalSidecar`` / ``V2FutureSidecar`` data carrier. This module is the +DB-touching wrapper: every loader is a time-safe SELECT against the +``data_platform`` ORM, and the synchronous assembler helpers convert the +loader outputs into the sidecar dataclasses. + +CROSS-SLICE: lives in the forecasting slice — ``app/shared/feature_frames/**`` +remains leaf-level. The scenarios slice has its own (smaller) inline +data_platform reads when it needs lifecycle / discontinue date for V2 future +frames. + +TIME-SAFETY: every ``where`` clause includes ``<= end_date`` (or the +equivalent ``< day`` event-time filter), so a horizon-day query never reads +beyond the forecast origin ``T``. +""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import date as date_type +from datetime import timedelta + +import structlog +from sqlalchemy import and_, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.features.data_platform.models import ( + ExogenousSignal, + InventorySnapshotDaily, + Product, + Promotion, + ReplenishmentEvent, + SalesReturn, +) +from app.shared.feature_frames import V2FutureSidecar, V2HistoricalSidecar + +logger = structlog.get_logger() + + +# ── Raw async DB loaders ───────────────────────────────────────────────────── + + +async def load_lifecycle_attrs( + db: AsyncSession, product_id: int +) -> tuple[date_type | None, date_type | None, str | None]: + """Return ``(launch_date, discontinue_date, lifecycle_stage)`` for a product. + + Both date fields may be ``None``. ``lifecycle_stage`` may be ``None`` when + the seeder did not classify the product. + """ + row = ( + await db.execute( + select(Product.launch_date, Product.discontinue_date, Product.lifecycle_stage).where( + Product.id == product_id + ) + ) + ).first() + if row is None: + return None, None, None + return row.launch_date, row.discontinue_date, row.lifecycle_stage + + +async def load_inventory_history( + db: AsyncSession, + store_id: int, + product_id: int, + start_date: date_type, + end_date: date_type, +) -> dict[date_type, tuple[int, bool]]: + """``{date: (on_hand_qty, is_stockout)}`` — time-safe filter ``<= end_date``.""" + rows = ( + await db.execute( + select( + InventorySnapshotDaily.date, + InventorySnapshotDaily.on_hand_qty, + InventorySnapshotDaily.is_stockout, + ).where( + and_( + InventorySnapshotDaily.store_id == store_id, + InventorySnapshotDaily.product_id == product_id, + InventorySnapshotDaily.date >= start_date, + InventorySnapshotDaily.date <= end_date, + ) + ) + ) + ).all() + out: dict[date_type, tuple[int, bool]] = { + row.date: (row.on_hand_qty, row.is_stockout) for row in rows + } + logger.info( + "forecasting.v2_loaders.inventory_loaded", + store_id=store_id, + product_id=product_id, + n_rows=len(out), + ) + return out + + +async def load_replenishment_history( + db: AsyncSession, + store_id: int, + product_id: int, + start_date: date_type, + end_date: date_type, +) -> tuple[list[date_type], list[int]]: + """``(event_dates, received_qty)`` sorted ascending — time-safe filter ``<= end_date``.""" + rows = ( + await db.execute( + select(ReplenishmentEvent.date, ReplenishmentEvent.received_qty) + .where( + and_( + ReplenishmentEvent.store_id == store_id, + ReplenishmentEvent.product_id == product_id, + ReplenishmentEvent.date >= start_date, + ReplenishmentEvent.date <= end_date, + ) + ) + .order_by(ReplenishmentEvent.date) + ) + ).all() + dates = [row.date for row in rows] + qty = [int(row.received_qty) for row in rows] + logger.info( + "forecasting.v2_loaders.replenishment_loaded", + store_id=store_id, + product_id=product_id, + n_events=len(dates), + ) + return dates, qty + + +async def load_returns_history( + db: AsyncSession, + store_id: int, + product_id: int, + start_date: date_type, + end_date: date_type, +) -> dict[date_type, int]: + """``{date: total_return_quantity}`` — time-safe filter ``<= end_date``.""" + rows = ( + await db.execute( + select(SalesReturn.date, SalesReturn.return_quantity).where( + and_( + SalesReturn.store_id == store_id, + SalesReturn.product_id == product_id, + SalesReturn.date >= start_date, + SalesReturn.date <= end_date, + ) + ) + ) + ).all() + per_day: dict[date_type, int] = defaultdict(int) + for row in rows: + per_day[row.date] += int(row.return_quantity) + logger.info( + "forecasting.v2_loaders.returns_loaded", + store_id=store_id, + product_id=product_id, + n_days_with_returns=len(per_day), + ) + return dict(per_day) + + +async def load_promotion_history( + db: AsyncSession, + store_id: int, + product_id: int, + start_date: date_type, + end_date: date_type, +) -> dict[date_type, tuple[frozenset[str], float]]: + """``{date: (kinds, max_discount_pct)}`` — expanded per-day from promo spans. + + Each day in the training window is mapped to the set of active promo kinds + that day plus the maximum discount_pct active that day. ``discount_pct`` + may be ``None`` in the DB (e.g. for ``bundle`` kind); treated as 0.0 for + the per-day aggregation. + """ + rows = ( + await db.execute( + select( + Promotion.start_date, + Promotion.end_date, + Promotion.kind, + Promotion.discount_pct, + ).where( + and_( + Promotion.product_id == product_id, + or_(Promotion.store_id == store_id, Promotion.store_id.is_(None)), + Promotion.start_date <= end_date, + Promotion.end_date >= start_date, + ) + ) + ) + ).all() + per_day_kinds: dict[date_type, set[str]] = defaultdict(set) + per_day_discount: dict[date_type, float] = {} + for promo in rows: + first_day = max(promo.start_date, start_date) + last_day = min(promo.end_date, end_date) + discount = float(promo.discount_pct) if promo.discount_pct is not None else 0.0 + day = first_day + while day <= last_day: + per_day_kinds[day].add(str(promo.kind)) + existing = per_day_discount.get(day, 0.0) + if discount > existing: + per_day_discount[day] = discount + day += timedelta(days=1) + out: dict[date_type, tuple[frozenset[str], float]] = {} + for day, kinds in per_day_kinds.items(): + out[day] = (frozenset(kinds), per_day_discount.get(day, 0.0)) + logger.info( + "forecasting.v2_loaders.promotion_loaded", + store_id=store_id, + product_id=product_id, + n_promo_days=len(out), + ) + return out + + +async def load_exogenous_history( + db: AsyncSession, + store_id: int, + start_date: date_type, + end_date: date_type, + signal_names: list[str] | None = None, +) -> dict[date_type, dict[str, float]]: + """``{date: {signal_name: value}}`` — per-store + global rows merged. + + Time-safe filter ``<= end_date``. Global rows (``is_global=True``) are + included alongside the per-store rows. When ``signal_names`` is supplied, + only those signals are returned. + """ + stmt = select(ExogenousSignal.date, ExogenousSignal.signal_name, ExogenousSignal.value).where( + and_( + ExogenousSignal.date >= start_date, + ExogenousSignal.date <= end_date, + or_(ExogenousSignal.store_id == store_id, ExogenousSignal.is_global.is_(True)), + ) + ) + if signal_names is not None: + stmt = stmt.where(ExogenousSignal.signal_name.in_(signal_names)) + rows = (await db.execute(stmt)).all() + out: dict[date_type, dict[str, float]] = defaultdict(dict) + for row in rows: + out[row.date][row.signal_name] = float(row.value) + logger.info( + "forecasting.v2_loaders.exogenous_loaded", + store_id=store_id, + n_days=len(out), + n_signals_filter=len(signal_names) if signal_names is not None else None, + ) + return dict(out) + + +# ── Pure sync assemblers (loader outputs → sidecar dataclasses) ───────────── + + +def assemble_v2_historical_sidecar( + *, + dates: list[date_type], + promo_dates: set[date_type], + holiday_dates: set[date_type], + launch_date: date_type | None, + discontinue_date: date_type | None, + inventory_per_day: dict[date_type, tuple[int, bool]], + replenishment_event_dates: list[date_type], + replenishment_event_qty: list[int], + returns_per_day: dict[date_type, int], + promo_per_day: dict[date_type, tuple[frozenset[str], float]], + weather_per_day: dict[date_type, dict[str, float]], + macro_per_day: dict[date_type, dict[str, float]], +) -> V2HistoricalSidecar: + """Build a :class:`V2HistoricalSidecar` from already-loaded DB inputs. + + Per-day arrays are aligned with ``dates``. Days with no entry in + ``inventory_per_day`` / ``returns_per_day`` / ``promo_per_day`` get the + safe default (None for on_hand_qty, False for is_stockout, 0 for returns, + empty frozenset / 0.0 for promo). + """ + on_hand: list[float | None] = [] + stockout: list[bool] = [] + for day in dates: + if day in inventory_per_day: + qty, flag = inventory_per_day[day] + on_hand.append(float(qty)) + stockout.append(bool(flag)) + else: + on_hand.append(None) + stockout.append(False) + returns_qty = [int(returns_per_day.get(day, 0)) for day in dates] + promo_kinds_per_day = tuple(promo_per_day.get(day, (frozenset(), 0.0))[0] for day in dates) + promo_discount = tuple(float(promo_per_day.get(day, (frozenset(), 0.0))[1]) for day in dates) + return V2HistoricalSidecar( + promo_dates=frozenset(promo_dates), + holiday_dates=frozenset(holiday_dates), + launch_date=launch_date, + discontinue_date=discontinue_date, + on_hand_qty=tuple(on_hand), + is_stockout_per_day=tuple(stockout), + replenishment_event_dates=tuple(replenishment_event_dates), + replenishment_event_qty=tuple(replenishment_event_qty), + returns_qty_per_day=tuple(returns_qty), + promo_kinds_per_day=promo_kinds_per_day, + promo_discount_pct_per_day=promo_discount, + weather_per_day=dict(weather_per_day), + macro_per_day=dict(macro_per_day), + ) + + +def assemble_v2_future_sidecar( + *, + holiday_dates: set[date_type], + launch_date: date_type | None, + discontinue_date: date_type | None, + price_factor_per_day: list[float | None] | None = None, + promo_active_per_day: list[bool] | None = None, + promo_kinds_per_day: list[frozenset[str]] | None = None, + promo_discount_pct_per_day: list[float] | None = None, + inventory_on_hand_per_day: list[float | None] | None = None, + weather_per_day: dict[date_type, dict[str, float]] | None = None, + macro_per_day: dict[date_type, dict[str, float]] | None = None, +) -> V2FutureSidecar: + """Build a :class:`V2FutureSidecar` from already-resolved future inputs.""" + return V2FutureSidecar( + holiday_dates=frozenset(holiday_dates), + launch_date=launch_date, + discontinue_date=discontinue_date, + price_factor_per_day=tuple(price_factor_per_day or ()), + promo_active_per_day=tuple(promo_active_per_day or ()), + promo_kinds_per_day=tuple(promo_kinds_per_day or ()), + promo_discount_pct_per_day=tuple(promo_discount_pct_per_day or ()), + inventory_on_hand_per_day=tuple(inventory_on_hand_per_day or ()), + weather_per_day=dict(weather_per_day or {}), + macro_per_day=dict(macro_per_day or {}), + ) + + +__all__ = [ + "assemble_v2_future_sidecar", + "assemble_v2_historical_sidecar", + "load_exogenous_history", + "load_inventory_history", + "load_lifecycle_attrs", + "load_promotion_history", + "load_replenishment_history", + "load_returns_history", +] diff --git a/app/features/scenarios/feature_frame.py b/app/features/scenarios/feature_frame.py index f8307288..8062e040 100644 --- a/app/features/scenarios/feature_frame.py +++ b/app/features/scenarios/feature_frame.py @@ -52,14 +52,17 @@ from sqlalchemy import select from app.core.logging import get_logger -from app.features.data_platform.models import Calendar +from app.features.data_platform.models import Calendar, Product from app.shared.feature_frames import ( CALENDAR_COLUMNS, EXOGENOUS_COLUMNS, EXOGENOUS_LAGS, HISTORY_TAIL_DAYS, + FeatureGroup, FutureFeatureFrame, + V2FutureSidecar, build_calendar_columns, + build_future_feature_rows_v2, build_long_lag_columns, canonical_feature_columns, ) @@ -240,13 +243,20 @@ async def build_future_frame( history_tail: list[float], assumptions: ScenarioAssumptions, launch_date: date | None = None, + feature_frame_version: int = 1, + history_tail_dates: list[date] | None = None, + feature_groups: dict[str, list[str]] | None = None, ) -> FutureFeatureFrame: """Build the future feature frame for one ``(store, product)`` series. - The only database read is the ``calendar`` holiday lookup for the horizon - window — a ``calendar`` row is a timeless attribute, so reading it is not - leakage. Everything else is derived from ``history_tail`` (observed, - ``<= T``), the dates, or the assumptions. + Dispatches on ``feature_frame_version``: + + * V1 (default) — unchanged byte-for-byte. Reads calendar holidays for + the horizon window and delegates to :func:`assemble_future_frame`. + * V2 (PRP-35) — when the bundle was trained with the richer V2 contract. + Reads holidays + product discontinue date, assembles a + :class:`~app.shared.feature_frames.V2FutureSidecar` from the + assumptions, and delegates to ``build_future_feature_rows_v2``. Args: db: Async database session (used only for the calendar lookup). @@ -259,6 +269,13 @@ async def build_future_frame( history_tail: Observed target values ending at ``T``. assumptions: The scenario assumptions. launch_date: The product's launch date, or ``None``. + feature_frame_version: 1 (default) or 2. V1 bundles MAY omit this and + the legacy path is preserved. + history_tail_dates: V2 only — observed dates aligned with + ``history_tail``. Required for V2 same-DOW lookups and exogenous + sidecar lookups (omit → empty list / NaN cells). + feature_groups: V2 only — bundle's ``feature_groups`` metadata. When + provided, drives which V2 columns the future builder emits. Returns: The assembled future feature frame. @@ -280,6 +297,31 @@ async def build_future_frame( ) holiday_dates: set[date] = set(result.scalars().all()) + if feature_frame_version == 2: + frame = await _build_future_frame_v2( + db, + store_id=store_id, + product_id=product_id, + dates=dates, + feature_columns=feature_columns, + history_tail=history_tail, + history_tail_dates=history_tail_dates or [], + assumptions=assumptions, + holiday_dates=holiday_dates, + launch_date=launch_date, + feature_groups=feature_groups, + ) + logger.info( + "scenarios.future_frame_built", + store_id=store_id, + product_id=product_id, + horizon=horizon, + n_features=len(feature_columns), + n_calendar_holidays=len(holiday_dates), + feature_frame_version=2, + ) + return frame + frame = assemble_future_frame( dates=dates, feature_columns=feature_columns, @@ -297,3 +339,144 @@ async def build_future_frame( n_calendar_holidays=len(holiday_dates), ) return frame + + +async def _build_future_frame_v2( + db: AsyncSession, + *, + store_id: int, + product_id: int, + dates: list[date], + feature_columns: list[str], + history_tail: list[float], + history_tail_dates: list[date], + assumptions: ScenarioAssumptions, + holiday_dates: set[date], + launch_date: date | None, + feature_groups: dict[str, list[str]] | None, +) -> FutureFeatureFrame: + """V2 future-frame assembly. + + Loads discontinue_date (a timeless attribute) inline — same-slice + data_platform.models read, mirroring the ``Calendar`` import already used + in the V1 path. Then builds a :class:`V2FutureSidecar` from the + assumptions and delegates to ``build_future_feature_rows_v2``. + + Note: ``store_id`` is unused by V2 sidecar assembly (the future frame is + driven by the assumptions); kept on the signature for parity with V1. + """ + _ = store_id # parameter parity with V1; not read by the V2 assembly + horizon = len(dates) + # Load discontinue_date inline (same-slice data_platform.models read, like + # the V1 path's Calendar lookup). + discontinue_date: date | None = await db.scalar( + select(Product.discontinue_date).where(Product.id == product_id) + ) + + # Build per-day assumption-driven inputs. + price = assumptions.price + promotion = assumptions.promotion + assumption_holidays: set[date] = ( + set(assumptions.holiday.dates) if assumptions.holiday is not None else set() + ) + horizon_holidays = holiday_dates | assumption_holidays + + price_factor_per_day: list[float | None] = [] + promo_active_per_day: list[bool] = [] + promo_kinds_per_day: list[frozenset[str]] = [] + promo_discount_per_day: list[float] = [] + for point in dates: + # price_factor — 1.0 baseline, (1 + change_pct) inside an assumption window + if price is not None and _in_window(point, price.start_date, price.end_date): + price_factor_per_day.append(1.0 + float(price.change_pct)) + else: + price_factor_per_day.append(1.0) + in_promo = promotion is not None and _in_window( + point, promotion.start_date, promotion.end_date + ) + promo_active_per_day.append(bool(in_promo)) + # Default V2 MVP: scenario PromotionAssumption has no kind / discount + # plumbing yet — assume an empty kind set and 0.0 discount when active. + # A future PRP can widen ScenarioAssumptions.promotion to carry these. + promo_kinds_per_day.append(frozenset()) + promo_discount_per_day.append(0.0) + + sidecar = V2FutureSidecar( + holiday_dates=frozenset(horizon_holidays), + launch_date=launch_date, + discontinue_date=discontinue_date, + price_factor_per_day=tuple(price_factor_per_day), + promo_active_per_day=tuple(promo_active_per_day), + promo_kinds_per_day=tuple(promo_kinds_per_day), + promo_discount_pct_per_day=tuple(promo_discount_per_day), + ) + + # Resolve groups from the bundle's persisted feature_groups dict; default + # to all groups present in feature_columns (best-effort) when the bundle + # didn't record one. + if feature_groups: + group_names = list(feature_groups.keys()) + valid: dict[str, FeatureGroup] = {g.value: g for g in FeatureGroup} + resolved_groups: tuple[FeatureGroup, ...] = tuple( + valid[name] for name in group_names if name in valid + ) + else: + resolved_groups = () + if not resolved_groups: + # Fallback: infer from columns present in feature_columns. The future + # builder will silently NaN-fill any column not produced (defensive, + # mirrors V1 assemble_future_frame). + from app.shared.feature_frames import canonical_feature_columns_v2 + + # Try all groups; the builder will emit ALL columns the manifest + # contains for those groups, which may differ from ``feature_columns``. + # We let the caller's ``feature_columns`` be the authoritative output + # column order — any extras are dropped below. + try: + _ = canonical_feature_columns_v2() + resolved_groups = tuple(g for g in FeatureGroup) + except ValueError: # pragma: no cover — defensive + resolved_groups = () + + # Build the V2 future matrix (full groups), then project to the bundle's + # ``feature_columns`` order — any column the bundle didn't expect is + # dropped; any column the bundle expected but the V2 builder doesn't + # produce is NaN-filled (defensive shape, mirrors V1). + import math + + full_rows = build_future_feature_rows_v2( + test_dates=dates, + history_tail=history_tail, + history_tail_dates=history_tail_dates, + gap=0, + baseline_price=1.0, # price_factor is already the ratio; baseline is unitary + sidecar=sidecar, + groups=resolved_groups, + ) + full_columns = list(_columns_for_resolved_groups(resolved_groups)) + full_index = {name: i for i, name in enumerate(full_columns)} + matrix: list[list[float]] = [] + for j in range(horizon): + row: list[float] = [] + for column in feature_columns: + if column in full_index: + row.append(full_rows[j][full_index[column]]) + else: + row.append(math.nan) + matrix.append(row) + return FutureFeatureFrame( + dates=list(dates), + feature_columns=list(feature_columns), + matrix=matrix, + ) + + +def _columns_for_resolved_groups( + groups: tuple[FeatureGroup, ...], +) -> list[str]: + """Resolve the full V2 column list for the supplied groups (best-effort).""" + from app.shared.feature_frames import canonical_feature_columns_v2 + + if not groups: + return [] + return canonical_feature_columns_v2(groups=groups) diff --git a/app/features/scenarios/service.py b/app/features/scenarios/service.py index fc038d9a..6f78a1f4 100644 --- a/app/features/scenarios/service.py +++ b/app/features/scenarios/service.py @@ -228,6 +228,25 @@ async def _simulate_model_exogenous( launch_raw = bundle.metadata.get("launch_date") launch_date = date.fromisoformat(launch_raw) if isinstance(launch_raw, str) else None + # PRP-35 — read feature_frame_version from bundle metadata; V1 bundles + # (no such key) default to 1, preserving the V1 byte-stable path. + version_raw = bundle.metadata.get("feature_frame_version", 1) + feature_frame_version = int(version_raw) if isinstance(version_raw, int | str) else 1 + history_tail_dates_raw = bundle.metadata.get("history_tail_dates") + history_tail_dates: list[date] = [] + if isinstance(history_tail_dates_raw, list): + for value in cast("list[object]", history_tail_dates_raw): + if isinstance(value, str): + history_tail_dates.append(date.fromisoformat(value)) + feature_groups_raw = bundle.metadata.get("feature_groups") + feature_groups: dict[str, list[str]] | None = None + if isinstance(feature_groups_raw, dict): + feature_groups = { + str(k): [str(c) for c in cast(list[object], v)] + for k, v in cast(dict[object, object], feature_groups_raw).items() + if isinstance(v, list) + } + scenario_frame = await build_future_frame( db, store_id=store_id, @@ -238,6 +257,9 @@ async def _simulate_model_exogenous( history_tail=history_tail, assumptions=request.assumptions, launch_date=launch_date, + feature_frame_version=feature_frame_version, + history_tail_dates=history_tail_dates, + feature_groups=feature_groups, ) # The baseline is the SAME frame with the assumptions stripped. baseline_frame = await build_future_frame( @@ -250,6 +272,9 @@ async def _simulate_model_exogenous( history_tail=history_tail, assumptions=ScenarioAssumptions(), launch_date=launch_date, + feature_frame_version=feature_frame_version, + history_tail_dates=history_tail_dates, + feature_groups=feature_groups, ) scenario_x = np.array(scenario_frame.matrix, dtype=np.float64) diff --git a/app/features/scenarios/tests/test_future_frame_v2_leakage.py b/app/features/scenarios/tests/test_future_frame_v2_leakage.py new file mode 100644 index 00000000..535846f1 --- /dev/null +++ b/app/features/scenarios/tests/test_future_frame_v2_leakage.py @@ -0,0 +1,158 @@ +"""V2 leakage spec for the scenarios future frame — LOAD-BEARING (PRP-35). + +Mirrors ``test_future_frame_leakage.py`` for the V2 builder. Pure (no DB): +exercises ``build_future_feature_rows_v2`` directly with the +:class:`V2FutureSidecar` shape the scenarios slice assembles when re-forecasting +a V2 bundle. + +Must NEVER be weakened to make a feature pass (AGENTS.md § Safety). +""" + +from __future__ import annotations + +import math +from datetime import date, timedelta + +from app.shared.feature_frames import ( + EXOGENOUS_LAGS_V2, + FeatureGroup, + V2FutureSidecar, + build_future_feature_rows_v2, + canonical_feature_columns_v2, + v2_feature_safety, +) +from app.shared.feature_frames.contract import FeatureSafety + +_ORIGIN = date(2026, 6, 30) +_HORIZON = 14 +_HISTORY_TAIL = [1000.0 + float(i) for i in range(400)] +_HISTORY_TAIL_DATES = [_ORIGIN - timedelta(days=399 - i) for i in range(400)] + + +def test_v2_future_assumption_driven_price_factor_reflects_input() -> None: + """When the caller supplies ``price_factor_per_day`` it appears in the cell.""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + posited = [0.85] * _HORIZON # 15% price cut every day + sidecar = V2FutureSidecar( + price_factor_per_day=tuple(posited), + promo_active_per_day=tuple([False] * _HORIZON), + promo_kinds_per_day=tuple([frozenset() for _ in range(_HORIZON)]), + promo_discount_pct_per_day=tuple([0.0] * _HORIZON), + ) + columns = canonical_feature_columns_v2(groups=(FeatureGroup.PRICE_PROMO,)) + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_TAIL_DATES, + gap=0, + baseline_price=1.0, + sidecar=sidecar, + groups=(FeatureGroup.PRICE_PROMO,), + ) + col_index = columns.index("price_factor") + for j in range(_HORIZON): + assert rows[j][col_index] == 0.85, ( + f"day {j + 1}: price_factor expected 0.85, got {rows[j][col_index]}" + ) + + +def test_v2_future_unsupplied_price_promo_yields_nan() -> None: + """When the sidecar omits the assumption arrays, PRICE_PROMO cells are NaN.""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + sidecar = V2FutureSidecar() # nothing posited + columns = canonical_feature_columns_v2(groups=(FeatureGroup.PRICE_PROMO,)) + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_TAIL_DATES, + gap=0, + baseline_price=1.0, + sidecar=sidecar, + groups=(FeatureGroup.PRICE_PROMO,), + ) + for column in columns: + assert v2_feature_safety(column) is FeatureSafety.UNSAFE_UNLESS_SUPPLIED + for j in range(_HORIZON): + for column in columns: + cell = rows[j][columns.index(column)] + assert math.isnan(cell), ( + f"day {j + 1}: PRICE_PROMO column {column!r} expected NaN, got {cell}" + ) + + +def test_v2_future_lag_cells_drawn_only_from_history() -> None: + """Every non-NaN ``lag_*`` cell in the V2 future frame is from history_tail.""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + sidecar = V2FutureSidecar() + columns = canonical_feature_columns_v2(groups=(FeatureGroup.TARGET_HISTORY,)) + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_TAIL_DATES, + gap=0, + baseline_price=1.0, + sidecar=sidecar, + groups=(FeatureGroup.TARGET_HISTORY,), + ) + history_values = set(_HISTORY_TAIL) + future_targets = {9000.0 + float(i) for i in range(_HORIZON)} + for lag in EXOGENOUS_LAGS_V2: + col_index = columns.index(f"lag_{lag}") + for j in range(_HORIZON): + cell = rows[j][col_index] + if math.isnan(cell): + continue + assert cell in history_values, f"lag_{lag} day {j + 1}: leaked non-history value {cell}" + assert cell not in future_targets, f"lag_{lag} day {j + 1}: leaked future target {cell}" + + +def test_v2_future_weather_macro_nan_when_sidecar_empty() -> None: + """EXOGENOUS_WEATHER / MACRO columns are NaN when sidecar dicts are empty.""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + sidecar = V2FutureSidecar() + columns = canonical_feature_columns_v2( + groups=(FeatureGroup.EXOGENOUS_WEATHER, FeatureGroup.EXOGENOUS_MACRO) + ) + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_TAIL_DATES, + gap=0, + baseline_price=1.0, + sidecar=sidecar, + groups=(FeatureGroup.EXOGENOUS_WEATHER, FeatureGroup.EXOGENOUS_MACRO), + ) + for j in range(_HORIZON): + for column in columns: + cell = rows[j][columns.index(column)] + assert math.isnan(cell), ( + f"day {j + 1}: {column!r} expected NaN (empty sidecar), got {cell}" + ) + + +def test_v2_future_lifecycle_safe_when_launch_date_supplied() -> None: + """LIFECYCLE columns are SAFE (pure function of dates + launch/discontinue).""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + launch = _ORIGIN - timedelta(days=100) # 100 days before T + sidecar = V2FutureSidecar(launch_date=launch) + columns = canonical_feature_columns_v2(groups=(FeatureGroup.LIFECYCLE,)) + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_TAIL_DATES, + gap=0, + baseline_price=1.0, + sidecar=sidecar, + groups=(FeatureGroup.LIFECYCLE,), + ) + days_since_idx = columns.index("days_since_launch") + # Test day 1 → days_since_launch = 101 + assert rows[0][days_since_idx] == 101.0 + # Test day 14 → 114 + assert rows[13][days_since_idx] == 114.0 + # is_mature_product = 1.0 (>= 180 days threshold? no — 101 days < 180), so 0.0 + is_mature_idx = columns.index("is_mature_product") + assert rows[0][is_mature_idx] == 0.0 + # is_new_product = 0.0 (>= 30 days) + is_new_idx = columns.index("is_new_product") + assert rows[0][is_new_idx] == 0.0 diff --git a/app/shared/feature_frames/__init__.py b/app/shared/feature_frames/__init__.py index df0568b4..99dfb325 100644 --- a/app/shared/feature_frames/__init__.py +++ b/app/shared/feature_frames/__init__.py @@ -1,11 +1,15 @@ -"""Shared feature-frame contract for feature-aware forecasting (MLZOO-A). +"""Shared feature-frame contract for feature-aware forecasting (MLZOO-A + PRP-35). The single, cross-cutting home for the regression feature-frame contract — the -pinned constants, the canonical column set, the :class:`FutureFeatureFrame` -carrier, the leakage-safe pure builders, and the :class:`FeatureSafety` -taxonomy. Both the ``forecasting`` slice (historical training frame) and the -``scenarios`` slice (future prediction frame) import from here, so the contract -is defined exactly once. +pinned constants, the canonical column sets (V1 and V2), the +:class:`FutureFeatureFrame` carrier, the leakage-safe pure builders, and the +:class:`FeatureSafety` taxonomy. Both the ``forecasting`` slice (historical +training frame) and the ``scenarios`` slice (future prediction frame) import +from here, so the contract is defined exactly once. + +V2 (PRP-35) adds a richer, opt-in surface alongside V1: every V1 export below +remains at the same position and behaviour; V2 callers reach the V2 manifest / +sidecars / row builders through the same package. This package is leaf-level: it imports nothing from ``app/features/**``. """ @@ -23,23 +27,86 @@ canonical_feature_columns, feature_safety, ) +from app.shared.feature_frames.contract_v2 import ( + DEFAULT_V2_GROUPS, + EXOGENOUS_LAGS_V2, + FEATURE_FRAME_VERSION_V1, + FEATURE_FRAME_VERSION_V2, + HISTORY_TAIL_DAYS_V2, + INVENTORY_AVAILABILITY_WINDOW_V2, + LIFECYCLE_MATURE_THRESHOLD_DAYS, + LIFECYCLE_NEW_THRESHOLD_DAYS, + MACRO_SIGNAL_NAMES_V2, + REPLENISHMENT_QTY_WINDOW_V2, + REPLENISHMENT_WINDOW_V2, + RETURNS_RATE_WINDOW_V2, + RETURNS_WINDOWS_V2, + ROLLING_WINDOWS_V2, + SAME_DOW_MEAN_LOOKBACKS_V2, + STOCKOUT_WINDOWS_V2, + TREND_WINDOWS_V2, + WEATHER_SIGNAL_NAMES_V2, + FeatureGroup, + V2ColumnSpec, + canonical_feature_columns_v2, + v2_column_manifest, + v2_feature_groups_dict, + v2_feature_safety, + v2_feature_safety_classes, + v2_pinned_constants, +) from app.shared.feature_frames.rows import ( build_future_feature_rows, build_historical_feature_rows, ) +from app.shared.feature_frames.rows_v2 import ( + build_future_feature_rows_v2, + build_historical_feature_rows_v2, +) +from app.shared.feature_frames.sidecar import V2FutureSidecar, V2HistoricalSidecar __all__ = [ "CALENDAR_COLUMNS", + "DEFAULT_V2_GROUPS", "EXOGENOUS_COLUMNS", "EXOGENOUS_LAGS", + "EXOGENOUS_LAGS_V2", "FEATURE_CLASS", + "FEATURE_FRAME_VERSION_V1", + "FEATURE_FRAME_VERSION_V2", "HISTORY_TAIL_DAYS", + "HISTORY_TAIL_DAYS_V2", + "INVENTORY_AVAILABILITY_WINDOW_V2", + "LIFECYCLE_MATURE_THRESHOLD_DAYS", + "LIFECYCLE_NEW_THRESHOLD_DAYS", + "MACRO_SIGNAL_NAMES_V2", + "REPLENISHMENT_QTY_WINDOW_V2", + "REPLENISHMENT_WINDOW_V2", + "RETURNS_RATE_WINDOW_V2", + "RETURNS_WINDOWS_V2", + "ROLLING_WINDOWS_V2", + "SAME_DOW_MEAN_LOOKBACKS_V2", + "STOCKOUT_WINDOWS_V2", + "TREND_WINDOWS_V2", + "WEATHER_SIGNAL_NAMES_V2", + "FeatureGroup", "FeatureSafety", "FutureFeatureFrame", + "V2ColumnSpec", + "V2FutureSidecar", + "V2HistoricalSidecar", "build_calendar_columns", "build_future_feature_rows", + "build_future_feature_rows_v2", "build_historical_feature_rows", + "build_historical_feature_rows_v2", "build_long_lag_columns", "canonical_feature_columns", + "canonical_feature_columns_v2", "feature_safety", + "v2_column_manifest", + "v2_feature_groups_dict", + "v2_feature_safety", + "v2_feature_safety_classes", + "v2_pinned_constants", ] diff --git a/app/shared/feature_frames/contract_v2.py b/app/shared/feature_frames/contract_v2.py new file mode 100644 index 00000000..dd352213 --- /dev/null +++ b/app/shared/feature_frames/contract_v2.py @@ -0,0 +1,370 @@ +"""Feature-frame contract V2 — richer, opt-in feature manifest (PRP-35). + +V2 extends :mod:`app.shared.feature_frames.contract` with a richer set of +columns (yearly seasonality, rolling demand level, trend, lifecycle, optional +phase-2 sidecar signals) WITHOUT changing V1 byte-for-byte. V1 callers continue +to see V1 columns at the same positions; V2 callers opt in via +``TrainRequest.feature_frame_version=2`` + an optional ``feature_groups`` list. + +LEAF-LEVEL: like ``contract.py`` this module may NEVER import from +``app/features/**``. Every symbol is pure stdlib (``math``, ``dataclasses``, +``enum``, ``datetime``). + +The leakage rule the V2 builders obey mirrors V1 exactly: + + A future feature value for horizon day ``D`` may use ONLY information + knowable at the forecast origin ``T``: the observed history up to and + including ``T``, the calendar (a pure function of the date), launch / + discontinue dates (timeless attributes), or scenario-assumption inputs + posited by the caller. It may NEVER read an observed target — or any + sidecar value — at a horizon day ``D``. + +Every V2 column has a :class:`~app.shared.feature_frames.contract.FeatureSafety` +classification (resolved via :func:`v2_feature_safety_classes`) so a downstream +consumer can tell at a glance which cells may be NaN at a future horizon row. + +The V2 column manifest is a function of the enabled :class:`FeatureGroup` +subset. Group enablement decides which columns appear in the output matrix +(disabled group = silent omission, NOT a NaN-filled placeholder). Per-cell +NaN signals "source data unknown for this day"; HGBR tolerates NaN natively. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from app.shared.feature_frames.contract import ( + CALENDAR_COLUMNS, + FeatureSafety, +) + +# ── Versions ──────────────────────────────────────────────────────────────── +FEATURE_FRAME_VERSION_V1: int = 1 +FEATURE_FRAME_VERSION_V2: int = 2 + +# ── Pinned V2 modelling constants (PRP-35 DECISIONS LOCKED) ───────────────── +# Lag offsets — daily, weekly, fortnightly, four-week, eight-week, yearly. +# ``lag_364`` (not ``lag_365``) preserves day-of-week (52 * 7 = 364). +EXOGENOUS_LAGS_V2: tuple[int, ...] = (1, 7, 14, 28, 56, 364) +# Same-day-of-week mean lookbacks: average of the N most recent same-weekday +# observations strictly before each row. +SAME_DOW_MEAN_LOOKBACKS_V2: tuple[int, ...] = (4, 8) +# Rolling-mean windows (also feed median / std). +ROLLING_WINDOWS_V2: tuple[int, ...] = (7, 28, 90) +# Trend windows — linear slope (numpy.polyfit) over the trailing N days. +TREND_WINDOWS_V2: tuple[int, ...] = (30, 90) +# Stockout / replenishment / returns aggregate windows. +STOCKOUT_WINDOWS_V2: tuple[int, ...] = (7, 28) +REPLENISHMENT_WINDOW_V2: int = 14 +REPLENISHMENT_QTY_WINDOW_V2: int = 28 +RETURNS_WINDOWS_V2: tuple[int, ...] = (7, 28) +RETURNS_RATE_WINDOW_V2: int = 28 +INVENTORY_AVAILABILITY_WINDOW_V2: int = 28 +# Lifecycle thresholds (days from launch). +LIFECYCLE_NEW_THRESHOLD_DAYS: int = 30 +LIFECYCLE_MATURE_THRESHOLD_DAYS: int = 180 +# Observed-target tail length fed to the future builder. Must comfortably +# exceed ``max(EXOGENOUS_LAGS_V2)`` and the largest rolling/trend window. +HISTORY_TAIL_DAYS_V2: int = 400 # 364 + 28 buffer + 8 safety margin + +# Canonical signal names emitted by the EXOGENOUS_* groups in V2. The MVP +# pins a small, stable set; future PRPs can extend the manifest. +WEATHER_SIGNAL_NAMES_V2: tuple[str, ...] = ("weather_temp_c", "weather_precip_mm") +MACRO_SIGNAL_NAMES_V2: tuple[str, ...] = ("macro_index",) + + +# ── Feature groups ────────────────────────────────────────────────────────── + + +class FeatureGroup(str, Enum): + """Coarse grouping of V2 feature columns — drives opt-in enablement. + + Enabling a group emits its columns into the manifest in the order the + group is listed below. Disabling a group omits its columns entirely (NOT a + NaN-fill placeholder). Per-day NaN inside an enabled group signals + "source data unknown for this day"; the model (HGBR) handles NaN natively. + """ + + TARGET_HISTORY = "target_history" + CALENDAR = "calendar" + ROLLING = "rolling" + TREND = "trend" + PRICE_PROMO = "price_promo" + INVENTORY = "inventory" + LIFECYCLE = "lifecycle" + REPLENISHMENT = "replenishment" + RETURNS = "returns" + EXOGENOUS_WEATHER = "exogenous_weather" + EXOGENOUS_MACRO = "exogenous_macro" + + +# Canonical group order — the V2 manifest emits columns in exactly this order. +_GROUP_ORDER: tuple[FeatureGroup, ...] = ( + FeatureGroup.TARGET_HISTORY, + FeatureGroup.CALENDAR, + FeatureGroup.ROLLING, + FeatureGroup.TREND, + FeatureGroup.PRICE_PROMO, + FeatureGroup.INVENTORY, + FeatureGroup.LIFECYCLE, + FeatureGroup.REPLENISHMENT, + FeatureGroup.RETURNS, + FeatureGroup.EXOGENOUS_WEATHER, + FeatureGroup.EXOGENOUS_MACRO, +) + +# Default groups when ``feature_groups`` is None on the request. Phase-2 +# sidecar groups (INVENTORY / REPLENISHMENT / RETURNS / EXOGENOUS_*) are off +# by default so the MVP stays green on smaller seeded DBs. +DEFAULT_V2_GROUPS: tuple[FeatureGroup, ...] = ( + FeatureGroup.TARGET_HISTORY, + FeatureGroup.CALENDAR, + FeatureGroup.ROLLING, + FeatureGroup.TREND, + FeatureGroup.PRICE_PROMO, + FeatureGroup.LIFECYCLE, +) + + +# ── Column manifests per group ────────────────────────────────────────────── +# Each tuple is the in-group column order. Tests pin both the per-group +# membership and the overall canonical order built from these blocks. + +_TARGET_HISTORY_COLUMNS: tuple[str, ...] = ( + *(f"lag_{k}" for k in EXOGENOUS_LAGS_V2), + *(f"same_dow_mean_{n}" for n in SAME_DOW_MEAN_LOOKBACKS_V2), +) + +# V1 calendar columns first (V1 ordering preserved within the V1 subset), then +# V2 extensions. ``is_holiday`` (V1 EXOGENOUS_COLUMNS) is calendar-derived and +# placed last in the V2 CALENDAR group — see PRP-35 § Open Design Decisions. +_CALENDAR_COLUMNS_V2: tuple[str, ...] = ( + *CALENDAR_COLUMNS, # dow_sin, dow_cos, month_sin, month_cos, is_weekend, is_month_end + "week_of_year_sin", + "week_of_year_cos", + "day_of_month_sin", + "day_of_month_cos", + "is_holiday", +) + +_ROLLING_COLUMNS: tuple[str, ...] = ( + "rolling_mean_7", + "rolling_mean_28", + "rolling_mean_90", + "rolling_median_28", + "rolling_std_28", +) + +_TREND_COLUMNS: tuple[str, ...] = ( + "trend_30", + "trend_90", + "rolling_mean_7_vs_28", + "rolling_mean_28_vs_prev_28", +) + +# V1 price_factor + promo_active first, then V2 extensions. +_PRICE_PROMO_COLUMNS: tuple[str, ...] = ( + "price_factor", + "promo_active", + "promo_discount_pct", + "promo_kind_markdown_active", + "promo_kind_bundle_active", +) + +_INVENTORY_COLUMNS: tuple[str, ...] = ( + "is_stockout_lag1", + "stockout_days_7", + "stockout_days_28", + "inventory_available_ratio_28", +) + +# V1 days_since_launch first, then V2 extensions. +_LIFECYCLE_COLUMNS: tuple[str, ...] = ( + "days_since_launch", + "is_new_product", + "is_mature_product", + "is_discontinued", + "days_until_discontinue", +) + +_REPLENISHMENT_COLUMNS: tuple[str, ...] = ( + "days_since_last_replenishment", + "replenishment_count_14", + "replenishment_qty_28", +) + +_RETURNS_COLUMNS: tuple[str, ...] = ( + "returns_qty_7", + "returns_qty_28", + "returns_rate_28", +) + +_EXOGENOUS_WEATHER_COLUMNS: tuple[str, ...] = tuple( + f"exo_{name}" for name in WEATHER_SIGNAL_NAMES_V2 +) +_EXOGENOUS_MACRO_COLUMNS: tuple[str, ...] = tuple(f"exo_{name}" for name in MACRO_SIGNAL_NAMES_V2) + + +_GROUP_COLUMNS: dict[FeatureGroup, tuple[str, ...]] = { + FeatureGroup.TARGET_HISTORY: _TARGET_HISTORY_COLUMNS, + FeatureGroup.CALENDAR: _CALENDAR_COLUMNS_V2, + FeatureGroup.ROLLING: _ROLLING_COLUMNS, + FeatureGroup.TREND: _TREND_COLUMNS, + FeatureGroup.PRICE_PROMO: _PRICE_PROMO_COLUMNS, + FeatureGroup.INVENTORY: _INVENTORY_COLUMNS, + FeatureGroup.LIFECYCLE: _LIFECYCLE_COLUMNS, + FeatureGroup.REPLENISHMENT: _REPLENISHMENT_COLUMNS, + FeatureGroup.RETURNS: _RETURNS_COLUMNS, + FeatureGroup.EXOGENOUS_WEATHER: _EXOGENOUS_WEATHER_COLUMNS, + FeatureGroup.EXOGENOUS_MACRO: _EXOGENOUS_MACRO_COLUMNS, +} + + +# Per-column safety class. Group enablement decides emission; this map decides +# leakage class. Every column V2 ever emits is classified here. +_COLUMN_SAFETY: dict[str, FeatureSafety] = { + # TARGET_HISTORY — all conditionally safe (target-derived) + **dict.fromkeys(_TARGET_HISTORY_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + # CALENDAR — pure functions of the date; SAFE + **dict.fromkeys(_CALENDAR_COLUMNS_V2, FeatureSafety.SAFE), + # ROLLING — target-derived rolling statistics + **dict.fromkeys(_ROLLING_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + # TREND — target-derived + **dict.fromkeys(_TREND_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + # PRICE_PROMO — UNSAFE unless caller supplies the future inputs + **dict.fromkeys(_PRICE_PROMO_COLUMNS, FeatureSafety.UNSAFE_UNLESS_SUPPLIED), + # INVENTORY — observed inventory series; future unknowable unless supplied + **dict.fromkeys(_INVENTORY_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + # LIFECYCLE — pure function of date + launch/discontinue dates (timeless) + **dict.fromkeys(_LIFECYCLE_COLUMNS, FeatureSafety.SAFE), + # REPLENISHMENT — observed event series; future unknowable + **dict.fromkeys(_REPLENISHMENT_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + # RETURNS — observed returns series; future unknowable + **dict.fromkeys(_RETURNS_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + # EXOGENOUS_* — observed signals; future unknowable unless supplied + **dict.fromkeys(_EXOGENOUS_WEATHER_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), + **dict.fromkeys(_EXOGENOUS_MACRO_COLUMNS, FeatureSafety.CONDITIONALLY_SAFE), +} + + +# ── Public surface ────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class V2ColumnSpec: + """One V2 feature column — name, group, safety class.""" + + name: str + group: FeatureGroup + safety: FeatureSafety + + +def resolve_v2_groups(groups: tuple[FeatureGroup, ...] | None) -> tuple[FeatureGroup, ...]: + """Return groups in canonical order. ``None`` → ``DEFAULT_V2_GROUPS``.""" + requested = DEFAULT_V2_GROUPS if groups is None else groups + if not requested: + raise ValueError( + "v2 feature manifest: at least one FeatureGroup must be enabled " + "(empty groups would produce a zero-column matrix)." + ) + requested_set = set(requested) + unknown = requested_set - set(_GROUP_ORDER) + if unknown: + raise ValueError(f"v2 feature manifest: unknown FeatureGroup(s): {sorted(unknown)!r}") + # Emit in canonical group order regardless of input order. + return tuple(g for g in _GROUP_ORDER if g in requested_set) + + +def v2_column_manifest( + groups: tuple[FeatureGroup, ...] | None = None, +) -> list[V2ColumnSpec]: + """The ordered, canonical V2 column manifest for the enabled groups. + + Args: + groups: The enabled :class:`FeatureGroup` subset. ``None`` resolves to + :data:`DEFAULT_V2_GROUPS`. Group ordering in the output follows the + canonical group order; the caller's input order is ignored. + + Returns: + Ordered list of :class:`V2ColumnSpec` — one per emitted column. + + Raises: + ValueError: When ``groups`` is empty or names an unknown group. + """ + resolved = resolve_v2_groups(groups) + manifest: list[V2ColumnSpec] = [] + for group in resolved: + for column in _GROUP_COLUMNS[group]: + manifest.append(V2ColumnSpec(name=column, group=group, safety=_COLUMN_SAFETY[column])) + return manifest + + +def canonical_feature_columns_v2( + groups: tuple[FeatureGroup, ...] | None = None, +) -> list[str]: + """Equivalent of ``canonical_feature_columns`` for V2.""" + return [spec.name for spec in v2_column_manifest(groups)] + + +def v2_feature_groups_dict(columns: list[str]) -> dict[str, list[str]]: + """Return a ``{group_name: [columns]}`` mapping for the supplied columns. + + Persisted into bundle metadata so the dashboard (Slice C) can render the + grouped column list. Columns not classifiable to a V2 group are silently + skipped (defensive — every column V2 emits is classified by construction). + """ + # Reverse map: column name → group + col_to_group: dict[str, FeatureGroup] = {} + for group_key, group_cols in _GROUP_COLUMNS.items(): + for column in group_cols: + col_to_group[column] = group_key + + grouped: dict[str, list[str]] = {} + for column in columns: + owning_group = col_to_group.get(column) + if owning_group is None: + continue + grouped.setdefault(owning_group.value, []).append(column) + return grouped + + +def v2_feature_safety_classes(columns: list[str]) -> dict[str, str]: + """Return ``{column: safety_class.value}`` for every supplied column. + + Persisted into bundle metadata. Unknown columns (defensive case) classify + as :data:`FeatureSafety.CONDITIONALLY_SAFE` to mirror V1's lag_* fallback. + """ + out: dict[str, str] = {} + for column in columns: + safety = _COLUMN_SAFETY.get(column) + if safety is None: + # Mirror V1 contract: any unclassified column conservatively + # routes to CONDITIONALLY_SAFE so downstream consumers don't fail. + safety = FeatureSafety.CONDITIONALLY_SAFE + out[column] = safety.value + return out + + +def v2_feature_safety(column: str) -> FeatureSafety: + """Return the V2 leakage classification of a single column.""" + if column in _COLUMN_SAFETY: + return _COLUMN_SAFETY[column] + raise KeyError(f"Unclassified V2 feature column: {column!r}") + + +def v2_pinned_constants() -> dict[str, list[int]]: + """Snapshot of the pinned V2 modelling constants — persisted to bundle metadata.""" + return { + "exogenous_lags": list(EXOGENOUS_LAGS_V2), + "same_dow_mean_lookbacks": list(SAME_DOW_MEAN_LOOKBACKS_V2), + "rolling_windows": list(ROLLING_WINDOWS_V2), + "trend_windows": list(TREND_WINDOWS_V2), + "stockout_windows": list(STOCKOUT_WINDOWS_V2), + "replenishment_window": [REPLENISHMENT_WINDOW_V2], + "replenishment_qty_window": [REPLENISHMENT_QTY_WINDOW_V2], + "returns_windows": list(RETURNS_WINDOWS_V2), + "returns_rate_window": [RETURNS_RATE_WINDOW_V2], + "inventory_availability_window": [INVENTORY_AVAILABILITY_WINDOW_V2], + "history_tail_days": [HISTORY_TAIL_DAYS_V2], + } diff --git a/app/shared/feature_frames/rows_v2.py b/app/shared/feature_frames/rows_v2.py new file mode 100644 index 00000000..6449d043 --- /dev/null +++ b/app/shared/feature_frames/rows_v2.py @@ -0,0 +1,1034 @@ +"""V2 historical + future row assemblers (PRP-35). + +Sibling of ``rows.py`` (V1). The two row assemblers below build the V2 feature +matrix in canonical column order (see :func:`canonical_feature_columns_v2`), +emitting only the columns whose owning :class:`FeatureGroup` is enabled. + +LEAF-LEVEL: like ``rows.py`` and ``contract_v2.py`` this module imports nothing +from ``app/features/**``. Every helper is pure (stdlib + numpy for ``polyfit`` +on the trend columns). ``tests/test_contract.py`` and ``test_contract_v2.py`` +extend the AST-walk invariant over this module too. + +Leakage rule the V2 builders obey (mirrors V1): + + A future feature value for horizon day ``D`` may use ONLY information + knowable at the forecast origin ``T``: the observed history up to and + including ``T``, the calendar (a pure function of the date), launch / + discontinue dates, or scenario-assumption inputs posited by the caller. + It NEVER reads an observed target — or any sidecar value — at a + horizon day ``D``. + +Group-gated emission: the column manifest is derived from the ``groups`` +parameter. A disabled group's columns do NOT appear (silent omission, NOT a +NaN-fill placeholder). When a group IS enabled but a specific day lacks +source data, that cell is NaN. + +LOUD failure (ValueError) — programmer / contract errors only: + +* ``groups`` is empty (zero-column matrix is a misuse). +* ``groups`` contains an unknown :class:`FeatureGroup` name. +* A sidecar per-day array's length disagrees with ``dates`` / ``test_dates`` + for an enabled group whose columns read that array. + +NEVER raise ValueError because a single day's source is missing within an +enabled group — that's the NaN case. +""" + +from __future__ import annotations + +import math +from datetime import date + +import numpy as np + +from app.shared.feature_frames.contract import ( + CALENDAR_COLUMNS, + build_calendar_columns, + build_long_lag_columns, +) +from app.shared.feature_frames.contract_v2 import ( + EXOGENOUS_LAGS_V2, + INVENTORY_AVAILABILITY_WINDOW_V2, + LIFECYCLE_MATURE_THRESHOLD_DAYS, + LIFECYCLE_NEW_THRESHOLD_DAYS, + MACRO_SIGNAL_NAMES_V2, + REPLENISHMENT_QTY_WINDOW_V2, + REPLENISHMENT_WINDOW_V2, + RETURNS_RATE_WINDOW_V2, + RETURNS_WINDOWS_V2, + ROLLING_WINDOWS_V2, + SAME_DOW_MEAN_LOOKBACKS_V2, + STOCKOUT_WINDOWS_V2, + TREND_WINDOWS_V2, + WEATHER_SIGNAL_NAMES_V2, + FeatureGroup, + canonical_feature_columns_v2, + resolve_v2_groups, +) +from app.shared.feature_frames.sidecar import V2FutureSidecar, V2HistoricalSidecar + +# ── Pure column helpers (historical) ──────────────────────────────────────── + + +def _rolling_mean_column(quantities: list[float], window: int) -> list[float]: + """Leakage-safe rolling mean: row ``i`` reads ``quantities[i-window..i-1]`` only. + + First ``window`` rows are NaN. NEVER includes ``quantities[i]``. + """ + out: list[float] = [] + for i in range(len(quantities)): + if i < window: + out.append(math.nan) + else: + out.append(sum(quantities[i - window : i]) / window) + return out + + +def _rolling_median_column(quantities: list[float], window: int) -> list[float]: + out: list[float] = [] + for i in range(len(quantities)): + if i < window: + out.append(math.nan) + continue + window_slice = sorted(quantities[i - window : i]) + mid = window // 2 + if window % 2 == 1: + out.append(window_slice[mid]) + else: + out.append((window_slice[mid - 1] + window_slice[mid]) / 2.0) + return out + + +def _rolling_std_column(quantities: list[float], window: int) -> list[float]: + """Sample standard deviation over the trailing ``window`` strictly-earlier rows.""" + out: list[float] = [] + for i in range(len(quantities)): + if i < window: + out.append(math.nan) + continue + slice_ = quantities[i - window : i] + mean = sum(slice_) / window + variance = sum((v - mean) ** 2 for v in slice_) / (window - 1) if window > 1 else 0.0 + out.append(math.sqrt(variance)) + return out + + +def _same_dow_mean_column(dates: list[date], quantities: list[float], n_back: int) -> list[float]: + """Mean of the ``n_back`` most recent EARLIER observations with the same weekday. + + NaN when fewer than ``n_back`` same-weekday earlier observations exist. + """ + out: list[float] = [] + for i, day in enumerate(dates): + same_dow = [quantities[j] for j in range(i) if dates[j].weekday() == day.weekday()] + if len(same_dow) >= n_back: + out.append(sum(same_dow[-n_back:]) / n_back) + else: + out.append(math.nan) + return out + + +def _trend_column(quantities: list[float], window: int) -> list[float]: + """Linear slope (numpy.polyfit, deg=1) over the trailing ``window`` rows. + + NaN when fewer than ``window`` earlier rows exist. + """ + out: list[float] = [] + for i in range(len(quantities)): + if i < window: + out.append(math.nan) + continue + y = np.asarray(quantities[i - window : i], dtype=np.float64) + x = np.arange(window, dtype=np.float64) + # polyfit returns [slope, intercept] for deg=1 + slope = float(np.polyfit(x, y, 1)[0]) + out.append(slope) + return out + + +def _ratio_two_means_column( + quantities: list[float], num_window: int, den_window: int +) -> list[float]: + """Ratio of two trailing-window means (both strictly earlier than row ``i``). + + NaN when either window has insufficient history. ``den == 0`` → NaN. + """ + out: list[float] = [] + for i in range(len(quantities)): + if i < num_window or i < den_window: + out.append(math.nan) + continue + num = sum(quantities[i - num_window : i]) / num_window + den = sum(quantities[i - den_window : i]) / den_window + out.append(num / den if den != 0.0 else math.nan) + return out + + +def _ratio_window_vs_prev_window_column(quantities: list[float], window: int) -> list[float]: + """Ratio of trailing window mean to the window before it. + + For row ``i``: num = mean(quantities[i-window..i-1]), + den = mean(quantities[i-2*window..i-window-1]). NaN until 2*window + earlier rows exist. ``den == 0`` → NaN. + """ + out: list[float] = [] + for i in range(len(quantities)): + if i < 2 * window: + out.append(math.nan) + continue + num = sum(quantities[i - window : i]) / window + den = sum(quantities[i - 2 * window : i - window]) / window + out.append(num / den if den != 0.0 else math.nan) + return out + + +def _v2_calendar_columns(dates: list[date]) -> dict[str, list[float]]: + """V1 calendar columns + V2 extensions (week_of_year, day_of_month). + + Pure function of each date — zero leakage risk. + """ + base = build_calendar_columns(dates) + week_sin: list[float] = [] + week_cos: list[float] = [] + dom_sin: list[float] = [] + dom_cos: list[float] = [] + for day in dates: + # ISO week number — 1..53 + iso_week = day.isocalendar().week + week_sin.append(math.sin(2.0 * math.pi * iso_week / 53.0)) + week_cos.append(math.cos(2.0 * math.pi * iso_week / 53.0)) + # Day of month — 1..31 (use 31 for cyclical encoding) + dom_sin.append(math.sin(2.0 * math.pi * day.day / 31.0)) + dom_cos.append(math.cos(2.0 * math.pi * day.day / 31.0)) + return { + **base, + "week_of_year_sin": week_sin, + "week_of_year_cos": week_cos, + "day_of_month_sin": dom_sin, + "day_of_month_cos": dom_cos, + } + + +def _lifecycle_columns( + dates: list[date], + launch_date: date | None, + discontinue_date: date | None, +) -> dict[str, list[float]]: + """V1 days_since_launch + V2 lifecycle flags. Pure function of dates + attrs.""" + days_since: list[float] = [] + is_new: list[float] = [] + is_mature: list[float] = [] + is_disc: list[float] = [] + days_until_disc: list[float] = [] + for day in dates: + if launch_date is None: + days_since.append(math.nan) + is_new.append(math.nan) + is_mature.append(math.nan) + else: + since = (day - launch_date).days + days_since.append(float(since)) + is_new.append(1.0 if 0 <= since < LIFECYCLE_NEW_THRESHOLD_DAYS else 0.0) + is_mature.append(1.0 if since >= LIFECYCLE_MATURE_THRESHOLD_DAYS else 0.0) + if discontinue_date is None: + is_disc.append(0.0) + days_until_disc.append(math.nan) + else: + is_disc.append(1.0 if day >= discontinue_date else 0.0) + days_until_disc.append(float((discontinue_date - day).days)) + return { + "days_since_launch": days_since, + "is_new_product": is_new, + "is_mature_product": is_mature, + "is_discontinued": is_disc, + "days_until_discontinue": days_until_disc, + } + + +def _stockout_columns( + is_stockout_per_day: tuple[bool, ...], + on_hand_qty: tuple[float | None, ...], + n_rows: int, +) -> dict[str, list[float]]: + """Inventory-derived columns; every cell reads only strictly-earlier days. + + Caller must pass per-day arrays of length ``n_rows`` (validated by caller). + """ + stockout_flags = [1.0 if flag else 0.0 for flag in is_stockout_per_day] + is_stockout_lag1: list[float] = [] + for i in range(n_rows): + is_stockout_lag1.append(stockout_flags[i - 1] if i >= 1 else math.nan) + stockout_per_window: dict[int, list[float]] = {} + for window in STOCKOUT_WINDOWS_V2: + col: list[float] = [] + for i in range(n_rows): + if i < window: + col.append(math.nan) + else: + col.append(float(sum(stockout_flags[i - window : i]))) + stockout_per_window[window] = col + # inventory_available_ratio_28: trailing-28-day mean(on_hand_qty / max_on_hand_in_window) + avail_ratio: list[float] = [] + window = INVENTORY_AVAILABILITY_WINDOW_V2 + for i in range(n_rows): + if i < window: + avail_ratio.append(math.nan) + continue + slice_ = on_hand_qty[i - window : i] + observed = [v for v in slice_ if v is not None] + if not observed: + avail_ratio.append(math.nan) + continue + max_on_hand = max(observed) + if max_on_hand <= 0.0: + avail_ratio.append(math.nan) + continue + mean_observed = sum(observed) / len(observed) + avail_ratio.append(mean_observed / max_on_hand) + return { + "is_stockout_lag1": is_stockout_lag1, + "stockout_days_7": stockout_per_window[7], + "stockout_days_28": stockout_per_window[28], + "inventory_available_ratio_28": avail_ratio, + } + + +def _replenishment_columns( + dates: list[date], + event_dates: tuple[date, ...], + event_qty: tuple[int, ...], +) -> dict[str, list[float]]: + """Replenishment cadence columns; every cell reads only events strictly before the row.""" + n_rows = len(dates) + days_since: list[float] = [] + count_14: list[float] = [] + qty_28: list[float] = [] + for i, day in enumerate(dates): + # Strictly-earlier events: event date < day + prior = [(d, q) for d, q in zip(event_dates, event_qty, strict=True) if d < day] + if prior: + last_event_date = max(d for d, _ in prior) + days_since.append(float((day - last_event_date).days)) + else: + days_since.append(math.nan) + # Counts / qty inside the [day - W, day) windows + win14_start = day.toordinal() - REPLENISHMENT_WINDOW_V2 + win28_start = day.toordinal() - REPLENISHMENT_QTY_WINDOW_V2 + count_14.append(float(sum(1 for d, _ in prior if d.toordinal() >= win14_start))) + qty_28.append(float(sum(q for d, q in prior if d.toordinal() >= win28_start))) + # Suppress unused-loop-variable warning + _ = i + if n_rows == 0: # defensive; never hit in practice + return { + "days_since_last_replenishment": [], + "replenishment_count_14": [], + "replenishment_qty_28": [], + } + return { + "days_since_last_replenishment": days_since, + "replenishment_count_14": count_14, + "replenishment_qty_28": qty_28, + } + + +def _returns_columns( + quantities: list[float], + returns_qty_per_day: tuple[int, ...], + n_rows: int, +) -> dict[str, list[float]]: + """Returns-window columns; every cell reads only strictly-earlier days.""" + returns_floats = [float(v) for v in returns_qty_per_day] + out: dict[str, list[float]] = {} + for window in RETURNS_WINDOWS_V2: + col: list[float] = [] + for i in range(n_rows): + if i < window: + col.append(math.nan) + else: + col.append(float(sum(returns_floats[i - window : i]))) + out[f"returns_qty_{window}"] = col + # returns_rate_28: sum(returns) / max(1, sum(sales)) over the trailing window + rate: list[float] = [] + window = RETURNS_RATE_WINDOW_V2 + for i in range(n_rows): + if i < window: + rate.append(math.nan) + continue + ret_sum = sum(returns_floats[i - window : i]) + sales_sum = sum(quantities[i - window : i]) + rate.append(ret_sum / sales_sum if sales_sum > 0.0 else 0.0) + out["returns_rate_28"] = rate + return out + + +def _price_promo_columns_historical( + *, + dates: list[date], + prices: list[float], + baseline_price: float, + promo_dates: frozenset[date], + promo_kinds_per_day: tuple[frozenset[str], ...], + promo_discount_pct_per_day: tuple[float, ...], + n_rows: int, +) -> dict[str, list[float]]: + """V2 PRICE_PROMO columns for the historical builder. + + ``promo_kinds_per_day`` / ``promo_discount_pct_per_day`` MAY be empty + tuples (then ``promo_discount_pct`` and the kind flags are all 0.0); when + non-empty they MUST have length ``n_rows`` (caller validates). + """ + price_factor = [prices[i] / baseline_price for i in range(n_rows)] + promo_active = [1.0 if day in promo_dates else 0.0 for day in dates] + if promo_discount_pct_per_day: + promo_discount = [float(v) for v in promo_discount_pct_per_day] + else: + promo_discount = [0.0] * n_rows + if promo_kinds_per_day: + markdown = [1.0 if "markdown" in promo_kinds_per_day[i] else 0.0 for i in range(n_rows)] + bundle = [1.0 if "bundle" in promo_kinds_per_day[i] else 0.0 for i in range(n_rows)] + else: + markdown = [0.0] * n_rows + bundle = [0.0] * n_rows + return { + "price_factor": price_factor, + "promo_active": promo_active, + "promo_discount_pct": promo_discount, + "promo_kind_markdown_active": markdown, + "promo_kind_bundle_active": bundle, + } + + +def _exogenous_columns( + dates: list[date], + signal_names: tuple[str, ...], + per_day: dict[date, dict[str, float]], +) -> dict[str, list[float]]: + """Per-day exogenous-signal columns; NaN where the date has no entry.""" + out: dict[str, list[float]] = {} + for name in signal_names: + col: list[float] = [] + for day in dates: + entry = per_day.get(day) + if entry is None or name not in entry: + col.append(math.nan) + else: + col.append(float(entry[name])) + out[f"exo_{name}"] = col + return out + + +# ── Public builders ───────────────────────────────────────────────────────── + + +def _validate_per_day_length( + *, + name: str, + actual: int, + expected: int, + group: FeatureGroup, +) -> None: + """Raise ValueError when a sidecar per-day array's length disagrees with ``dates``.""" + if actual != expected: + raise ValueError( + f"v2 builder: sidecar field {name!r} has length {actual}, but the " + f"{group.value} group requires length {expected} (must align with `dates`)." + ) + + +def build_historical_feature_rows_v2( + *, + dates: list[date], + quantities: list[float], + prices: list[float], + baseline_price: float, + sidecar: V2HistoricalSidecar, + groups: tuple[FeatureGroup, ...] | None = None, +) -> list[list[float]]: + """Assemble the V2 historical regression feature matrix — pure, leakage-safe. + + Every row reads only data strictly earlier than that row (target lags, + rolling, trend, stockout, replenishment, returns) or same-day attributes + that carry no leakage (calendar, lifecycle, observed price / promotion / + exogenous signal). NO column reads a future observation. + + Group-gated emission: ``groups`` decides which columns appear. ``None`` + resolves to :data:`DEFAULT_V2_GROUPS`. The output column order follows + :func:`canonical_feature_columns_v2`. + + Args: + dates: Observed days in chronological order. + quantities: Observed target values aligned with ``dates``. + prices: Observed unit prices aligned with ``dates``. + baseline_price: Typical price; ``price_factor`` is the ratio to it. + sidecar: All V2 inputs beyond the V1 surface. + groups: Enabled :class:`FeatureGroup` subset. + + Returns: + Row-major matrix ``[n_observations][n_features]``; NaN where a cell's + source data is missing for that day. + + Raises: + ValueError: When ``dates`` / ``quantities`` / ``prices`` lengths + disagree, when ``baseline_price`` is not finite and > 0, when + ``groups`` is empty or names an unknown group, or when an enabled + group's sidecar per-day array length disagrees with ``len(dates)``. + """ + n_rows = len(dates) + if len(quantities) != n_rows or len(prices) != n_rows: + raise ValueError( + f"build_historical_feature_rows_v2: dates ({n_rows}), quantities " + f"({len(quantities)}), prices ({len(prices)}) must all share length." + ) + if not math.isfinite(baseline_price) or baseline_price <= 0.0: + raise ValueError( + f"build_historical_feature_rows_v2: baseline_price must be finite and > 0, got {baseline_price!r}" + ) + resolved_groups = resolve_v2_groups(groups) + resolved_set = set(resolved_groups) + columns = canonical_feature_columns_v2(groups) + column_data: dict[str, list[float]] = {} + + # TARGET_HISTORY + if FeatureGroup.TARGET_HISTORY in resolved_set: + for lag in EXOGENOUS_LAGS_V2: + col: list[float] = [] + for i in range(n_rows): + col.append(quantities[i - lag] if i >= lag else math.nan) + column_data[f"lag_{lag}"] = col + for n_back in SAME_DOW_MEAN_LOOKBACKS_V2: + column_data[f"same_dow_mean_{n_back}"] = _same_dow_mean_column( + dates, quantities, n_back + ) + + # CALENDAR + if FeatureGroup.CALENDAR in resolved_set: + cal = _v2_calendar_columns(dates) + column_data.update(cal) + column_data["is_holiday"] = [1.0 if day in sidecar.holiday_dates else 0.0 for day in dates] + + # ROLLING + if FeatureGroup.ROLLING in resolved_set: + for window in ROLLING_WINDOWS_V2: + column_data[f"rolling_mean_{window}"] = _rolling_mean_column(quantities, window) + column_data["rolling_median_28"] = _rolling_median_column(quantities, 28) + column_data["rolling_std_28"] = _rolling_std_column(quantities, 28) + + # TREND + if FeatureGroup.TREND in resolved_set: + for window in TREND_WINDOWS_V2: + column_data[f"trend_{window}"] = _trend_column(quantities, window) + column_data["rolling_mean_7_vs_28"] = _ratio_two_means_column(quantities, 7, 28) + column_data["rolling_mean_28_vs_prev_28"] = _ratio_window_vs_prev_window_column( + quantities, 28 + ) + + # PRICE_PROMO + if FeatureGroup.PRICE_PROMO in resolved_set: + if sidecar.promo_kinds_per_day: + _validate_per_day_length( + name="promo_kinds_per_day", + actual=len(sidecar.promo_kinds_per_day), + expected=n_rows, + group=FeatureGroup.PRICE_PROMO, + ) + if sidecar.promo_discount_pct_per_day: + _validate_per_day_length( + name="promo_discount_pct_per_day", + actual=len(sidecar.promo_discount_pct_per_day), + expected=n_rows, + group=FeatureGroup.PRICE_PROMO, + ) + column_data.update( + _price_promo_columns_historical( + dates=dates, + prices=prices, + baseline_price=baseline_price, + promo_dates=sidecar.promo_dates, + promo_kinds_per_day=sidecar.promo_kinds_per_day, + promo_discount_pct_per_day=sidecar.promo_discount_pct_per_day, + n_rows=n_rows, + ) + ) + + # INVENTORY + if FeatureGroup.INVENTORY in resolved_set: + _validate_per_day_length( + name="is_stockout_per_day", + actual=len(sidecar.is_stockout_per_day), + expected=n_rows, + group=FeatureGroup.INVENTORY, + ) + _validate_per_day_length( + name="on_hand_qty", + actual=len(sidecar.on_hand_qty), + expected=n_rows, + group=FeatureGroup.INVENTORY, + ) + column_data.update( + _stockout_columns( + is_stockout_per_day=sidecar.is_stockout_per_day, + on_hand_qty=sidecar.on_hand_qty, + n_rows=n_rows, + ) + ) + + # LIFECYCLE + if FeatureGroup.LIFECYCLE in resolved_set: + column_data.update( + _lifecycle_columns( + dates, + launch_date=sidecar.launch_date, + discontinue_date=sidecar.discontinue_date, + ) + ) + + # REPLENISHMENT + if FeatureGroup.REPLENISHMENT in resolved_set: + if len(sidecar.replenishment_event_dates) != len(sidecar.replenishment_event_qty): + raise ValueError( + "build_historical_feature_rows_v2: replenishment_event_dates and " + "replenishment_event_qty must have equal length" + ) + column_data.update( + _replenishment_columns( + dates=dates, + event_dates=sidecar.replenishment_event_dates, + event_qty=sidecar.replenishment_event_qty, + ) + ) + + # RETURNS + if FeatureGroup.RETURNS in resolved_set: + _validate_per_day_length( + name="returns_qty_per_day", + actual=len(sidecar.returns_qty_per_day), + expected=n_rows, + group=FeatureGroup.RETURNS, + ) + column_data.update( + _returns_columns( + quantities=quantities, + returns_qty_per_day=sidecar.returns_qty_per_day, + n_rows=n_rows, + ) + ) + + # EXOGENOUS_WEATHER + if FeatureGroup.EXOGENOUS_WEATHER in resolved_set: + column_data.update( + _exogenous_columns(dates, WEATHER_SIGNAL_NAMES_V2, sidecar.weather_per_day) + ) + + # EXOGENOUS_MACRO + if FeatureGroup.EXOGENOUS_MACRO in resolved_set: + column_data.update(_exogenous_columns(dates, MACRO_SIGNAL_NAMES_V2, sidecar.macro_per_day)) + + rows: list[list[float]] = [[column_data[name][i] for name in columns] for i in range(n_rows)] + return rows + + +# ── Future builder ────────────────────────────────────────────────────────── + + +def _future_rolling_mean_column( + history_tail: list[float], horizon: int, window: int +) -> list[float]: + """Future rolling mean — only horizon day j=1 is computable; j>=2 → NaN. + + For horizon day ``j`` (1..horizon) the source window is + ``T+j-window .. T+j-1``. The window touches only history (``<= T``) iff + ``j == 1``. For ``j >= 2`` the window includes at least one future day + whose target is unobserved → NaN. + """ + out: list[float] = [] + for j in range(1, horizon + 1): + if j == 1 and len(history_tail) >= window: + out.append(sum(history_tail[-window:]) / window) + else: + out.append(math.nan) + return out + + +def _future_rolling_median_column( + history_tail: list[float], horizon: int, window: int +) -> list[float]: + out: list[float] = [] + for j in range(1, horizon + 1): + if j == 1 and len(history_tail) >= window: + window_slice = sorted(history_tail[-window:]) + mid = window // 2 + if window % 2 == 1: + out.append(window_slice[mid]) + else: + out.append((window_slice[mid - 1] + window_slice[mid]) / 2.0) + else: + out.append(math.nan) + return out + + +def _future_rolling_std_column(history_tail: list[float], horizon: int, window: int) -> list[float]: + out: list[float] = [] + for j in range(1, horizon + 1): + if j == 1 and len(history_tail) >= window: + slice_ = history_tail[-window:] + mean = sum(slice_) / window + variance = sum((v - mean) ** 2 for v in slice_) / (window - 1) if window > 1 else 0.0 + out.append(math.sqrt(variance)) + else: + out.append(math.nan) + return out + + +def _future_trend_column(history_tail: list[float], horizon: int, window: int) -> list[float]: + out: list[float] = [] + for j in range(1, horizon + 1): + if j == 1 and len(history_tail) >= window: + y = np.asarray(history_tail[-window:], dtype=np.float64) + x = np.arange(window, dtype=np.float64) + slope = float(np.polyfit(x, y, 1)[0]) + out.append(slope) + else: + out.append(math.nan) + return out + + +def _future_ratio_two_means_column( + history_tail: list[float], horizon: int, num_window: int, den_window: int +) -> list[float]: + out: list[float] = [] + for j in range(1, horizon + 1): + if j == 1 and len(history_tail) >= max(num_window, den_window): + num = sum(history_tail[-num_window:]) / num_window + den = sum(history_tail[-den_window:]) / den_window + out.append(num / den if den != 0.0 else math.nan) + else: + out.append(math.nan) + return out + + +def _future_ratio_window_vs_prev_window_column( + history_tail: list[float], horizon: int, window: int +) -> list[float]: + out: list[float] = [] + for j in range(1, horizon + 1): + if j == 1 and len(history_tail) >= 2 * window: + num = sum(history_tail[-window:]) / window + den = sum(history_tail[-2 * window : -window]) / window + out.append(num / den if den != 0.0 else math.nan) + else: + out.append(math.nan) + return out + + +def _future_same_dow_mean_column( + history_tail_dates: list[date], + history_tail: list[float], + test_dates: list[date], + n_back: int, +) -> list[float]: + """For each test day with weekday w, average the n_back most recent same-DOW history values. + + Reads only ``history_tail`` (entirely ``<= T``); a test day's same-DOW + history slice never moves with horizon offset (no recursion). + """ + out: list[float] = [] + for test_day in test_dates: + same_dow = [ + history_tail[k] + for k in range(len(history_tail_dates)) + if history_tail_dates[k].weekday() == test_day.weekday() + ] + if len(same_dow) >= n_back: + out.append(sum(same_dow[-n_back:]) / n_back) + else: + out.append(math.nan) + return out + + +def build_future_feature_rows_v2( + *, + test_dates: list[date], + history_tail: list[float], + history_tail_dates: list[date], + gap: int, + baseline_price: float, + sidecar: V2FutureSidecar, + history_tail_stockouts: tuple[bool, ...] = (), + history_tail_on_hand: tuple[float | None, ...] = (), + history_tail_replenishment_dates: tuple[date, ...] = (), + history_tail_replenishment_qty: tuple[int, ...] = (), + history_tail_returns_qty: tuple[int, ...] = (), + groups: tuple[FeatureGroup, ...] | None = None, +) -> list[list[float]]: + """Assemble the V2 future feature matrix — leakage-safe. + + A horizon day has no observed target — so the future builder NEVER reads a + target value at a horizon row. Window-aggregate columns (rolling, trend, + stockout/replenishment/returns windows) emit NaN for any horizon day whose + window would touch ``T+1 …``; only horizon day ``j == 1`` is computable + (its window slice is entirely ``<= T``). + + Args: + test_dates: The horizon days ``T+gap+1 … T+gap+horizon`` (chronological). + history_tail: Observed targets ending at the origin ``T`` (entirely + ``<= T``); ``history_tail[-1] == y[T]``. + history_tail_dates: ISO dates aligned with ``history_tail``. + gap: Latency between train end and test start (days). + baseline_price: Median positive training-window price. + sidecar: Future inputs (calendar / lifecycle / assumed price-promo). + history_tail_stockouts: V2 INVENTORY group — per-day stockout flags + aligned with ``history_tail_dates``. + history_tail_on_hand: Per-day on-hand inventory aligned with + ``history_tail_dates``. + history_tail_replenishment_dates: Event-time dates of replenishment + receipts in the data window, sorted ascending. + history_tail_replenishment_qty: Event-time received quantities aligned + with ``history_tail_replenishment_dates``. + history_tail_returns_qty: Per-day returns quantities aligned with + ``history_tail_dates``. + groups: Enabled :class:`FeatureGroup` subset (matches the bundle). + + Returns: + Row-major matrix ``[len(test_dates)][n_features]`` in canonical V2 + column order; NaN-where-future for every CONDITIONALLY_SAFE cell. + + Raises: + ValueError: When ``gap`` is negative, ``baseline_price`` is invalid, + ``groups`` is empty / unknown, or per-day sidecar arrays have + length mismatching ``test_dates`` for an enabled group. + """ + horizon = len(test_dates) + if gap < 0: + raise ValueError(f"build_future_feature_rows_v2: gap must be >= 0, got {gap}") + if not math.isfinite(baseline_price) or baseline_price <= 0.0: + raise ValueError( + f"build_future_feature_rows_v2: baseline_price must be finite and > 0, got {baseline_price!r}" + ) + if len(history_tail) != len(history_tail_dates): + raise ValueError( + "build_future_feature_rows_v2: history_tail and history_tail_dates must have equal length" + ) + resolved_groups = resolve_v2_groups(groups) + resolved_set = set(resolved_groups) + columns = canonical_feature_columns_v2(groups) + column_data: dict[str, list[float]] = {} + + # TARGET_HISTORY — V1 long-lag helper extended over EXOGENOUS_LAGS_V2, + # then gap-trimmed. + if FeatureGroup.TARGET_HISTORY in resolved_set: + lag_cols = build_long_lag_columns(history_tail, gap + horizon, EXOGENOUS_LAGS_V2) + for lag in EXOGENOUS_LAGS_V2: + column_data[f"lag_{lag}"] = lag_cols[f"lag_{lag}"][gap:] + for n_back in SAME_DOW_MEAN_LOOKBACKS_V2: + column_data[f"same_dow_mean_{n_back}"] = _future_same_dow_mean_column( + history_tail_dates, history_tail, test_dates, n_back + ) + + # CALENDAR + if FeatureGroup.CALENDAR in resolved_set: + cal = _v2_calendar_columns(test_dates) + column_data.update(cal) + column_data["is_holiday"] = [ + 1.0 if day in sidecar.holiday_dates else 0.0 for day in test_dates + ] + + # ROLLING — j=1 computable, j>=2 NaN. + if FeatureGroup.ROLLING in resolved_set: + for window in ROLLING_WINDOWS_V2: + column_data[f"rolling_mean_{window}"] = _future_rolling_mean_column( + history_tail, horizon, window + ) + column_data["rolling_median_28"] = _future_rolling_median_column(history_tail, horizon, 28) + column_data["rolling_std_28"] = _future_rolling_std_column(history_tail, horizon, 28) + + # TREND — j=1 computable, j>=2 NaN. + if FeatureGroup.TREND in resolved_set: + for window in TREND_WINDOWS_V2: + column_data[f"trend_{window}"] = _future_trend_column(history_tail, horizon, window) + column_data["rolling_mean_7_vs_28"] = _future_ratio_two_means_column( + history_tail, horizon, 7, 28 + ) + column_data["rolling_mean_28_vs_prev_28"] = _future_ratio_window_vs_prev_window_column( + history_tail, horizon, 28 + ) + + # PRICE_PROMO — driven entirely by the future sidecar (UNSAFE_UNLESS_SUPPLIED). + if FeatureGroup.PRICE_PROMO in resolved_set: + for name, arr in ( + ("price_factor_per_day", sidecar.price_factor_per_day), + ("promo_active_per_day", sidecar.promo_active_per_day), + ("promo_kinds_per_day", sidecar.promo_kinds_per_day), + ("promo_discount_pct_per_day", sidecar.promo_discount_pct_per_day), + ): + if arr and len(arr) != horizon: + _validate_per_day_length( + name=name, + actual=len(arr), + expected=horizon, + group=FeatureGroup.PRICE_PROMO, + ) + price_factor = ( + [math.nan if v is None else float(v) for v in sidecar.price_factor_per_day] + if sidecar.price_factor_per_day + else [math.nan] * horizon + ) + promo_active = ( + [1.0 if v else 0.0 for v in sidecar.promo_active_per_day] + if sidecar.promo_active_per_day + else [math.nan] * horizon + ) + promo_discount = ( + [float(v) for v in sidecar.promo_discount_pct_per_day] + if sidecar.promo_discount_pct_per_day + else [math.nan] * horizon + ) + if sidecar.promo_kinds_per_day: + markdown = [ + 1.0 if "markdown" in sidecar.promo_kinds_per_day[i] else 0.0 for i in range(horizon) + ] + bundle = [ + 1.0 if "bundle" in sidecar.promo_kinds_per_day[i] else 0.0 for i in range(horizon) + ] + else: + markdown = [math.nan] * horizon + bundle = [math.nan] * horizon + column_data["price_factor"] = price_factor + column_data["promo_active"] = promo_active + column_data["promo_discount_pct"] = promo_discount + column_data["promo_kind_markdown_active"] = markdown + column_data["promo_kind_bundle_active"] = bundle + + # INVENTORY — j=1 may be computable from history_tail; j>=2 NaN unless + # caller supplies projected stockouts (V2 MVP does NOT support + # caller-supplied projections, so j>=2 is always NaN). + if FeatureGroup.INVENTORY in resolved_set: + is_stockout_lag1 = ( + [float(1.0 if history_tail_stockouts[-1] else 0.0)] + [math.nan] * (horizon - 1) + if history_tail_stockouts + else [math.nan] * horizon + ) + stockout_7 = ( + [float(sum(1 if flag else 0 for flag in history_tail_stockouts[-7:]))] + + [math.nan] * (horizon - 1) + if len(history_tail_stockouts) >= 7 + else [math.nan] * horizon + ) + stockout_28 = ( + [float(sum(1 if flag else 0 for flag in history_tail_stockouts[-28:]))] + + [math.nan] * (horizon - 1) + if len(history_tail_stockouts) >= 28 + else [math.nan] * horizon + ) + # inventory_available_ratio_28 — j=1: mean(observed)/max(observed) + window = INVENTORY_AVAILABILITY_WINDOW_V2 + if len(history_tail_on_hand) >= window: + slice_ = history_tail_on_hand[-window:] + observed = [v for v in slice_ if v is not None] + if observed and max(observed) > 0.0: + avail = sum(observed) / len(observed) / max(observed) + else: + avail = math.nan + avail_ratio = [avail] + [math.nan] * (horizon - 1) + else: + avail_ratio = [math.nan] * horizon + column_data["is_stockout_lag1"] = is_stockout_lag1 + column_data["stockout_days_7"] = stockout_7 + column_data["stockout_days_28"] = stockout_28 + column_data["inventory_available_ratio_28"] = avail_ratio + + # LIFECYCLE — pure function of test dates + launch/discontinue (knowable at T). + if FeatureGroup.LIFECYCLE in resolved_set: + column_data.update( + _lifecycle_columns( + test_dates, + launch_date=sidecar.launch_date, + discontinue_date=sidecar.discontinue_date, + ) + ) + + # REPLENISHMENT — events strictly before each test day. With V2 MVP we + # only consider events from history (the caller does not posit future + # replenishments), so j>=2 uses the same prior-event set as j=1. + if FeatureGroup.REPLENISHMENT in resolved_set: + if len(history_tail_replenishment_dates) != len(history_tail_replenishment_qty): + raise ValueError("build_future_feature_rows_v2: replenishment dates and qty must align") + days_since: list[float] = [] + count_14: list[float] = [] + qty_28: list[float] = [] + for j, day in enumerate(test_dates): + prior = [ + (d, q) + for d, q in zip( + history_tail_replenishment_dates, + history_tail_replenishment_qty, + strict=True, + ) + if d < day + ] + if prior: + last = max(d for d, _ in prior) + days_since.append(float((day - last).days)) + else: + days_since.append(math.nan) + # Counts / qty only on j=1 to mirror the historical builder's + # strictly-earlier rule; further horizon days have no new events + # in the supplied sidecar. + if j == 0: + win14_start = day.toordinal() - REPLENISHMENT_WINDOW_V2 + win28_start = day.toordinal() - REPLENISHMENT_QTY_WINDOW_V2 + count_14.append(float(sum(1 for d, _ in prior if d.toordinal() >= win14_start))) + qty_28.append(float(sum(q for d, q in prior if d.toordinal() >= win28_start))) + else: + count_14.append(math.nan) + qty_28.append(math.nan) + column_data["days_since_last_replenishment"] = days_since + column_data["replenishment_count_14"] = count_14 + column_data["replenishment_qty_28"] = qty_28 + + # RETURNS — j=1 computable from history_tail_returns_qty; j>=2 NaN. + if FeatureGroup.RETURNS in resolved_set: + returns_floats = [float(v) for v in history_tail_returns_qty] + for window in RETURNS_WINDOWS_V2: + if len(returns_floats) >= window: + first = float(sum(returns_floats[-window:])) + else: + first = math.nan + column_data[f"returns_qty_{window}"] = [first] + [math.nan] * (horizon - 1) + rate_window = RETURNS_RATE_WINDOW_V2 + if len(returns_floats) >= rate_window and len(history_tail) >= rate_window: + ret_sum = sum(returns_floats[-rate_window:]) + sales_sum = sum(history_tail[-rate_window:]) + first = ret_sum / sales_sum if sales_sum > 0.0 else 0.0 + else: + first = math.nan + column_data["returns_rate_28"] = [first] + [math.nan] * (horizon - 1) + + # EXOGENOUS_WEATHER / MACRO — NaN when the date has no entry in the sidecar. + if FeatureGroup.EXOGENOUS_WEATHER in resolved_set: + column_data.update( + _exogenous_columns(test_dates, WEATHER_SIGNAL_NAMES_V2, sidecar.weather_per_day) + ) + if FeatureGroup.EXOGENOUS_MACRO in resolved_set: + column_data.update( + _exogenous_columns(test_dates, MACRO_SIGNAL_NAMES_V2, sidecar.macro_per_day) + ) + + # Defensive: any column the manifest expects but the dispatcher above did + # not produce becomes an all-NaN column (cannot happen in practice — every + # enabled group fills every one of its columns — but mirrors the V1 + # ``assemble_future_frame`` defensive shape). + for column in columns: + if column not in column_data: + column_data[column] = [math.nan] * horizon + + rows: list[list[float]] = [[column_data[name][j] for name in columns] for j in range(horizon)] + return rows + + +__all__ = [ + "build_future_feature_rows_v2", + "build_historical_feature_rows_v2", +] + +# Cross-reference the V1 calendar columns set so static analysers see it used. +_ = CALENDAR_COLUMNS diff --git a/app/shared/feature_frames/sidecar.py b/app/shared/feature_frames/sidecar.py new file mode 100644 index 00000000..526dcfe5 --- /dev/null +++ b/app/shared/feature_frames/sidecar.py @@ -0,0 +1,116 @@ +"""V2 sidecar dataclasses — pure data carriers for V2 row builders (PRP-35). + +The V2 row builders (``rows_v2.py``) accept every input beyond the V1 surface +through these two frozen dataclasses. They are pure data — stdlib only, +``app/shared/**`` leaf-level — so the DB-loading side (``v2_loaders.py`` in the +forecasting slice) stays cross-slice-import-free for ``app/shared``. + +Alignment contract (the row builders raise ``ValueError`` when violated): + +- Every per-day tuple aligned with ``dates`` (or ``test_dates`` for the future + sidecar) MUST have the same length as that ``dates`` tuple WHENEVER the + owning :class:`~app.shared.feature_frames.contract_v2.FeatureGroup` is + enabled. Length mismatch is a programmer/contract error, not a missing-data + case. +- Sets / mappings (``promo_dates``, ``holiday_dates``, ``weather_per_day``, + ``macro_per_day``) are queried by date membership. A date with no entry → a + ``NaN`` cell at that row, NEVER a zero-fill. +- ``replenishment_event_dates`` / ``replenishment_event_qty`` are event-time + arrays (one entry per receipt event), NOT per-day-aligned. Their only + alignment invariant is length parity between the two tuples. + +When a feature group is NOT enabled, the matching sidecar fields MAY be empty +tuples / dicts; the row builder will not read them. When a group IS enabled +but a per-day source value is missing (``on_hand_qty[i] is None``, no entry in +``weather_per_day[dates[i]]``, no replenishment event before day ``i``), the +cell is NaN. HGBR consumes NaN natively. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import date + + +@dataclass(frozen=True) +class V2HistoricalSidecar: + """Inputs the historical V2 builder needs beyond the V1 surface. + + See module docstring for the alignment invariants. Empty defaults mean + "this group's data is not supplied" — safe to leave when the matching + :class:`FeatureGroup` is disabled, an error if the group IS enabled + (caught loud in the row builder). + + Attributes: + promo_dates: V1 carryover — days a promotion covered. + holiday_dates: V1 carryover — calendar holiday days. + launch_date: V1 carryover — product's launch date, or None. + discontinue_date: Product's discontinue date, or None. + on_hand_qty: Per-day on-hand inventory, aligned with ``dates``; + entries MAY be None when the snapshot is absent. + is_stockout_per_day: Per-day stockout flag, aligned with ``dates``. + replenishment_event_dates: Event-time dates of replenishment receipts + within the data window, sorted ascending. + replenishment_event_qty: Event-time received quantities; same length + as ``replenishment_event_dates``. + returns_qty_per_day: Per-day returned-units count, aligned with + ``dates``; ``0`` for days with no return. + promo_kinds_per_day: Per-day set of active promotion kinds (subset of + ``{"pct_off", "bogo", "bundle", "markdown"}``); empty set on days + with no promotion. + promo_discount_pct_per_day: Per-day discount fraction (0.0..1.0); + ``0.0`` on days with no promotion. + weather_per_day: ``{date: {signal_name: value}}`` for store-specific + weather signals; absent dates → NaN cell. + macro_per_day: ``{date: {signal_name: value}}`` for chain-wide macro + signals; absent dates → NaN cell. + """ + + # V1 carryover + promo_dates: frozenset[date] = field(default_factory=frozenset) + holiday_dates: frozenset[date] = field(default_factory=frozenset) + launch_date: date | None = None + # Lifecycle + discontinue_date: date | None = None + # Inventory (per-day, aligned with dates) + on_hand_qty: tuple[float | None, ...] = () + is_stockout_per_day: tuple[bool, ...] = () + # Replenishment (timestamps, NOT per-day) + replenishment_event_dates: tuple[date, ...] = () + replenishment_event_qty: tuple[int, ...] = () + # Returns (per-day quantity, 0 when no return) + returns_qty_per_day: tuple[int, ...] = () + # Promotion (per-day kind set + discount pct) + promo_kinds_per_day: tuple[frozenset[str], ...] = () + promo_discount_pct_per_day: tuple[float, ...] = () + # Exogenous (date → signal_name → value) + weather_per_day: dict[date, dict[str, float]] = field(default_factory=dict) + macro_per_day: dict[date, dict[str, float]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class V2FutureSidecar: + """Inputs the future V2 builder accepts when re-forecasting. + + EVERY field is either knowable at origin ``T`` (calendar holidays, + ``launch_date`` / ``discontinue_date``) or *posited by the caller as an + assumption* (price, promotion). For truly-unknowable groups (weather, + macro) the caller MAY supply observed-then-projected values or leave the + dict empty → the corresponding column is NaN at the horizon row. + + See module docstring for alignment invariants — all per-day tuples align + with ``test_dates``. + """ + + holiday_dates: frozenset[date] = field(default_factory=frozenset) + launch_date: date | None = None + discontinue_date: date | None = None + # Per-day exogenous inputs (None / 0.0 / empty == "not posited" → NaN cell) + price_factor_per_day: tuple[float | None, ...] = () + promo_active_per_day: tuple[bool, ...] = () + promo_kinds_per_day: tuple[frozenset[str], ...] = () + promo_discount_pct_per_day: tuple[float, ...] = () + # Phase 2 future inputs — typically all-None / empty for V2 MVP + inventory_on_hand_per_day: tuple[float | None, ...] = () + weather_per_day: dict[date, dict[str, float]] = field(default_factory=dict) + macro_per_day: dict[date, dict[str, float]] = field(default_factory=dict) diff --git a/app/shared/feature_frames/tests/test_contract_v2.py b/app/shared/feature_frames/tests/test_contract_v2.py new file mode 100644 index 00000000..adc98a9c --- /dev/null +++ b/app/shared/feature_frames/tests/test_contract_v2.py @@ -0,0 +1,288 @@ +"""Unit tests for the V2 feature-frame contract (PRP-35). + +Mirrors ``test_contract.py``: pins the V2 pinned constants, the column manifest ++ order, group enablement semantics, the V2 safety taxonomy coverage, and the +leaf-level architectural invariant (``app/shared/**`` never imports +``app/features/**``) — now extended to walk ``contract_v2.py``, ``rows_v2.py``, +and ``sidecar.py``. + +The leakage invariants live separately in ``test_leakage_v2.py`` (load-bearing). +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from app.shared.feature_frames import ( + CALENDAR_COLUMNS, + DEFAULT_V2_GROUPS, + EXOGENOUS_LAGS_V2, + HISTORY_TAIL_DAYS_V2, + ROLLING_WINDOWS_V2, + SAME_DOW_MEAN_LOOKBACKS_V2, + TREND_WINDOWS_V2, + FeatureGroup, + FeatureSafety, + V2ColumnSpec, + canonical_feature_columns, + canonical_feature_columns_v2, + v2_column_manifest, + v2_feature_groups_dict, + v2_feature_safety, + v2_feature_safety_classes, + v2_pinned_constants, +) + +# --- pinned constants --------------------------------------------------------- + + +def test_pinned_constants_v2() -> None: + """V2 modelling constants hold their decided values.""" + assert EXOGENOUS_LAGS_V2 == (1, 7, 14, 28, 56, 364) + assert 364 in EXOGENOUS_LAGS_V2 # DOW-preserving yearly lag + assert 365 not in EXOGENOUS_LAGS_V2 + assert ROLLING_WINDOWS_V2 == (7, 28, 90) + assert TREND_WINDOWS_V2 == (30, 90) + assert SAME_DOW_MEAN_LOOKBACKS_V2 == (4, 8) + assert HISTORY_TAIL_DAYS_V2 == 400 + + +def test_default_groups_subset() -> None: + """Default groups exclude the Phase-2 sidecar groups (MVP-green default).""" + default = set(DEFAULT_V2_GROUPS) + assert FeatureGroup.TARGET_HISTORY in default + assert FeatureGroup.CALENDAR in default + assert FeatureGroup.ROLLING in default + assert FeatureGroup.TREND in default + assert FeatureGroup.PRICE_PROMO in default + assert FeatureGroup.LIFECYCLE in default + # Off by default + for group in ( + FeatureGroup.INVENTORY, + FeatureGroup.REPLENISHMENT, + FeatureGroup.RETURNS, + FeatureGroup.EXOGENOUS_WEATHER, + FeatureGroup.EXOGENOUS_MACRO, + ): + assert group not in default + + +# --- manifest order + group enablement --------------------------------------- + + +def test_default_v2_manifest_contains_yearly_lag_and_calendar_extensions() -> None: + columns = canonical_feature_columns_v2() + assert "lag_364" in columns + assert "same_dow_mean_4" in columns + assert "same_dow_mean_8" in columns + # V2 calendar extensions + assert "week_of_year_sin" in columns + assert "day_of_month_cos" in columns + # V2 rolling + trend columns + assert "rolling_mean_7" in columns + assert "rolling_mean_28" in columns + assert "rolling_mean_90" in columns + assert "trend_30" in columns + assert "trend_90" in columns + + +def test_v2_manifest_subset_when_groups_narrowed() -> None: + narrow = canonical_feature_columns_v2( + groups=(FeatureGroup.TARGET_HISTORY, FeatureGroup.CALENDAR) + ) + # Only target_history + calendar columns appear + for name in narrow: + assert ( + name.startswith("lag_") + or name.startswith("same_dow_mean_") + or name + in { + "dow_sin", + "dow_cos", + "month_sin", + "month_cos", + "is_weekend", + "is_month_end", + "week_of_year_sin", + "week_of_year_cos", + "day_of_month_sin", + "day_of_month_cos", + "is_holiday", + } + ) + # And rolling / trend / price columns must NOT appear + assert "rolling_mean_7" not in narrow + assert "price_factor" not in narrow + + +def test_v2_column_order_is_deterministic() -> None: + """Two calls with the same groups produce the same column list (byte-stable).""" + first = canonical_feature_columns_v2() + second = canonical_feature_columns_v2() + assert first == second + + +def test_v2_manifest_respects_canonical_group_order() -> None: + """Caller's group ordering is normalised to canonical group order.""" + a = canonical_feature_columns_v2(groups=(FeatureGroup.CALENDAR, FeatureGroup.TARGET_HISTORY)) + b = canonical_feature_columns_v2(groups=(FeatureGroup.TARGET_HISTORY, FeatureGroup.CALENDAR)) + assert a == b + # target_history columns come strictly before calendar columns + assert a.index("lag_1") < a.index("dow_sin") + + +def test_v2_includes_v1_calendar_columns_at_same_relative_position() -> None: + """The V1 CALENDAR_COLUMNS appear in the V2 manifest in their V1 order. + + The V2 manifest may add columns within the V2 CALENDAR group (week_of_year, + day_of_month) but must preserve the V1 in-group order for back-compat + consumers. + """ + v2_calendar = canonical_feature_columns_v2(groups=(FeatureGroup.CALENDAR,)) + v1_present = [c for c in v2_calendar if c in CALENDAR_COLUMNS] + assert v1_present == list(CALENDAR_COLUMNS) + + +def test_v2_includes_every_v1_canonical_column() -> None: + """Every V1 canonical column (V1 default lags + calendar + exogenous) is + reachable via the V2 manifest when the appropriate V2 groups are enabled.""" + v1_columns = set(canonical_feature_columns()) + v2_full = set( + canonical_feature_columns_v2( + groups=( + FeatureGroup.TARGET_HISTORY, + FeatureGroup.CALENDAR, + FeatureGroup.PRICE_PROMO, + FeatureGroup.LIFECYCLE, + ) + ) + ) + # All V1 columns must be in V2's full set (modulo the column home — V1's + # `is_holiday` is in V2's CALENDAR group, not PRICE_PROMO). + missing = v1_columns - v2_full + assert not missing, f"V2 manifest missing V1 columns: {sorted(missing)}" + + +# --- V2 safety taxonomy ------------------------------------------------------- + + +def test_every_default_v2_column_is_classifiable() -> None: + """Every default-V2 column resolves to a FeatureSafety class via v2_feature_safety.""" + for column in canonical_feature_columns_v2(): + assert isinstance(v2_feature_safety(column), FeatureSafety) + + +def test_v2_calendar_and_lifecycle_columns_are_SAFE() -> None: + """Calendar + lifecycle columns are pure functions of the date — SAFE.""" + for column in canonical_feature_columns_v2(groups=(FeatureGroup.CALENDAR,)): + assert v2_feature_safety(column) is FeatureSafety.SAFE + for column in canonical_feature_columns_v2(groups=(FeatureGroup.LIFECYCLE,)): + assert v2_feature_safety(column) is FeatureSafety.SAFE + + +def test_v2_target_history_columns_are_CONDITIONALLY_SAFE() -> None: + for column in canonical_feature_columns_v2(groups=(FeatureGroup.TARGET_HISTORY,)): + assert v2_feature_safety(column) is FeatureSafety.CONDITIONALLY_SAFE + + +def test_v2_price_promo_columns_are_UNSAFE_UNLESS_SUPPLIED() -> None: + for column in canonical_feature_columns_v2(groups=(FeatureGroup.PRICE_PROMO,)): + assert v2_feature_safety(column) is FeatureSafety.UNSAFE_UNLESS_SUPPLIED + + +def test_v2_feature_safety_rejects_an_unclassified_column() -> None: + try: + v2_feature_safety("mystery_feature_v2") + except KeyError: + pass + else: + raise AssertionError("v2_feature_safety must raise KeyError for an unknown column") + + +# --- v2_feature_groups_dict + v2_feature_safety_classes ----------------------- + + +def test_v2_feature_groups_dict_maps_columns_to_group_names() -> None: + columns = canonical_feature_columns_v2() + mapping = v2_feature_groups_dict(columns) + # Every default group is represented + for group in DEFAULT_V2_GROUPS: + assert group.value in mapping + assert mapping[group.value], f"group {group.value} has no columns" + # The combined columns reconstruct the full default manifest + all_columns_back = [c for group_cols in mapping.values() for c in group_cols] + assert set(all_columns_back) == set(columns) + + +def test_v2_feature_safety_classes_returns_full_map() -> None: + columns = canonical_feature_columns_v2() + classes = v2_feature_safety_classes(columns) + assert set(classes.keys()) == set(columns) + assert set(classes.values()) <= {"safe", "conditionally_safe", "unsafe_unless_supplied"} + + +# --- v2_pinned_constants ------------------------------------------------------ + + +def test_v2_pinned_constants_snapshot_matches_constants() -> None: + snap = v2_pinned_constants() + assert tuple(snap["exogenous_lags"]) == EXOGENOUS_LAGS_V2 + assert tuple(snap["rolling_windows"]) == ROLLING_WINDOWS_V2 + assert tuple(snap["trend_windows"]) == TREND_WINDOWS_V2 + + +# --- v2_column_manifest dataclass shape --------------------------------------- + + +def test_v2_column_manifest_carries_spec_objects() -> None: + manifest = v2_column_manifest() + assert manifest # non-empty + for spec in manifest: + assert isinstance(spec, V2ColumnSpec) + assert isinstance(spec.name, str) + assert isinstance(spec.group, FeatureGroup) + assert isinstance(spec.safety, FeatureSafety) + + +# --- LOUD failure modes -------------------------------------------------------- + + +def test_empty_groups_raises() -> None: + """An empty groups tuple is a misuse — zero-column matrix is forbidden.""" + try: + canonical_feature_columns_v2(groups=()) + except ValueError: + pass + else: + raise AssertionError("expected ValueError for empty groups") + + +# --- architectural invariant (extended to V2 modules) ------------------------ + + +def test_v2_modules_are_leaf_level() -> None: + """``app/shared/feature_frames/**`` is leaf-level — never imports vertical slices. + + Extended over contract_v2.py, rows_v2.py, sidecar.py so the AST-walk + invariant catches a V2 regression. + """ + pkg_dir = Path(__file__).resolve().parents[1] + walked: set[str] = set() + for py_file in pkg_dir.rglob("*.py"): + walked.add(py_file.name) + source = py_file.read_text(encoding="utf-8") + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom) and node.module: + assert not node.module.startswith("app.features"), ( + f"ARCHITECTURE BREACH: {py_file} imports from {node.module}" + ) + if isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith("app.features"), ( + f"ARCHITECTURE BREACH: {py_file} imports {alias.name}" + ) + # The V2 modules must exist and be covered by the walk above. + assert {"contract_v2.py", "rows_v2.py", "sidecar.py"} <= walked, ( + f"expected contract_v2.py + rows_v2.py + sidecar.py in the walk, got {sorted(walked)}" + ) diff --git a/app/shared/feature_frames/tests/test_leakage_v2.py b/app/shared/feature_frames/tests/test_leakage_v2.py new file mode 100644 index 00000000..2afd3547 --- /dev/null +++ b/app/shared/feature_frames/tests/test_leakage_v2.py @@ -0,0 +1,339 @@ +"""Leakage spec for the V2 feature-frame builders — LOAD-BEARING (PRP-35). + +This file IS the spec, mirroring ``app/shared/feature_frames/tests/test_leakage.py``: +it must NEVER be weakened to make a feature pass (AGENTS.md § Safety). + +The V2 builders extend V1 with rolling / trend / lifecycle / inventory / +replenishment / returns / exogenous columns. The invariant is the same as V1: + + A future feature value for horizon day ``D`` may use ONLY information + knowable at the forecast origin ``T``: the observed history up to and + including ``T``, the calendar (a pure function of the date), launch / + discontinue dates, or scenario-assumption inputs posited by the caller. + It NEVER reads an observed target — or any sidecar value — at a horizon + day ``D`` (which lies after ``T``). + +Sequential targets (1.0 … N.0) are used so leakage is mathematically +detectable: a rolling-mean cell at row ``i`` MUST be strictly less than the +current row's target ``i+1`` for the sequential fixture. A disjoint future +target set ({9000.0 … 9999.0}) pins the future-builder side: any future-target +value appearing in any feature cell is a leak. +""" + +from __future__ import annotations + +import math +from datetime import date, timedelta + +import pytest + +from app.shared.feature_frames import ( + EXOGENOUS_LAGS_V2, + ROLLING_WINDOWS_V2, + SAME_DOW_MEAN_LOOKBACKS_V2, + TREND_WINDOWS_V2, + FeatureGroup, + V2FutureSidecar, + V2HistoricalSidecar, + build_future_feature_rows_v2, + build_historical_feature_rows_v2, + canonical_feature_columns_v2, +) + +# Sequential observed history: 400 days so lag_364 / rolling_90 / trend_90 are +# all resolvable for the future builder's j=1 row. +_N = 400 +_ORIGIN = date(2026, 6, 30) +_HISTORY_DATES = [date(2026, 1, 1) + timedelta(days=offset) for offset in range(_N)] +_HISTORY_TAIL = [1000.0 + float(i) for i in range(_N)] # 1000.0 … 1399.0 +# A DISJOINT "future target" set the V2 builders must never read. +_HORIZON = 21 +_FUTURE_TARGETS = {9000.0 + float(i) for i in range(_HORIZON)} + + +# ─── Historical builder — leakage by sequential-target detection ──────────── + + +def _build_historical() -> tuple[list[str], list[list[float]]]: + """Assemble a V2 historical matrix from sequential targets.""" + columns = canonical_feature_columns_v2() + sidecar = V2HistoricalSidecar() + rows = build_historical_feature_rows_v2( + dates=_HISTORY_DATES, + quantities=_HISTORY_TAIL, + prices=[10.0] * _N, + baseline_price=10.0, + sidecar=sidecar, + ) + return columns, rows + + +def test_v2_lag_columns_read_only_strictly_earlier_observations() -> None: + """Every V2 ``lag_*`` cell with sequential targets is ``< quantity[i]`` or NaN.""" + columns, rows = _build_historical() + for lag in EXOGENOUS_LAGS_V2: + col_index = columns.index(f"lag_{lag}") + for i in range(_N): + cell = rows[i][col_index] + if i < lag: + assert math.isnan(cell), f"row {i}: lag_{lag} expected NaN, got {cell}" + continue + expected = _HISTORY_TAIL[i - lag] + assert cell == expected, f"LEAKAGE at row {i}: lag_{lag}={cell} != expected={expected}" + assert cell < _HISTORY_TAIL[i], ( + f"LEAKAGE at row {i}: lag_{lag}={cell} >= current={_HISTORY_TAIL[i]}" + ) + + +def test_v2_rolling_mean_reads_only_strictly_earlier_rows() -> None: + """``rolling_mean_W`` at row ``i`` strictly < ``quantity[i]`` (sequential fixture).""" + columns, rows = _build_historical() + for window in ROLLING_WINDOWS_V2: + col_index = columns.index(f"rolling_mean_{window}") + for i in range(_N): + cell = rows[i][col_index] + if i < window: + assert math.isnan(cell), f"row {i}: rolling_mean_{window} expected NaN" + continue + expected = sum(_HISTORY_TAIL[i - window : i]) / window + assert cell == expected, f"row {i}: rolling_mean_{window}={cell} != expected={expected}" + assert cell < _HISTORY_TAIL[i], ( + f"LEAKAGE at row {i}: rolling_mean_{window}={cell} >= current={_HISTORY_TAIL[i]}" + ) + + +def test_v2_rolling_std_first_rows_are_nan() -> None: + columns, rows = _build_historical() + col_index = columns.index("rolling_std_28") + for i in range(28): + assert math.isnan(rows[i][col_index]), f"rolling_std_28 row {i}: expected NaN" + # After 28 rows the std becomes computable. + for i in range(28, _N): + assert not math.isnan(rows[i][col_index]), ( + f"rolling_std_28 row {i}: expected a value, got NaN" + ) + + +def test_v2_same_dow_mean_reads_only_strictly_earlier_observations() -> None: + """Same-DOW means only see earlier same-weekday rows.""" + columns, rows = _build_historical() + for n_back in SAME_DOW_MEAN_LOOKBACKS_V2: + col_index = columns.index(f"same_dow_mean_{n_back}") + for i in range(_N): + cell = rows[i][col_index] + if math.isnan(cell): + continue + # If non-NaN: cell must be strictly < current quantity (sequential + # fixture: any earlier index ⇒ smaller value). + assert cell < _HISTORY_TAIL[i], ( + f"LEAKAGE at row {i}: same_dow_mean_{n_back}={cell} >= current" + ) + + +def test_v2_trend_columns_first_window_rows_are_nan() -> None: + columns, rows = _build_historical() + for window in TREND_WINDOWS_V2: + col_index = columns.index(f"trend_{window}") + for i in range(window): + assert math.isnan(rows[i][col_index]), ( + f"trend_{window} row {i}: expected NaN (insufficient history)" + ) + + +# ─── Future builder — no future-target value may ever appear ──────────────── + + +def _build_future(gap: int = 0, horizon: int = _HORIZON) -> tuple[list[str], list[list[float]]]: + test_dates = [_ORIGIN + timedelta(days=gap + offset) for offset in range(1, horizon + 1)] + history_tail_dates = _HISTORY_DATES + columns = canonical_feature_columns_v2() + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=history_tail_dates, + gap=gap, + baseline_price=10.0, + sidecar=V2FutureSidecar(), + ) + return columns, rows + + +def test_future_v2_lag_cells_are_drawn_only_from_history() -> None: + """Every non-NaN ``lag_*`` cell in the V2 future matrix is from ``history_tail``.""" + columns, rows = _build_future(gap=0) + history_values = set(_HISTORY_TAIL) + for lag in EXOGENOUS_LAGS_V2: + col_index = columns.index(f"lag_{lag}") + for j in range(_HORIZON): + cell = rows[j][col_index] + if math.isnan(cell): + continue + assert cell in history_values, ( + f"future lag_{lag} day {j}: leaked non-history value {cell}" + ) + assert cell not in _FUTURE_TARGETS, ( + f"future lag_{lag} day {j}: leaked FUTURE target {cell}" + ) + + +@pytest.mark.parametrize("gap", [0, 3, 7]) +def test_future_v2_lag_nan_pattern_matches_source_index(gap: int) -> None: + """A V2 ``lag_k`` cell is NaN exactly when its source day is in the test window. + + For lag ``k`` and test day ``j`` (0-indexed) the source day relative to + ``T`` is ``gap + j + 1 - k``. The cell MUST be NaN exactly when + ``gap + j - k >= 0`` (source is a future day). + """ + columns, rows = _build_future(gap=gap, horizon=_HORIZON) + for lag in EXOGENOUS_LAGS_V2: + col_index = columns.index(f"lag_{lag}") + for j in range(_HORIZON): + cell = rows[j][col_index] + if gap + j - lag >= 0: + assert math.isnan(cell), ( + f"gap={gap} lag_{lag} day {j}: source future — expected NaN, got {cell}" + ) + else: + assert not math.isnan(cell), ( + f"gap={gap} lag_{lag} day {j}: source in history — expected value" + ) + + +def test_future_v2_rolling_mean_only_horizon_day_1_is_computable() -> None: + """``rolling_mean_W`` is computable at horizon ``j=1`` (window entirely ``<= T``); + NaN for every ``j >= 2`` (window touches future). + """ + columns, rows = _build_future(gap=0) + for window in ROLLING_WINDOWS_V2: + col_index = columns.index(f"rolling_mean_{window}") + # j=0 (test day 1) — computable + first = rows[0][col_index] + expected = sum(_HISTORY_TAIL[-window:]) / window + assert first == expected, ( + f"future rolling_mean_{window} day 1: expected {expected}, got {first}" + ) + # j>=1 — NaN + for j in range(1, _HORIZON): + assert math.isnan(rows[j][col_index]), ( + f"future rolling_mean_{window} day {j + 1}: expected NaN, got {rows[j][col_index]}" + ) + + +def test_future_v2_trend_only_horizon_day_1_is_computable() -> None: + columns, rows = _build_future(gap=0) + for window in TREND_WINDOWS_V2: + col_index = columns.index(f"trend_{window}") + assert not math.isnan(rows[0][col_index]), f"future trend_{window} day 1: expected a value" + for j in range(1, _HORIZON): + assert math.isnan(rows[j][col_index]), ( + f"future trend_{window} day {j + 1}: expected NaN" + ) + + +def test_future_v2_calendar_columns_independent_of_target_series() -> None: + """Calendar columns read only the dates — they cannot leak the target.""" + columns, rows = _build_future(gap=0) + history_values = set(_HISTORY_TAIL) + cal_names = { + "dow_sin", + "dow_cos", + "month_sin", + "month_cos", + "is_weekend", + "is_month_end", + "week_of_year_sin", + "week_of_year_cos", + "day_of_month_sin", + "day_of_month_cos", + "is_holiday", + } + for name in cal_names: + col_index = columns.index(name) + for j in range(_HORIZON): + cell = rows[j][col_index] + assert cell not in history_values, ( + f"calendar {name} day {j}: cell {cell} accidentally coincides with history" + ) + assert cell not in _FUTURE_TARGETS, ( + f"calendar {name} day {j}: cell {cell} accidentally coincides with future target" + ) + + +def test_future_v2_lag_364_is_dow_aligned() -> None: + """``lag_364`` at horizon day 1 reads ``history_tail[-364]`` — same weekday as day 1.""" + columns, rows = _build_future(gap=0) + col_index = columns.index("lag_364") + expected = _HISTORY_TAIL[-364] + assert rows[0][col_index] == expected, ( + f"future lag_364 day 1: expected {expected}, got {rows[0][col_index]}" + ) + # Day 365 → source index (365-1) - 364 = 0 (non-negative) → NaN + rows365 = build_future_feature_rows_v2( + test_dates=[_ORIGIN + timedelta(days=offset) for offset in range(1, 366)], + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_DATES, + gap=0, + baseline_price=10.0, + sidecar=V2FutureSidecar(), + ) + assert math.isnan(rows365[364][col_index]), ( + "future lag_364 at horizon day 365: source is T+1 (future) — expected NaN" + ) + + +def test_future_v2_inventory_group_off_default_omits_inventory_columns() -> None: + """Default-V2 manifest does not include INVENTORY columns (off by default).""" + columns, _ = _build_future(gap=0) + assert "is_stockout_lag1" not in columns + assert "stockout_days_7" not in columns + assert "inventory_available_ratio_28" not in columns + + +def test_future_v2_inventory_stockout_days_horizon_2_plus_nan() -> None: + """When INVENTORY enabled but no caller-supplied projection, j>=2 is NaN.""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_DATES, + gap=0, + baseline_price=10.0, + sidecar=V2FutureSidecar(), + history_tail_stockouts=tuple([False] * _N), + groups=(FeatureGroup.INVENTORY,), + ) + columns = canonical_feature_columns_v2(groups=(FeatureGroup.INVENTORY,)) + for name in ("is_stockout_lag1", "stockout_days_7", "stockout_days_28"): + col_index = columns.index(name) + # Day 1 may be a value (computable from history) or NaN; day >= 2 must be NaN + for j in range(1, _HORIZON): + assert math.isnan(rows[j][col_index]), ( + f"{name} day {j + 1}: expected NaN (no projected stockouts), got {rows[j][col_index]}" + ) + + +def test_future_v2_price_promo_is_nan_when_unsupplied() -> None: + """PRICE_PROMO columns are UNSAFE_UNLESS_SUPPLIED — empty sidecar arrays → NaN.""" + test_dates = [_ORIGIN + timedelta(days=offset) for offset in range(1, _HORIZON + 1)] + rows = build_future_feature_rows_v2( + test_dates=test_dates, + history_tail=_HISTORY_TAIL, + history_tail_dates=_HISTORY_DATES, + gap=0, + baseline_price=10.0, + sidecar=V2FutureSidecar(), # no posited price / promo + groups=(FeatureGroup.PRICE_PROMO,), + ) + columns = canonical_feature_columns_v2(groups=(FeatureGroup.PRICE_PROMO,)) + for name in ( + "price_factor", + "promo_active", + "promo_discount_pct", + "promo_kind_markdown_active", + "promo_kind_bundle_active", + ): + col_index = columns.index(name) + for j in range(_HORIZON): + assert math.isnan(rows[j][col_index]), ( + f"{name} day {j + 1}: expected NaN (sidecar empty), got {rows[j][col_index]}" + ) diff --git a/docs/optional-features/10-baseforecaster-feature-contract.md b/docs/optional-features/10-baseforecaster-feature-contract.md index 4d6f89a1..5a34c8bb 100644 --- a/docs/optional-features/10-baseforecaster-feature-contract.md +++ b/docs/optional-features/10-baseforecaster-feature-contract.md @@ -114,3 +114,43 @@ The change should document that: - Joblib persistence documentation: https://joblib.readthedocs.io/en/stable/persistence.html - Pydantic documentation: https://docs.pydantic.dev/latest/ +## V2 Feature Contract (PRP-35 — opt-in) + +Starting with PRP-35 the feature-frame contract is versioned. V1 (the 14-column manifest documented above) remains the default and the back-compat path; V2 is an opt-in richer manifest reachable via `TrainRequest.feature_frame_version=2`. + +**Pinned V2 constants** (`app/shared/feature_frames/contract_v2.py`): +- `EXOGENOUS_LAGS_V2 = (1, 7, 14, 28, 56, 364)` — `lag_364` (not `lag_365`) preserves day-of-week. +- `ROLLING_WINDOWS_V2 = (7, 28, 90)` — leakage-safe via `shift(1).rolling(window)` semantics (`s[i-window..i-1]` for row `i`). +- `TREND_WINDOWS_V2 = (30, 90)` — `numpy.polyfit(deg=1)` slope over the trailing window. +- `HISTORY_TAIL_DAYS_V2 = 400` — comfortably exceeds `lag_364`. + +**Feature groups** (`FeatureGroup` enum) — every V2 column belongs to exactly one group; group enablement decides emission. The default `feature_groups=None` resolves to the MVP-green default: + +| Group | Default | Columns (example) | +|-------|---------|-------------------| +| `target_history` | ✅ | `lag_1`, `lag_7`, …, `lag_364`, `same_dow_mean_4`, `same_dow_mean_8` | +| `calendar` | ✅ | V1 calendar + `week_of_year_sin/cos`, `day_of_month_sin/cos`, `is_holiday` | +| `rolling` | ✅ | `rolling_mean_7/28/90`, `rolling_median_28`, `rolling_std_28` | +| `trend` | ✅ | `trend_30`, `trend_90`, `rolling_mean_7_vs_28`, `rolling_mean_28_vs_prev_28` | +| `price_promo` | ✅ | V1 price + `promo_discount_pct`, `promo_kind_markdown_active`, `promo_kind_bundle_active` | +| `lifecycle` | ✅ | V1 `days_since_launch` + `is_new_product`, `is_mature_product`, `is_discontinued`, `days_until_discontinue` | +| `inventory` | opt-in | `is_stockout_lag1`, `stockout_days_7/28`, `inventory_available_ratio_28` | +| `replenishment` | opt-in | `days_since_last_replenishment`, `replenishment_count_14`, `replenishment_qty_28` | +| `returns` | opt-in | `returns_qty_7/28`, `returns_rate_28` | +| `exogenous_weather` | opt-in | `exo_weather_temp_c`, `exo_weather_precip_mm` | +| `exogenous_macro` | opt-in | `exo_macro_index` | + +**Safety classification** — every V2 column carries a `FeatureSafety` class (`SAFE` / `CONDITIONALLY_SAFE` / `UNSAFE_UNLESS_SUPPLIED`). Persisted into bundle metadata via `v2_feature_safety_classes` so the dashboard can surface the leakage class per column. + +**Leakage spec** — the V2 builders obey the same rule as V1: a horizon day reads only information knowable at the forecast origin `T`. The load-bearing specs are `app/shared/feature_frames/tests/test_leakage_v2.py` (cross-cutting) and `app/features/forecasting/tests/test_regression_features_v2_leakage.py` (slice-layer). Both must stay green; neither may be weakened. + +**Bundle metadata (additive)** — a V2 bundle's `metadata` dict adds: +- `feature_frame_version: 2` +- `feature_groups: {group_name: [columns]}` +- `feature_safety_classes: {column: safety.value}` +- `feature_pinned_constants: {...}` — reproducibility audit snapshot + +V1 bundles default to `metadata.get("feature_frame_version", 1)` at load; the V1 byte-stable path remains the default code path. + +**Preview script**: `uv run python examples/forecasting/feature_frame_v2_preview.py --store-id 15 --product-id 52 --cutoff-date 2025-12-31` dumps V1 + V2 columns + per-group NaN counts side by side. + diff --git a/examples/forecasting/feature_frame_v2_preview.py b/examples/forecasting/feature_frame_v2_preview.py new file mode 100644 index 00000000..a98fd028 --- /dev/null +++ b/examples/forecasting/feature_frame_v2_preview.py @@ -0,0 +1,111 @@ +"""V1 vs V2 feature-frame preview (PRP-35). + +Read-only diagnostic — dumps the V1 and V2 feature-column lists side by side +plus the first three rows of each matrix for a given ``(store_id, product_id, +cutoff_date)``. Also prints per-group NaN counts in the V2 matrix so a +developer can spot when a smaller seeded DB lacks the source data for a +specific opt-in group. + +Local-development only — no network egress, no DB writes. Requires +``docker compose up -d`` for the local Postgres. + +Usage: + uv run python examples/forecasting/feature_frame_v2_preview.py \\ + --store-id 15 --product-id 52 --cutoff-date 2025-12-31 \\ + [--groups target_history,calendar,rolling] +""" + +from __future__ import annotations + +import argparse +import asyncio +import math +from datetime import date as date_type + +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.config import get_settings +from app.features.forecasting.service import ( + ForecastingService, + _resolve_feature_groups, +) +from app.shared.feature_frames import FeatureGroup + + +async def _run(args: argparse.Namespace) -> None: + settings = get_settings() + engine = create_async_engine(settings.database_url, echo=False) + session_maker = async_sessionmaker(engine, expire_on_commit=False) + + service = ForecastingService() + start_date = date_type.fromisoformat(args.start_date) + end_date = date_type.fromisoformat(args.cutoff_date) + + groups_input = args.groups.split(",") if args.groups else None + resolved_groups: tuple[FeatureGroup, ...] = ( + _resolve_feature_groups(groups_input) if groups_input is not None else () + ) + + async with session_maker() as session: + try: + v1 = await service._build_regression_features( + db=session, + store_id=args.store_id, + product_id=args.product_id, + start_date=start_date, + end_date=end_date, + ) + print(f"V1 — {len(v1.feature_columns)} columns:") + print(" " + ", ".join(v1.feature_columns)) + print("V1 — first 3 rows:") + for row in v1.X[:3]: + print(" " + ", ".join(f"{v:.3f}" if not math.isnan(v) else "nan" for v in row)) + print() + except ValueError as exc: + print(f"V1 build skipped: {exc}") + + try: + v2 = await service._build_regression_features_v2( + db=session, + store_id=args.store_id, + product_id=args.product_id, + start_date=start_date, + end_date=end_date, + groups=resolved_groups if resolved_groups else (FeatureGroup.TARGET_HISTORY,), + ) + print(f"V2 — {len(v2.feature_columns)} columns:") + print(" " + ", ".join(v2.feature_columns)) + print("V2 — first 3 rows:") + for row in v2.X[:3]: + print(" " + ", ".join(f"{v:.3f}" if not math.isnan(v) else "nan" for v in row)) + print() + # Per-group NaN counts + print("V2 — NaN counts per column:") + for i, name in enumerate(v2.feature_columns): + nan_count = int(sum(1 for row in v2.X if math.isnan(row[i]))) + if nan_count: + print(f" {name}: {nan_count}/{len(v2.X)}") + except ValueError as exc: + print(f"V2 build skipped: {exc}") + + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="V1 vs V2 feature-frame preview") + parser.add_argument("--store-id", type=int, required=True) + parser.add_argument("--product-id", type=int, required=True) + parser.add_argument("--start-date", type=str, default="2025-01-01") + parser.add_argument("--cutoff-date", type=str, required=True) + parser.add_argument( + "--groups", + type=str, + default=None, + help="Comma-separated FeatureGroup names; default → DEFAULT_V2_GROUPS", + ) + args = parser.parse_args() + asyncio.run(_run(args)) + + +if __name__ == "__main__": + main()