From fba0f474a6928cabb6f4ad64ab43b99dc33bc260 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 7 May 2026 07:44:22 +0000 Subject: [PATCH] =?UTF-8?q?docs:=20deep=20code=20review=20=E2=80=94=20fix?= =?UTF-8?q?=20drift=20string=20bug=20and=20update=20all=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-Logs-Url: https://github.com/rpwalsh/DispatchLayer/sessions/1b670092-cad4-4684-af7e-9a733f2e2894 Co-authored-by: rpwalsh <10300352+rpwalsh@users.noreply.github.com> --- API.md | 200 +++++++++++++++--- ARCHITECTURE.md | 33 ++- DOMAIN_MODEL.md | 197 ++++++++++++++++- PYTHON_DEEPDIVE.md | 100 +++++++-- QUICKSTART.md | 71 +++++-- docs/mathematics/predictive-math-core.md | 99 ++++++++- .../structural_drift.py | 4 +- 7 files changed, 617 insertions(+), 87 deletions(-) diff --git a/API.md b/API.md index c7f0c25..bd164ed 100644 --- a/API.md +++ b/API.md @@ -2,60 +2,196 @@ Base URL: /api/v1 -This reference focuses on the endpoints used in the aligned forecasting workflow and dashboard. +This reference covers all registered endpoints. -## Core Read Endpoints +## Health + +- GET /health (root-level alias) +- GET /api/v1/health + +Both return `{"status": "ok", "service": "dispatchlayer-api", "version": "0.1.0"}`. + +## Overview -- GET /health - GET /overview/source-summary -- GET /providers -- GET /providers/health + +Returns the dataset catalog, site list, coverage dates, total hourly points, and +a `power_data_status` block describing what measured vs. modeled data is available. + +## Timeseries + - GET /timeseries/{site_id}?hours={1..43800} + +Returns archive rows and `hourly_units` for a site. All values are Open-Meteo +ERA5/ECMWF reanalysis data. No synthetic or interpolated values are added. + +Common fields per row: + +- temperature_2m +- wind_speed_10m, wind_speed_80m, wind_speed_120m +- wind_direction_10m, wind_gusts_10m +- shortwave_radiation, direct_normal_irradiance, diffuse_radiation +- cloud_cover, cloud_cover_low, cloud_cover_mid, cloud_cover_high +- relative_humidity_2m, precipitation, pressure_msl, vapour_pressure_deficit + +## Sites + +- POST /sites/evaluate + + Live evaluation using real provider calls (Open-Meteo). Returns forecast context, + data-quality confidence, structural drift warning, and audit trace. Every response + includes a `sources` block with attribution, freshness, and degraded-mode warnings. + - GET /sites/{site_id}/pipeline?history_hours={24..43800}&horizon_hours={24..} -- GET /assets/{asset_id}/health -## Core Write/Compute Endpoints + Archive-backed pipeline for a known demo site. Returns: + + - current_signals and normalized_signals + - residuals and structural_drift + - projection array with ts, p10, p50, p90 + - recommendations and audit_trace + +## Telemetry + +- POST /telemetry/ingest + + Ingest raw TelemetryPoint records. In the public demo, writes to an in-process + list (not persisted across restarts). + +- POST /telemetry/normalize + + Normalise raw TelemetryPoint records into AssetTelemetrySnapshot summaries grouped + by (site_id, asset_id, asset_type). + +- GET /sites/{site_id}/telemetry/latest?data_mode={source|live} + + Latest AssetTelemetrySnapshot per asset at the site. + `data_mode=source` loads from `data/source_snapshots/*.json`. + `data_mode=live` returns from the in-process ingest store. + +- GET /assets/{asset_id}/health?data_mode={source|live} + + Health summary for a single asset including snapshot and anomaly residual. + +## Forecasts - POST /forecasts/site + + Single-site point forecast with P10/P50/P90 bounds and decision trace. + Supports `asset_type`: `wind_turbine` (requires `wind_speed_mps`) or + `solar_inverter` (requires `ghi_wm2`). + - POST /forecasts/portfolio + + Aggregates site forecasts into portfolio P10/P50/P90 in MWh using + quadrature-sum spread combination across sites. + +## Dispatch + - POST /dispatch/optimize -- POST /sites/evaluate -- POST /signals/evaluate + + Battery dispatch decision (charge / discharge / hold) based on SoC, + price signals, solar generation forecast, and demand forecast. + Returns action, target SoC, estimated revenue, cycle cost, and decision trace. + +## Anomalies + - POST /anomalies/detect -## Forecast Page Data Dependencies + Detect generation anomaly for a single asset. Returns deviation flag, + condition, residual percentage, causal hypotheses, and decision trace. -The dashboard Forecast page primarily uses: +- GET /anomalies/conditions -1. GET /overview/source-summary -2. GET /timeseries/{site_id} -3. GET /sites/{site_id}/pipeline + List all supported AnomalyCondition enum values. -## Timeseries Contract Notes +## Signals -- rows are hourly samples with weather/resource signals -- hourly_units provides per-field units for display -- expected max demo window is 43800 hours (5 years) +- POST /signals/evaluate -Common fields: + Multi-asset signal evaluation. Runs anomaly detection across a list of assets + and returns ranked SignalEvent records with threshold state and audit hash. -- temperature_2m -- wind_speed_10m, wind_speed_80m, wind_speed_120m -- wind_direction_10m, wind_gusts_10m -- shortwave_radiation, direct_normal_irradiance, diffuse_radiation -- cloud_cover, cloud_cover_low, cloud_cover_mid, cloud_cover_high -- relative_humidity_2m, precipitation, pressure_msl +- GET /signals/states + + List all ThresholdState enum values. + +## Predictive (prefix: /predictive) + +- POST /predictive/signal-state + + Normalise a set of named signals. Returns per-signal confidence, missing flag, + and anomalous flag after range validation. + +- POST /predictive/residual + + Compute absolute delta, percent delta, direction, and significance for an + expected vs. actual value pair. + +- POST /predictive/forecast-bounds + + Compute P10/P50/P90 bounds from a point forecast and optional historical error + distribution. P10/P90 use ±1.28σ around P50. + +- POST /predictive/reconcile + + Apply historical bias correction and optional telemetry adjustment to a raw + forecast. Returns adjusted forecast and confidence. + +- POST /predictive/causal-attribution + + Attribute underproduction to ranked causal hypotheses for `wind_turbine` or + `solar_inverter` assets using evidence graphs. + +- POST /predictive/confidence + + Aggregate evidence-node weights into a hypothesis confidence score. -## Pipeline Contract Notes +## Providers -Pipeline response contains: +- GET /providers + + List all configured providers and whether their API key is set. + +- GET /providers/health + + Live reachability probe for key-free providers (Open-Meteo, NASA POWER, + NOAA/NWS). Key-gated providers (NREL, EIA, ENTSO-E) report `unconfigured` + when no key is set. + +## Ingest + +- POST /ingest/weather + + Fetch a weather forecast window from a named provider (currently supports + `open_meteo`). Returns timestamped samples. + +## Connectors + +- GET /connectors/state + + Current state of all platform connectors: OTEL, OPC UA, MQTT, AWS SiteWise, + S3/Parquet. Each connector probe runs at call time with no fixture mode. + Returns state, sample count, and error type on failure. -- current_signals and normalized_signals -- residuals and structural_drift -- projection array with ts, p10, p50, p90 -- recommendations and audit_trace +- GET /connectors/protocols -The Forecast Bands table is derived directly from projection values and latest timeseries row. + List all supported connector protocols with purpose and read-only flag. + +## Audit + +- GET /audit/trace/{trace_id} + + Placeholder. Traces are currently embedded inline in each API response. + Persistent trace lookup requires a database backend. + +## Forecast Page Data Dependencies + +The dashboard Forecast page primarily uses: + +1. GET /overview/source-summary +2. GET /timeseries/{site_id} +3. GET /sites/{site_id}/pipeline ## Validation Semantics (Dashboard) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a0f7d29..7c06a05 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -5,12 +5,15 @@ DispatchLayer is organized as an API plus dashboard around a shared predictive c ## High-Level Components 1. Data layer - - Archive weather/resource snapshots (hourly) - - Site catalog with 10 demo sites + - Archive weather/resource snapshots (hourly, Open-Meteo ERA5/ECMWF) + - Site catalog with 10 demo sites (5 solar, 5 wind) + - Source snapshots under `data/source_snapshots/` 2. API layer (apps/api) - - FastAPI routes for summary, timeseries, pipeline, forecasting, dispatch + - FastAPI routes for summary, timeseries, pipeline, forecasting, dispatch, + anomalies, signals, connectors, audit, and predictive primitives - Request-level composition of forecast, uncertainty, and trace artifacts + - Auto-generated docs at `/docs` (Swagger) and `/redoc` 3. Dashboard layer (apps/dashboard) - Vue page flows for validation, forecast, and diagnostics @@ -18,6 +21,14 @@ DispatchLayer is organized as an API plus dashboard around a shared predictive c 4. Package layer (packages/*) - Domain entities, predictive math, dispatch logic, adapters/connectors + - See PYTHON_DEEPDIVE.md for full topology + +5. Connector layer (packages/connectors/*) + - Read-only industrial-protocol connectors + - OpenTelemetry/OTLP, OPC UA/SCADA, MQTT, AWS IoT SiteWise, S3/Parquet + - Exposed via `GET /api/v1/connectors/state` and `/connectors/protocols` + - All connectors fail soft: runtime errors are caught and reported as + connector state `"ERROR"` without crashing the API ## Forecast Data Flow @@ -51,4 +62,20 @@ DispatchLayer is organized as an API plus dashboard around a shared predictive c - Pipeline includes audit_trace and recommendation evidence. - Dashboard exposes forecast bands plus input state and spectral signal tables. +- Each API response embeds a DecisionTrace with trace_id, step inputs/outputs, + reasoning strings, and model version tags. + +## Makefile Quick Reference + +| Target | Description | +|---------------------|--------------------------------------------------------| +| `make install` | Install all Python packages (editable) + npm install | +| `make api` | Start API on port 8000 | +| `make dashboard` | Start Vite dev server on port 3000 | +| `make test` | Run all Python tests | +| `make lint-language`| Check for forbidden instrumentation-boundary terms | +| `make frontend` | Production build of the dashboard | +| `make verify` | Full check: tests + lint + frontend build | +| `make docker` | Build and run via docker compose | +| `make snapshots-recommended` | Capture 5-year archive snapshots for all demo sites | diff --git a/DOMAIN_MODEL.md b/DOMAIN_MODEL.md index 4b653ce..6b3ca92 100644 --- a/DOMAIN_MODEL.md +++ b/DOMAIN_MODEL.md @@ -2,9 +2,131 @@ This model describes the core concepts used by API and dashboard. -## Primary Entities +--- -## Site +## Core Value Types (`packages/domain/models.py`) + +### GeoPoint + +- latitude: float +- longitude: float + +### ForecastWindow + +- start_utc: datetime (UTC) +- end_utc: datetime (UTC) +- resolution_minutes: int + +--- + +## Asset Types + +### AssetType (enum) + +Values: `wind_turbine`, `solar_inverter`, `battery`, `meter` + +--- + +## Weather and Resource Samples + +### WeatherSample + +- timestamp_utc: datetime (UTC) +- temperature_c: Optional[float] +- wind_speed_mps: Optional[float] +- wind_direction_deg: Optional[float] +- cloud_cover_pct: Optional[float] +- shortwave_radiation_wm2: Optional[float] +- direct_radiation_wm2: Optional[float] +- diffuse_radiation_wm2: Optional[float] +- source: str + +### SolarResourceSample + +- timestamp_utc: datetime +- ghi_wm2: Optional[float] +- dni_wm2: Optional[float] +- dhi_wm2: Optional[float] +- temperature_c: Optional[float] +- source: str + +### GridDemandSample + +- timestamp_utc: datetime +- balancing_authority: str +- demand_mw: Optional[float] +- demand_forecast_mw: Optional[float] +- net_generation_mw: Optional[float] +- source: str + +### MarketSignal + +- timestamp_utc: datetime +- region: str +- price_per_mwh: Optional[float] +- source: str + +--- + +## Asset Telemetry + +### AssetTelemetry + +- timestamp_utc: datetime +- asset_id: str +- site_id: str +- asset_type: AssetType +- output_kw: Optional[float] +- capacity_kw: float +- curtailment_flag: bool +- auxiliary_data: dict + +--- + +## Aggregate / Forecast Types + +### WeatherForecast + +- location: GeoPoint +- window: ForecastWindow +- samples: tuple[WeatherSample, ...] +- source: str + +### SolarResource + +- location: GeoPoint +- window: ForecastWindow +- samples: tuple[SolarResourceSample, ...] +- source: str + +### GridDemand + +- balancing_authority: str +- window: ForecastWindow +- samples: tuple[GridDemandSample, ...] +- source: str + +### GenerationMix + +- timestamp_utc: datetime +- balancing_authority: str +- wind_mw, solar_mw, natural_gas_mw, coal_mw, nuclear_mw, hydro_mw, other_mw: Optional[float] +- source: str + +### PortfolioSummary + +- portfolio_id: str +- window: ForecastWindow +- expected_generation_mwh: float +- p10_generation_mwh, p50_generation_mwh, p90_generation_mwh: float +- primary_constraints: tuple[str, ...] +- forecast_basis: tuple[str, ...] + +--- + +## Dashboard-Facing Forecast Entities + +### Site (catalog record) - site_id - name @@ -12,26 +134,26 @@ This model describes the core concepts used by API and dashboard. - region, latitude, longitude - capacity_mw -## TimeseriesSample +### TimeseriesSample - ts (UTC) - weather/resource signals (temperature, wind, irradiance, cloud, humidity, precipitation, pressure) - unit metadata via hourly_units -## ModeledOutput +### ModeledOutput - observed_mw (weather-to-power estimate for dashboard analysis) - expected baseline per hour - residual = observed - expected -## ProjectionPoint +### ProjectionPoint - ts - p10 - p50 - p90 -## HoldoutEvaluation +### HoldoutEvaluation - holdout_year (forced to 2025 when available) - training_months and holdout_months @@ -39,7 +161,7 @@ This model describes the core concepts used by API and dashboard. - error_pct - hit flag (true when error <= 6%) -## SpectralSignal +### SpectralSignal - label - period_h @@ -47,7 +169,7 @@ This model describes the core concepts used by API and dashboard. - variance_share_pct - interpretation -## PipelineArtifact +### PipelineArtifact - current_signals - normalized_signals @@ -56,6 +178,63 @@ This model describes the core concepts used by API and dashboard. - recommendations - audit_trace +--- + +## Telemetry Models (`packages/domain/telemetry.py`) + +### Quality (enum) + +Values: `GOOD`, `UNCERTAIN`, `BAD`, `MISSING`, `STALE` + +Maps OPC UA / IEC quality codes to a canonical five-state enum. + +### TelemetrySample + +Unified connector output (timestamp-quality-value). All connector adapters +produce this type. + +- source_id: str +- channel_id: str +- asset_id: Optional[str] +- timestamp_utc: datetime +- value: float | str | bool | None +- unit: Optional[str] +- quality: Quality +- source_timestamp_utc: Optional[datetime] +- ingest_timestamp_utc: datetime +- tags: dict[str, str] +- audit_hash: str (auto-computed SHA-256 truncated to 16 hex chars) + +### TelemetryPoint + +Single timestamped, typed signal value as received from an asset. + +- timestamp_utc: datetime +- site_id, asset_id: str +- asset_type: `solar_inverter` | `wind_turbine` | `bess` | `meter` | `weather_station` +- signal: str (field name) +- value: float | str | bool | None +- unit: Optional[str] +- quality: `good` | `suspect` | `bad` | `missing` +- source: str + +### AssetTelemetrySnapshot + +Normalised per-asset operational snapshot. Combines generation, health, and +asset-type-specific signals. All signal fields are Optional so a snapshot can +be partially populated. + +- timestamp_utc, site_id, asset_id, asset_type: str +- capacity_kw: float +- power_kw, expected_power_kw, availability_pct: Optional[float] +- Solar inverter: dc_voltage_v, dc_current_a, ac_power_kw, inverter_efficiency_pct, frequency_hz, string_current_a +- Wind turbine: rotor_rpm, wind_speed_mps, wind_direction_deg, nacelle_direction_deg, yaw_error_deg, blade_pitch_deg, gearbox_temperature_c, generator_temperature_c, vibration_mm_s +- BESS: state_of_charge_pct, state_of_health_pct, charge_power_kw, discharge_power_kw, cell_temperature_c, pack_voltage_v, pack_current_a, cycle_count, thermal_derate_flag, inverter_status +- Meter/grid: reactive_power_kvar, voltage_v, power_factor, export_limit_kw, curtailment_signal +- Common: temperature_c, fault_code, curtailment_flag, quality_score, data_source + +--- + ## Relationships - Site has many TimeseriesSample @@ -63,4 +242,6 @@ This model describes the core concepts used by API and dashboard. - HoldoutEvaluation is derived from Site monthly aggregates - SpectralSignal is derived from modeled output history - PipelineArtifact links projection and explainability outputs +- AssetTelemetrySnapshot is the normalised form of multiple TelemetryPoint records +- TelemetrySample is the universal connector output used by all industrial-protocol connectors diff --git a/PYTHON_DEEPDIVE.md b/PYTHON_DEEPDIVE.md index 95a951d..3c6a04a 100644 --- a/PYTHON_DEEPDIVE.md +++ b/PYTHON_DEEPDIVE.md @@ -4,29 +4,80 @@ This document summarizes how Python components support the current API and dashb ## Package Topology -- packages/domain: entities, types, value semantics -- packages/forecasting: forecast logic and uncertainty primitives -- packages/predictive: residual, confidence, attribution, reconciliation helpers -- packages/dispatch: dispatch optimization and recommendation logic -- packages/signals: signal-state construction and quality handling -- packages/adapters: provider adapters (open-meteo, nasa-power, noaa, eia, entsoe, nrel) +Core domain and math: + +- packages/domain: entities, types, value semantics, telemetry models +- packages/forecasting: solar irradiance model, wind power curve, portfolio forecast aggregation +- packages/predictive: residual, confidence, attribution, reconciliation, structural drift, L/G/P/D pipeline layers +- packages/dispatch: battery dispatch optimization and recommendation logic +- packages/signals: signal-state construction, quality handling, event ranking +- packages/anomaly: anomaly detection and condition classification +- packages/simulation: Monte Carlo simulation utilities +- packages/recommendations: ranking and recommendation engine + +Provider adapters: + +- packages/adapters/open_meteo: Open-Meteo weather/archive adapter +- packages/adapters/nasa_power: NASA POWER historical irradiance adapter +- packages/adapters/noaa_nws: NOAA/NWS public weather adapter +- packages/adapters/nrel: NREL adapter (API key required) +- packages/adapters/eia: EIA grid data adapter (API key required) +- packages/adapters/entsoe: ENTSO-E European grid adapter (API key required) + +Connector packages (industrial protocol adapters, read-only): + +- packages/connectors/opentelemetry: OTLP collector connector +- packages/connectors/opcua: OPC UA / SCADA connector +- packages/connectors/mqtt: MQTT edge-gateway connector +- packages/connectors/sitewise: AWS IoT SiteWise connector +- packages/connectors/parquet: S3/Parquet historical archive connector ## API Route Responsibilities - routes/telemetry.py - - source summary - - archive-backed site timeseries - - telemetry ingest/normalize and asset health + - source summary (`GET /overview/source-summary`) + - archive-backed site timeseries (`GET /timeseries/{site_id}`) + - telemetry ingest and normalize (`POST /telemetry/ingest`, `POST /telemetry/normalize`) + - site telemetry latest snapshot (`GET /sites/{site_id}/telemetry/latest`) + - asset health (`GET /assets/{asset_id}/health`) - routes/sites.py - - /sites/evaluate - - /sites/{site_id}/pipeline with projection and audit artifacts + - live site evaluation with provider call (`POST /sites/evaluate`) + - archive-backed pipeline with projection and audit artifacts (`GET /sites/{site_id}/pipeline`) - routes/forecasts.py - - site and portfolio forecast endpoints + - single-site forecast (`POST /forecasts/site`) + - portfolio aggregate forecast (`POST /forecasts/portfolio`) - routes/predictive.py - - confidence, reconciliation, causal attribution, residual endpoints + - signal normalization, residual, forecast bounds, reconciliation, + causal attribution, confidence scoring + - all under `/predictive/*` prefix + +- routes/anomalies.py + - anomaly detection (`POST /anomalies/detect`) + - condition list (`GET /anomalies/conditions`) + +- routes/signals.py + - multi-asset signal evaluation (`POST /signals/evaluate`) + - state list (`GET /signals/states`) + +- routes/dispatch.py + - battery dispatch optimization (`POST /dispatch/optimize`) + +- routes/providers.py + - provider list and live health probes (`GET /providers`, `GET /providers/health`) + +- routes/ingest.py + - weather data ingest via provider (`POST /ingest/weather`) + +- routes/connectors.py + - connector state (`GET /connectors/state`) + - protocol list (`GET /connectors/protocols`) + +- routes/audit.py + - trace lookup stub (`GET /audit/trace/{trace_id}`) + - traces are currently embedded inline in each response; persistent storage requires a database backend ## Forecast Workflow in Code @@ -44,13 +95,26 @@ This document summarizes how Python components support the current API and dashb - holdout hit threshold set to 6% - missing projections do not count as hits -## Local Development Notes +## Local Development + +Install all packages and start services via the Makefile: + +```bash +make install # editable pip installs + npm install +make api # uvicorn on port 8000 +make dashboard # Vite dev server on port 3000 +make verify # tests + language lint + frontend build +``` + +To run the API manually, ensure the repo root is on `PYTHONPATH`: -Use repo root as PYTHONPATH when launching API: +```bash +# Linux / macOS +PYTHONPATH=$(pwd) uvicorn apps.api.src.dispatchlayer_api.main:app --port 8000 -```powershell -$env:PYTHONPATH = "c:\Users\react\DispatchLayer" -c:/python314/python.exe -m uvicorn apps.api.src.dispatchlayer_api.main:app --port 8000 +# Windows PowerShell +$env:PYTHONPATH = (Get-Location).Path +python -m uvicorn apps.api.src.dispatchlayer_api.main:app --port 8000 ``` ## Testing Focus diff --git a/QUICKSTART.md b/QUICKSTART.md index 4ac974c..4e4b2a7 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -12,45 +12,75 @@ This quickstart brings up the API and dashboard with the aligned forecast workfl - Python 3.11+ - Node.js 18+ - npm +- make (optional but recommended — wraps all common commands) -## 1) Start API +## 1) Install all packages From repo root: -```powershell -$env:PYTHONPATH = "c:\Users\react\DispatchLayer" -c:/python314/python.exe -m uvicorn apps.api.src.dispatchlayer_api.main:app --port 8000 +```bash +make install +``` + +This installs all Python packages in editable mode and runs `npm install` in +`apps/dashboard`. + +## 2) Start API + +```bash +make api +``` + +This runs: + +```bash +uvicorn dispatchlayer_api.main:app --reload --port 8000 +``` + +The API must be launched from the repo root with the repo root on +`PYTHONPATH`. The `make` target handles this automatically. To run manually: + +```bash +# Linux / macOS +PYTHONPATH=$(pwd) uvicorn apps.api.src.dispatchlayer_api.main:app --port 8000 + +# Windows PowerShell +$env:PYTHONPATH = (Get-Location).Path +python -m uvicorn apps.api.src.dispatchlayer_api.main:app --port 8000 ``` Health check: -```powershell -Invoke-WebRequest "http://127.0.0.1:8000/api/v1/health" -UseBasicParsing +```bash +curl http://localhost:8000/api/v1/health ``` -## 2) Start Dashboard +## 3) Start Dashboard + +```bash +make dashboard +``` -From apps/dashboard: +Or manually from `apps/dashboard`: -```powershell -npm install +```bash npm run dev ``` -Dashboard should run at http://localhost:3000. +Dashboard runs at http://localhost:3000. -## 3) Validate API Capacity +## 4) Validate API Capacity Verify 5-year window support: -```powershell -Invoke-WebRequest "http://127.0.0.1:8000/api/v1/timeseries/solar_albuquerque_1?hours=43800" -UseBasicParsing -Invoke-WebRequest "http://127.0.0.1:8000/api/v1/sites/solar_albuquerque_1/pipeline?history_hours=43800&horizon_hours=168" -UseBasicParsing +```bash +curl "http://localhost:8000/api/v1/timeseries/solar_albuquerque_1?hours=43800" +curl "http://localhost:8000/api/v1/sites/solar_albuquerque_1/pipeline?history_hours=43800&horizon_hours=168" ``` Both should return HTTP 200. -## 4) Validate Forecast Page +## 5) Validate Forecast Page Open http://localhost:3000/forecast and confirm: @@ -63,9 +93,18 @@ Open http://localhost:3000/forecast and confirm: - forecast output table (P10/P50/P90) - identified spectral signals table +## Full Verification Suite + +```bash +make verify +``` + +Runs all Python tests, the language-boundary lint check, and a frontend build. + ## Common Issues - If requests go to port 8001, check dashboard env vars and restart Vite. - If forecast panels are empty, confirm API is running on 8000. - If holdout says no months found, verify history window and source data availability. +- If Python packages are not found, confirm you installed from the repo root (`make install` or `pip install -e packages/... apps/api`). diff --git a/docs/mathematics/predictive-math-core.md b/docs/mathematics/predictive-math-core.md index f512780..760725d 100644 --- a/docs/mathematics/predictive-math-core.md +++ b/docs/mathematics/predictive-math-core.md @@ -1,28 +1,111 @@ # Predictive Math Core -This note summarizes the core computations exposed to the dashboard. +This note summarizes the core computations exposed to the dashboard, with the +actual equations implemented in the codebase. ## 1) Weather-to-Power Modeling A modeled hourly generation series is derived from archive weather/resource fields. +### Solar: PVWatts-style irradiance model (`solar_irradiance_model.py`) + +``` +irradiance_fraction = GHI / 1000 W/m² +cell_temp_C = ambient_temp_C + 30 × irradiance_fraction +temp_derate = max(0.5, 1 + (−0.004) × (cell_temp_C − 25)) +dc_output_kW = capacity_kW × irradiance_fraction × temp_derate +ac_output_kW = max(0, dc_output_kW × (1 − 0.14) / 1.2) +``` + +Constants: temperature coefficient −0.4%/°C, reference 25°C, system losses 14%, +DC/AC ratio 1.2. + +### Wind: cubic polynomial power curve (`wind_power_curve.py`) + +``` +if wind_speed < 3 m/s or ≥ 25 m/s: output = 0 +if wind_speed ≥ 12 m/s: output = rated_capacity_kW +else: + normalized = (wind_speed − 3) / (12 − 3) + output = rated_capacity_kW × normalized³ +``` + +Cut-in 3 m/s, rated 12 m/s, cut-out 25 m/s. +For wind sites, `wind_speed_80m` is used as the primary signal; falls back to +`wind_speed_10m` when unavailable. + ## 2) Holdout Evaluation Monthly totals are split into training and holdout sets. - Holdout year target: 2025 -- Hit condition: relative error <= 6% +- Training: all non-holdout months in the selected history window +- Hit condition: abs(projected − actual) / actual ≤ 6% +- Missing projections are not counted as hits (null, not true) + +## 3) Interval Forecasting — P10/P50/P90 (`forecast_bounds.py`) + +``` +base_spread = stdev(historical_errors) / max(|point_forecast|, 1) × 1.5 + (floor 0.05, cap 0.50; default 0.15 if < 5 errors available) +total_spread = base_spread + Σ(uncertainty_factor × 0.05) +σ = point_forecast × total_spread +p50 = point_forecast +p10 = max(0, p50 − 1.28 × σ) +p90 = p50 + 1.28 × σ +uncertainty_score = min(1.0, total_spread / 0.50) +``` + +1.28σ corresponds to the 10th/90th percentile of a normal distribution. -## 3) Interval Forecasting +## 4) Forecast Reconciliation (`reconciliation.py`) -Projection values are emitted as p10, p50, p90 per timestamp. +``` +bias_correction = −mean(historical_errors) [if ≥ 3 errors] + + raw_forecast × (telemetry_deviation% / 100) × 0.5 +adjusted = raw_forecast + bias_correction +confidence = min(0.5 + 0.1 × evidence_count, 0.95) + (capped further by residual variability when ≥ 10 errors) +``` -## 4) Spectral Analysis +## 5) Spectral Analysis A DFT/FFT-style decomposition ranks dominant periodic components by amplitude. -Variance share is computed from squared amplitudes. +Variance share is computed from squared amplitudes normalized to total power. +The 24-hour harmonic is the expected dominant frequency for solar production. + +## 6) Regime Diagnostics — Structural Drift (`structural_drift.py`) + +Detects when the residual distribution has shifted from its baseline: + +``` +shift_std = |mean(recent_residuals) − mean(baseline_residuals)| / std(baseline_residuals) +drift_magnitude = min(1.0, shift_std / (threshold × 2)) + +if shift_std < 1.0: risk = NONE +elif shift_std < 1.5: risk = LOW (mild drift, monitor) +elif shift_std < 3.0: risk = MEDIUM (regime transitioning) +else: risk = HIGH (significant structural drift) +``` + +Default `drift_threshold_std = 1.5`. Requires ≥ 2 recent residuals and ≥ 4 +baseline residuals to assess. + +## 7) L-Layer: Typed Temporal Signal Scoring (`local_signal_scorer.py`) + +Each interaction between operational entities is scored as: + +``` +score = W_{α,β} × raw_value × exp(−λ_{α,β} × Δt) × data_quality +``` -## 5) Regime Diagnostics +where: +- `W_{α,β}` is the type-pair base weight (0.70–0.95) +- `λ_{α,β}` is the type-pair decay rate (hours⁻¹) +- `Δt` is elapsed time since the observation (hours) -Rolling correlation-derived eigenvalue proxies provide structural drift signals. +Example decay rates: +- Weather → asset: λ = 0.20 (~5h half-life) +- Market price → dispatch: λ = 0.35 (~2.8h half-life) +- Maintenance event → asset: λ = 0.005 (~140h half-life) diff --git a/packages/predictive/src/dispatchlayer_predictive/structural_drift.py b/packages/predictive/src/dispatchlayer_predictive/structural_drift.py index b500ae9..bee0c4e 100644 --- a/packages/predictive/src/dispatchlayer_predictive/structural_drift.py +++ b/packages/predictive/src/dispatchlayer_predictive/structural_drift.py @@ -65,7 +65,7 @@ def detect_residual_drift( Parameters ---------- recent_residuals: - Forecast error history for the most recent window (e.g. last 612 observations). + Forecast error history for the most recent window (e.g. last 6–12 observations). baseline_residuals: Trailing baseline (e.g. last 14 days of forecast errors). drift_threshold_std: @@ -106,7 +106,7 @@ def detect_residual_drift( risk = DriftRisk.MEDIUM reason = f"Residual distribution has shifted ({direction}). Regime may be transitioning." effects = [ - "forecast confidence degraded for next 1218 hours", + "forecast confidence degraded for next 12–18 hours", "dispatch decisions should carry wider uncertainty margin", ] action = "Refresh forecast reconciliation before next dispatch planning cycle."