Support customer rates from database

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "skypro"
-version = "1.0.0"
+version = "1.1.0"
 description = "Skyprospector by Cepro"
 authors = ["damonrand <damon@cepro.energy>"]
 license = "AGPL-3.0"

src/skypro/commands/report/main.py

@@ -296,11 +296,19 @@ def report(
         time_index=time_index,
         rates_by_category=rates.mkt_fix,
         allow_vol_rates=False,
+        allow_fix_rates=True,
     )
-    customer_fixed_cost_dfs, customer_vol_rates_dfs = get_rates_dfs_by_type(
+    _, customer_vol_rates_dfs = get_rates_dfs_by_type(
         time_index=time_index,
-        rates_by_category=rates.customer,
+        rates_by_category=rates.customer_vol,
         allow_vol_rates=True,
+        allow_fix_rates=False,
+    )
+    customer_fixed_cost_dfs, _ = get_rates_dfs_by_type(
+        time_index=time_index,
+        rates_by_category=rates.customer_fix,
+        allow_vol_rates=False,
+        allow_fix_rates=True
     )
 
     return Report(

src/skypro/commands/report/microgrid_flow_calcs.py

@@ -156,8 +156,7 @@ def calculate_missing_net_flows_in_junction(
             pct_missing = (nans_df[col_to_predict].sum() / len(nans_df)) * 100
             df.loc[rows_to_predict, col_to_predict] = total * col_to_predict_direction
             notices.append(Notice(
-                detail=f"{pct_missing:.1f}% of '{col_to_predict}' fields are missing, but {pct_to_fill:.1f}% can be calculated"
-                    " using redundant microgrid metering data",
+                detail=f"{pct_missing:.1f}% of '{col_to_predict}' fields are missing, but {pct_to_fill:.1f}% can be calculated using redundant microgrid metering data",
                 level=pct_to_notice_level(pct_missing)
             ))
 

src/skypro/commands/report/rates.py

@@ -8,7 +8,7 @@
 from skypro.common.data.get_timeseries import get_timeseries
 from skypro.common.notice.notice import Notice
 from skypro.common.rate_utils.to_dfs import VolRatesForEnergyFlows
-from skypro.common.rates.rates import FixedRate, Rate
+from skypro.common.rates.rates import FixedRate, VolRate
 from skypro.common.timeutils.timeseries import get_steps_per_hh, get_step_size
 
 from skypro.commands.report.config.config import Config
@@ -21,8 +21,9 @@ class ParsedRates:
     This is just a container to hold the various rate objects
     """
     mkt_vol: VolRatesForEnergyFlows = field(default_factory=VolRatesForEnergyFlows)   # Volume-based (p/kWh) market rates for each energy flow, as predicted in real-time
-    mkt_fix: Dict[str, List[FixedRate]] = field(default_factory=dict)  # Fixed p/day rates associated with market/suppliers, keyed by user-specified string which can be used to categorise
-    customer: Dict[str, List[Rate]] = field(default_factory=dict)  # Volume and fixed rates charged to customers, keyed by user-specified string which can be used to categorise
+    mkt_fix: Dict[str, List[FixedRate]] = field(default_factory=dict)  # Fixed p/day rates associated with market/suppliers, keyed by a string which can be used to categorise
+    customer_vol: Dict[str, List[VolRate]] = field(default_factory=dict)  # Volume rates charged to customers, keyed by a string which can be used to categorise
+    customer_fix: Dict[str, List[FixedRate]] = field(default_factory=dict)  # Fixed rates charged to customers, keyed by a string which can be used to categorise
 
 
 def get_rates_from_config(
@@ -62,27 +63,37 @@ def get_rates_from_config(
     notices.extend(missing_data_warnings(imbalance_pricing, "Elexon imbalance data"))
 
     # Rates can either be read from the "rates database" or from local YAML files
-    if config.reporting.rates.rates_db is not None:
-        mkt_vol, fixed_import, fixed_export = get_rates_from_db(
-            supply_points_name=config.reporting.rates.rates_db.supply_points_name,
-            site_region=config.reporting.rates.rates_db.site_specific.region,
-            site_bands=config.reporting.rates.rates_db.site_specific.bands,
-            import_bundle_names=config.reporting.rates.rates_db.import_bundles,
-            export_bundle_names=config.reporting.rates.rates_db.export_bundles,
+    db_config = config.reporting.rates.rates_db
+    if db_config is not None:
+        db_rates = get_rates_from_db(
+            supply_points_name=db_config.supply_points_name,
+            site_region=db_config.site_specific.region,
+            site_bands=db_config.site_specific.bands,
+            import_bundle_names=db_config.import_bundles,
+            export_bundle_names=db_config.export_bundles,
             db_engine=rates_db_engine,
             imbalance_pricing=imbalance_pricing["imbalance_price"],
             import_grid_capacity=config.reporting.grid_connection.import_capacity,
             export_grid_capacity=config.reporting.grid_connection.export_capacity,
-            future_offset=timedelta(seconds=0)
+            future_offset=timedelta(seconds=0),
+            customer_import_bundle_names=db_config.customer.import_bundles if db_config.customer is not None else [],
+            customer_export_bundle_names=db_config.customer.export_bundles if db_config.customer is not None else [],
         )
 
         parsed_rates = ParsedRates(
-            mkt_vol=mkt_vol,
+            mkt_vol=db_rates.mkt_vol_by_flow,
             mkt_fix={
-                "import": fixed_import,
-                "export": fixed_export
+                "import": db_rates.mkt_fix_import,
+                "export": db_rates.mkt_fix_export,
+            },
+            customer_vol={
+                "import": db_rates.customer_vol_import,
+                "export": db_rates.customer_vol_export,
+            },
+            customer_fix={
+                "import": db_rates.customer_fix_import,
+                "export": db_rates.customer_fix_export,
             },
-            customer={}  # TODO: read customer rates from DB
         )
     else:  # Read rates from local YAML files...
         # Parse the supply points config file:
@@ -117,11 +128,20 @@ def get_rates_from_config(
 
             if exp_config.customer_load_files:
                 for category_str, files in exp_config.customer_load_files.items():
-                    parsed_rates.customer[category_str] = parse_rate_files(
+                    rates = parse_rate_files(
                         files=files,
                         supply_points=supply_points,
                         imbalance_pricing=None,
                         file_path_resolver_func=file_path_resolver_func
                     )
+                    parsed_rates.customer_fix[category_str] = []
+                    parsed_rates.customer_vol[category_str] = []
+                    for rate in rates:
+                        if isinstance(rate, FixedRate):
+                            parsed_rates.customer_fix[category_str].append(rate)
+                        elif isinstance(rate, VolRate):
+                            parsed_rates.customer_vol[category_str].append(rate)
+                        else:
+                            raise ValueError(f"Unknown rate type: {rate}")
 
     return parsed_rates, notices

src/skypro/commands/report/warnings.py

@@ -15,8 +15,8 @@ def missing_data_warnings(df: pd.DataFrame, data_name: str) -> List[Notice]:
         return [
              Notice(
                 detail=f"{pct:.1f}% of '{data_name}' data is missing ({num_missing} NaN fields)",
-                level=pct_to_notice_level(pct)
-            )
+                level=pct_to_notice_level(pct),
+             )
         ]
 
     return []

src/skypro/commands/simulator/main.py

@@ -21,7 +21,7 @@
 
 from skypro.common.rate_utils.to_dfs import get_vol_rates_dfs, get_rates_dfs_by_type, VolRatesForEnergyFlows
 from skypro.common.rate_utils.osam import calculate_osam_ncsp
-from skypro.common.rates.rates import FixedRate, Rate, OSAMFlatVolRate
+from skypro.common.rates.rates import FixedRate, VolRate, OSAMFlatVolRate
 from skypro.common.rate_utils.friendly_summary import get_friendly_rates_summary
 from skypro.common.timeutils.math import floor_hh
 from skypro.common.timeutils.timeseries import get_step_size
@@ -51,8 +51,9 @@ class ParsedRates:
     """
     live_mkt_vol: VolRatesForEnergyFlows = field(default_factory=VolRatesForEnergyFlows)   # Volume-based (p/kWh) market/supplier rates for each energy flow, as predicted in real-time
     final_mkt_vol: VolRatesForEnergyFlows = field(default_factory=VolRatesForEnergyFlows)  # Volume-based (p/kWh) market/supplier rates for each energy flow
-    mkt_fix: Dict[str, List[FixedRate]] = field(default_factory=dict)   # Fixed p/day rates associated with market/suppliers
-    customer: Dict[str, List[Rate]] = field(default_factory=dict)  # Volume and fixed rates charged to customers, in string categories
+    final_mkt_fix: Dict[str, List[FixedRate]] = field(default_factory=dict)   # Fixed p/day rates associated with market/suppliers
+    final_customer_vol: Dict[str, List[VolRate]] = field(default_factory=dict)  # Volume rates charged to customers, in string categories
+    final_customer_fix: Dict[str, List[FixedRate]] = field(default_factory=dict)  # Fixed rates charged to customers, in string categories
 
 
 def simulate(
@@ -197,13 +198,21 @@ def _run_one_simulation(
     # not affect the algorithm, but which are passed through into the output CSV
     mkt_fixed_cost_dfs, _ = get_rates_dfs_by_type(
         time_index=time_index,
-        rates_by_category=rates.mkt_fix,
+        rates_by_category=rates.final_mkt_fix,
         allow_vol_rates=False,
+        allow_fix_rates=True,
     )
-    customer_fixed_cost_dfs, customer_vol_rates_dfs = get_rates_dfs_by_type(
+    _, customer_vol_rates_dfs = get_rates_dfs_by_type(
         time_index=time_index,
-        rates_by_category=rates.customer,
+        rates_by_category=rates.final_customer_vol,
         allow_vol_rates=True,
+        allow_fix_rates=False,
+    )
+    customer_fix_costs_dfs, _ = get_rates_dfs_by_type(
+        time_index=time_index,
+        rates_by_category=rates.final_customer_fix,
+        allow_vol_rates=False,
+        allow_fix_rates=True,
     )
 
     # Generate an output file if configured to do so
@@ -217,7 +226,7 @@ def _run_one_simulation(
             int_live_vol_rates_dfs=None,  # These 'live' rates aren't available in the output CSV at the moment as they are
             mkt_live_vol_rates_dfs=None,  # calculated by the price curve algo internally and not returned
             mkt_fixed_costs_dfs=mkt_fixed_cost_dfs,
-            customer_fixed_cost_dfs=customer_fixed_cost_dfs,
+            customer_fixed_cost_dfs=customer_fix_costs_dfs,
             customer_vol_rates_dfs=customer_vol_rates_dfs,
             load_energy_breakdown_df=load_energy_breakdown_df,
             aggregate_timebase=simulation_output_config.aggregate,
@@ -255,7 +264,7 @@ def _run_one_simulation(
         int_live_vol_rates_dfs=None,  # These 'live' rates aren't available in the output CSV at the moment as they are
         mkt_live_vol_rates_dfs=None,  # calculated by the price curve algo internally and not returned
         mkt_fixed_costs_dfs=mkt_fixed_cost_dfs,
-        customer_fixed_cost_dfs=customer_fixed_cost_dfs,
+        customer_fixed_cost_dfs=customer_fix_costs_dfs,
         customer_vol_rates_dfs=customer_vol_rates_dfs,
         load_energy_breakdown_df=load_energy_breakdown_df,
         aggregate_timebase="all",
@@ -511,6 +520,8 @@ def read_imbalance_data(source: TimeseriesDataSource, context: str):
             import_grid_capacity=0,
             export_grid_capacity=0,
             future_offset=time_offset_str_to_timedelta(rates_config.live.rates_db.future_offset_str),
+            customer_import_bundle_names=[],
+            customer_export_bundle_names=[],
         )
         parsed_rates.final_mkt_vol, _, _ = get_rates_from_db(
             supply_points_name=rates_config.final.rates_db.supply_points_name,
@@ -523,8 +534,9 @@ def read_imbalance_data(source: TimeseriesDataSource, context: str):
             import_grid_capacity=0,
             export_grid_capacity=0,
             future_offset=time_offset_str_to_timedelta(rates_config.final.rates_db.future_offset_str),
+            customer_import_bundle_names=rates_config.final.rates_db.customer.import_bundles if rates_config.final.rates_db.customer is not None else [],
+            customer_export_bundle_names=rates_config.final.rates_db.customer.export_bundles if rates_config.final.rates_db.customer is not None else [],
         )
-        # TODO: support fixed and customer costs when reading from the rates DB
 
     else:   # Read rates from local YAML files...
         final_supply_points = parse_supply_points(
@@ -560,16 +572,25 @@ def read_imbalance_data(source: TimeseriesDataSource, context: str):
                     for rate in rates:
                         if not isinstance(rate, FixedRate):
                             raise ValueError(f"Only fixed rates can be specified in the fixedMarketFiles, got: '{rate.name}'")
-                    parsed_rates.mkt_fix[category_str] = cast(List[FixedRate], rates)
+                    parsed_rates.final_mkt_fix[category_str] = cast(List[FixedRate], rates)
 
             if rates_config.final.experimental.customer_load_files:
                 for category_str, files in rates_config.final.experimental.customer_load_files.items():
-                    parsed_rates.customer[category_str] = parse_rate_files(
+                    rates = parse_rate_files(
                         files=files,
                         supply_points=final_supply_points,
                         imbalance_pricing=None,
                         file_path_resolver_func=file_path_resolver_func
                     )
+                    parsed_rates.final_customer_fix[category_str] = []
+                    parsed_rates.final_customer_vol[category_str] = []
+                    for rate in rates:
+                        if isinstance(rate, FixedRate):
+                            parsed_rates.final_customer_fix[category_str].append(rate)
+                        elif isinstance(rate, VolRate):
+                            parsed_rates.final_customer_vol[category_str].append(rate)
+                        else:
+                            raise ValueError(f"Unknown rate type: {rate}")
 
     return parsed_rates, df
 

src/skypro/common/config/data_source.py

@@ -89,7 +89,7 @@ class ProfileDataSource:
     constant_profile_data_source: Optional[ConstantProfileDataSource] = field_with_opts(key="constant")
 
     def __post_init__(self):
-        enforce_one_option([self.csv_profile_data_source, self.constant_profile_data_source],"'csvProfile', 'constant'")
+        enforce_one_option([self.csv_profile_data_source, self.constant_profile_data_source], "'csvProfile', 'constant'")
 
 
 @dataclass

src/skypro/common/config/rates_dataclasses.py

@@ -19,6 +19,15 @@ class SiteSpecifier:
     bands: List[str]
 
 
+@dataclass
+class CustomerRatesDB:
+    """
+    Configures rates for customers (i.e. domestic homes) to be pulled from a database
+    """
+    import_bundles: List[str] = field_with_opts(key="importBundles")  # Names of any import rate bundles to use for the customer load
+    export_bundles: List[str] = field_with_opts(key="exportBundles")  # Names of any export rate bundles to use for the customer export
+
+
 @dataclass
 class RatesDB:
     """
@@ -29,6 +38,7 @@ class RatesDB:
     import_bundles: List[str] = field_with_opts(key="importBundles")  # Names of any import rate bundles to use in addition to the site specific ones (e.g. Supplier arrangements)
     export_bundles: List[str] = field_with_opts(key="exportBundles")  # Names of any export rate bundles to use in addition to the site specific ones (e.g. Supplier arrangements).
     future_offset_str: Optional[str] = field_with_opts(key="futureOffset")  # For simulations, it can be useful to bring the rates forwards in time, for example we might want to use the 2025 rates for a simulation run over 2024
+    customer: Optional[CustomerRatesDB]  # Optionally define rates for customers - these are only really used for reporting purposes as this doesn't affect control algorithms
 
 
 @dataclass