From 7630c860952d73475f9d9a43bda0b378aefadebf Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 7 Aug 2024 15:56:26 +0200 Subject: [PATCH 01/79] feat: create FillRateBasedControlTUNES which implements an ad-hoc FRBC control type to work only with TUNES RM (for now). Signed-off-by: Victor Garcia Reolid --- README.rst | 6 + setup.cfg | 5 +- src/flexmeasures_client/client.py | 3 +- src/flexmeasures_client/s2/__init__.py | 2 + src/flexmeasures_client/s2/cem.py | 9 +- .../s2/control_types/FRBC/__init__.py | 86 +++- .../s2/control_types/FRBC/frbc_simple.py | 17 - .../s2/control_types/FRBC/frbc_tunes.py | 375 ++++++++++++++++++ .../s2/control_types/translations.py | 193 +++++++++ src/flexmeasures_client/s2/wrapper.py | 13 + tests/conftest.py | 129 +++++- tests/{test_s2_coordinator.py => test_cem.py} | 180 ++++----- tests/test_frbc_tunes.py | 272 +++++++++++++ tests/test_s2_models.py | 31 ++ tests/test_s2_translations.py | 78 ++++ 15 files changed, 1262 insertions(+), 137 deletions(-) create mode 100644 src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py create mode 100644 src/flexmeasures_client/s2/control_types/translations.py create mode 100644 src/flexmeasures_client/s2/wrapper.py rename tests/{test_s2_coordinator.py => test_cem.py} (53%) create mode 100644 tests/test_frbc_tunes.py create mode 100644 tests/test_s2_models.py create mode 100644 tests/test_s2_translations.py diff --git a/README.rst b/README.rst index cc146781..5913504c 100644 --- a/README.rst +++ b/README.rst @@ -129,6 +129,12 @@ The schedule returns a Pandas ``DataFrame`` that can be used to regulate the fle +Development +============== + +If you want to develop this package it's necessary to install testing requirements:: + + pip install -e ".[testing]" .. _pyscaffold-notes: diff --git a/setup.cfg b/setup.cfg index 03e935f2..d5e3fe93 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,12 +49,13 @@ python_requires >= 3.9 # For more information, check out https://semver.org/. install_requires = importlib-metadata; python_version<"3.8" - aiohttp + aiohttp<=3.9.1 pandas pydantic>=1.10.8,<2.0 - s2-python + s2-python @ git+ssh://git@github.com/flexiblepower/s2-python@victor async_timeout + [options.packages.find] where = src exclude = diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index c54cf8a6..f8875ed0 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -55,9 +55,10 @@ class FlexMeasuresClient: polling_timeout: float = POLLING_TIMEOUT # seconds request_timeout: float = REQUEST_TIMEOUT # seconds polling_interval: float = POLLING_INTERVAL # seconds - session: ClientSession = ClientSession() def __post_init__(self): + self.session: ClientSession = ClientSession() + if not re.match(r".+\@.+\..+", self.email): raise EmailValidationError( f"{self.email} is not an email address format string" diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index 42f68135..bee45f37 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -80,6 +80,8 @@ class Handler: outgoing_messages_status: SizeLimitOrderedDict + background_tasks: set + def __init__(self, max_size: int = 100) -> None: """ Handler diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index c64bb828..0d5c9f5a 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -63,9 +63,13 @@ def __init__( def supports_control_type(self, control_type: ControlType): return control_type in self._resource_manager_details.available_control_types - def close(self): + async def close(self): self._is_closed = True + for control_type, handler in self._control_types_handlers.items(): + print(control_type, handler) + await handler.close() + def is_closed(self): return self._is_closed @@ -273,3 +277,6 @@ def handle_revoke_object(self, message: RevokeObject): ) return get_reception_status(message, ReceptionStatusValues.OK) + + async def send_message(self, message): + await self._sending_queue.put(message) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index e61e61a9..ad2b8c39 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -32,6 +32,7 @@ class FRBC(ControlTypeHandler): _timer_status_history: SizeLimitOrderedDict[str, FRBCTimerStatus] _actuator_status_history: SizeLimitOrderedDict[str, FRBCActuatorStatus] _storage_status_history: SizeLimitOrderedDict[str, FRBCStorageStatus] + background_tasks: set def __init__(self, max_size: int = 100) -> None: super().__init__(max_size) @@ -51,6 +52,7 @@ def __init__(self, max_size: int = 100) -> None: self._system_description_history = SizeLimitOrderedDict(max_size=max_size) self._leakage_behaviour_history = SizeLimitOrderedDict(max_size=max_size) self._usage_forecast_history = SizeLimitOrderedDict(max_size=max_size) + self.background_tasks = set() @register(FRBCSystemDescription) def handle_system_description( @@ -62,15 +64,25 @@ def handle_system_description( self._system_description_history[system_description_id] = message # schedule trigger_schedule to run soon concurrently - asyncio.create_task(self.trigger_schedule(system_description_id)) - + task = asyncio.create_task(self.trigger_schedule(system_description_id)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) - async def send_storage_status(self, status: FRBCStorageStatus): - raise NotImplementedError() + @register(FRBCUsageForecast) + def handle_usage_forecast(self, message: FRBCUsageForecast) -> pydantic.BaseModel: + message_id = str(message.message_id) - async def send_actuator_status(self, status: FRBCActuatorStatus): - raise NotImplementedError() + self._usage_forecast_history[message_id] = message + + task = asyncio.create_task(self.send_usage_forecast(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCStorageStatus) def handle_storage_status(self, message: FRBCStorageStatus) -> pydantic.BaseModel: @@ -78,8 +90,11 @@ def handle_storage_status(self, message: FRBCStorageStatus) -> pydantic.BaseMode self._storage_status_history[message_id] = message - asyncio.create_task(self.send_storage_status(message)) - + task = asyncio.create_task(self.send_storage_status(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCActuatorStatus) @@ -88,29 +103,64 @@ def handle_actuator_status(self, message: FRBCActuatorStatus) -> pydantic.BaseMo self._actuator_status_history[message_id] = message - asyncio.create_task(self.send_actuator_status(message)) - + task = asyncio.create_task(self.send_actuator_status(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCLeakageBehaviour) def handle_leakage_behaviour( self, message: FRBCLeakageBehaviour ) -> pydantic.BaseModel: - # return get_reception_status(message, status=ReceptionStatusValues.OK) - raise NotImplementedError() + message_id = str(message.message_id) - @register(FRBCUsageForecast) - def handle_usage_forecast(self, message: FRBCUsageForecast) -> pydantic.BaseModel: - # return get_reception_status(message, status=ReceptionStatusValues.OK) - raise NotImplementedError() + self._leakage_behaviour_history[message_id] = message - async def trigger_schedule(self, system_description_id: str): - raise NotImplementedError() + task = asyncio.create_task(self.send_leakage_behaviour(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) + + @register(FRBCFillLevelTargetProfile) + def handle_fill_level_target_profile( + self, message: FRBCFillLevelTargetProfile + ) -> pydantic.BaseModel: + message_id = str(message.message_id) + + self._fill_level_target_profile_history[message_id] = message + + task = asyncio.create_task(self.send_fill_level_target_profile(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCTimerStatus) def handle_frbc_timer_status(self, message: FRBCTimerStatus) -> pydantic.BaseModel: return get_reception_status(message, status=ReceptionStatusValues.OK) + async def send_storage_status(self, status: FRBCStorageStatus): + raise NotImplementedError() + + async def send_actuator_status(self, status: FRBCActuatorStatus): + raise NotImplementedError() + + async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): + raise NotImplementedError() + + async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): + raise NotImplementedError() + + async def send_fill_level_target_profile( + self, fill_level_target_profile: FRBCFillLevelTargetProfile + ): + raise NotImplementedError() + class FRBCTest(FRBC): """Dummy class to simulate the triggering of a schedule.""" diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 85acb557..0b023b73 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -75,23 +75,6 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - # await self._fm_client.post_measurements( - # self._soc_sensor_id - # ) - - # system_description = self.find_system_description_from_actuator() - - # if system_description is None: - # return - - # #for a - # if system_description is not None: - - # self._system_description_history[] - # status.active_operation_mode_id - # status.actuator_id - # status.operation_mode_factor - async def trigger_schedule(self, system_description_id: str): """Translates S2 System Description into FM API calls""" diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py new file mode 100644 index 00000000..a31fd177 --- /dev/null +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -0,0 +1,375 @@ +# flake8: noqa +""" +This control type is in a very EXPERIMENTAL stage. +Used it at your own risk :) +""" + +import asyncio +from datetime import datetime, timedelta + +import pandas as pd +import pydantic +import pytz +from s2python.common import NumberRange, ReceptionStatusValues +from s2python.frbc import ( + FRBCActuatorStatus, + FRBCFillLevelTargetProfile, + FRBCInstruction, + FRBCStorageStatus, + FRBCSystemDescription, + FRBCUsageForecast, +) + +from flexmeasures_client.s2 import register +from flexmeasures_client.s2.control_types.FRBC import FRBC +from flexmeasures_client.s2.control_types.translations import ( + translate_fill_level_target_profile, + translate_usage_forecast_to_fm, +) +from flexmeasures_client.s2.utils import get_reception_status, get_unique_id + +RESOLUTION = "15min" +POWER_UNIT = "MWh" +DIMENSIONLESS = "dimensionless" +PERCENTAGE = "%" +TASK_PERIOD_SECONDS = 2 +CONVERSION_EFFICIENCY_DURATION = "PT24H" + + +class FillRateBasedControlTUNES(FRBC): + _fill_level_sensor_id: int | None + + _fill_rate_sensor_id: int | None + _thp_fill_rate_sensor_id: int | None + _thp_efficiency_sensor_id: int | None + _nes_fill_rate_sensor_id: int | None + _nes_efficiency_sensor_id: int | None + + _schedule_duration: timedelta + + _usage_forecast_sensor_id: int | None + _soc_minima_sensor_id: int | None + _soc_maxima_sensor_id: int | None + _rm_discharge_sensor_id: int | None + + def __init__( + self, + soc_minima_sensor_id: int | None = None, + soc_maxima_sensor_id: int | None = None, + fill_level_sensor_id: int | None = None, + usage_forecast_sensor_id: int | None = None, + thp_fill_rate_sensor_id: int | None = None, + thp_efficiency_sensor_id: int | None = None, + nes_fill_rate_sensor_id: int | None = None, + nes_efficiency_sensor_id: int | None = None, + fill_rate_sensor_id: int | None = None, + rm_discharge_sensor_id: int | None = None, + timezone: str = "UTC", + schedule_duration: timedelta = timedelta(hours=12), + max_size: int = 100, + valid_from_shift: timedelta = timedelta(days=1), + **kwargs + ) -> None: + super().__init__(max_size) + + self._fill_level_sensor_id = fill_level_sensor_id + + self._fill_rate_sensor_id = fill_rate_sensor_id + self._thp_fill_rate_sensor_id = thp_fill_rate_sensor_id + self._thp_efficiency_sensor_id = thp_efficiency_sensor_id + self._nes_fill_rate_sensor_id = nes_fill_rate_sensor_id + self._nes_efficiency_sensor_id = nes_efficiency_sensor_id + + self._schedule_duration = schedule_duration + + self._usage_forecast_sensor_id = usage_forecast_sensor_id + self._soc_minima_sensor_id = soc_minima_sensor_id + self._soc_maxima_sensor_id = soc_maxima_sensor_id + self._rm_discharge_sensor_id = rm_discharge_sensor_id + + self._timezone = pytz.timezone(timezone) + + # delay the start of the schedule from the time `valid_from` + # of the FRBC.SystemDescritption + self._valid_from_shift = valid_from_shift + + self._active_recurring_schedule = False + + def now(self): + return self._timezone.localize(datetime.now()) + + async def send_storage_status(self, status: FRBCStorageStatus): + await self._fm_client.post_measurements( + self._fill_level_sensor_id, + start=self.now(), + values=[status.present_fill_level], + unit=POWER_UNIT, + duration=timedelta(minutes=0), # INSTANTANEOUS + ) + + async def send_actuator_status(self, status: FRBCActuatorStatus): + factor = status.operation_mode_factor + system_description: FRBCSystemDescription = list( + self._system_description_history.values() + )[-1] + + # find the active FRBCOperationMode + for op_pos, operation_mode in enumerate( + system_description.actuators[0].operation_modes + ): + if operation_mode.id == status.active_operation_mode_id: + break + + dt = status.transition_timestamp # self.now() + + # Assume that THP is op_pos = 0 and NES = op_pos = 1. + # TODO: should we rely on a sensor_id? For example, "nes-actuator-mode", "thp-actuator-mode" + if op_pos == 0: + active_operation_mode_fill_rate_sensor_id = self._thp_fill_rate_sensor_id + else: + active_operation_mode_fill_rate_sensor_id = self._nes_fill_rate_sensor_id + + # Operation Mode Factor to fill rate + fill_rate = operation_mode.elements[0].fill_rate + fill_rate = ( + fill_rate.start_of_range + + (fill_rate.end_of_range - fill_rate.start_of_range) * factor + ) + + # Send data to the sensor of the fill rate corresponding to the active operation mode + await self._fm_client.post_measurements( + sensor_id=active_operation_mode_fill_rate_sensor_id, + start=dt, + values=[fill_rate], + unit=POWER_UNIT, + duration=timedelta(0), + ) + + # Send data to the sensor of the input fill_rate to the storage device + await self._fm_client.post_measurements( + sensor_id=self._fill_rate_sensor_id, + start=dt, + values=[fill_rate], + unit=POWER_UNIT, + duration=timedelta(0), + ) + + async def start_trigger_schedule(self): + """ + Start a recurring task to create new schedules. + + This function ensures that the scheduling task is started only once. + """ + + if not self._active_recurring_schedule: + self._active_recurring_schedule = True + self._recurrent_task = asyncio.create_task(self.trigger_schedule_task()) + self.background_tasks.add( + self._recurrent_task + ) # important to avoid a task disappearing mid-execution. + self._recurrent_task.add_done_callback(self.background_tasks.discard) + + async def stop_trigger_schedule(self): + """ + Stop the recurring task that creates new schedules. + + This function ensures that the scheduling task is stopped gracefully. + """ + + if self._active_recurring_schedule: + self._active_recurring_schedule = False + self._recurrent_task.cancel() + + async def trigger_schedule_task(self): + """ + Recurring task to trigger the schedule creation process. + + This task runs continuously while the active recurring schedule is enabled. + """ + + while self._active_recurring_schedule: + await self.trigger_scehdule() + await asyncio.sleep(TASK_PERIOD_SECONDS) + + async def trigger_scehdule(self): + """ + Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager + """ + + # Retrieve the latest system description from history + system_description: FRBCSystemDescription = list( + self._system_description_history.values() + )[-1] + + actuator = system_description.actuators[0] + fill_level_range: NumberRange = system_description.storage.fill_level_range + + # get SOC Max and Min to be sent on the Flex Model + soc_min = fill_level_range.end_of_range + soc_max = fill_level_range.start_of_range + + operation_mode = actuator.operation_modes[0] + operation_mode_factor = 0.1 + + # TODO: 1) Call FlexMeasures + # TODO: 2) Select with which actuator to send the instruction + # TODO: 3) Create operation_mode_factor from power (we have a function for that) + + instruction = FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=actuator.id, + operation_mode=operation_mode.id, # Based on the expeted fill_level, select the best actuator (most efficient) to fulfill a certain fill_rate + operation_mode_factor=operation_mode_factor, + execution_time=self.now(), + abnormal_condition=False, + ) + + # Put the instruction in the sending queue + await self._sending_queue.put(instruction) + + @register(FRBCSystemDescription) + def handle_system_description( + self, message: FRBCSystemDescription + ) -> pydantic.BaseModel: + """ + Handle FRBC.SystemDescription messages. + + Process: + 1) Store system_description message for later. + 2) Send conversion efficiencies (COP) to FlexMeasures. + 3) Start a recurring tasks to trigger the scehduler. + """ + + system_description_id = str(message.message_id) + + # store system_description message for later + self._system_description_history[system_description_id] = message + + # send conversion efficiencies + task = asyncio.create_task(self.send_conversion_efficiencies(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + + # schedule trigger_schedule to run soon concurrently + task = asyncio.create_task(self.start_trigger_schedule()) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + + return get_reception_status(message, status=ReceptionStatusValues.OK) + + async def send_conversion_efficiencies( + self, system_description: FRBCSystemDescription + ): + """ + Send conversion efficiencies to FlexMeasures. + + Args: + system_description (FRBCSystemDescription): The system description containing actuator details. + """ + + start = system_description.valid_from + actuator = system_description.actuators[0] + + # Calculate the number of samples based on the conversion efficiency duration + N_SAMPLES = int( + pd.Timedelta(CONVERSION_EFFICIENCY_DURATION) / pd.Timedelta(RESOLUTION) + ) + + thp_op_mode_element = actuator.operation_modes[0].elements[-1] + nes_op_mode_element = actuator.operation_modes[1].elements[-1] + + # THP efficiencies: Calculate and post measurements for THP efficiencies + await self._fm_client.post_measurements( + sensor_id=self._thp_efficiency_sensor_id, + start=start, + values=[ + 100 + * thp_op_mode_element.fill_rate.end_of_range + / thp_op_mode_element.power_ranges[0].end_of_range + ] + * N_SAMPLES, + unit=PERCENTAGE, + duration=CONVERSION_EFFICIENCY_DURATION, + ) + + # NES efficiencies: Calculate and post measurements for NES efficiencies + await self._fm_client.post_measurements( + sensor_id=self._nes_efficiency_sensor_id, + start=start, + values=[ + 100 + * nes_op_mode_element.fill_rate.end_of_range + / nes_op_mode_element.power_ranges[0].end_of_range + ] + * N_SAMPLES, + unit=PERCENTAGE, + duration=CONVERSION_EFFICIENCY_DURATION, + ) + + async def close(self): + """ + Closing procedure: + 1) Stop recurrent task + """ + + await self.stop_trigger_schedule() + + async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): + """ + Send FRBC.UsageForecast to FlexMeasures. + + Args: + usage_forecast (FRBCUsageForecast): The usage forecast to be translated and sent. + """ + + usage_forecast = translate_usage_forecast_to_fm( + usage_forecast, RESOLUTION, strategy="mean" + ) + + await self._fm_client.post_measurements( + sensor_id=self._usage_forecast_sensor_id, + start=usage_forecast.start_time, + values=usage_forecast, + unit=POWER_UNIT, + duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), + ) + + async def send_fill_level_target_profile( + self, fill_level_target_profile: FRBCFillLevelTargetProfile + ): + """ + Send FRBC.FillLevelTargetProfile to FlexMeasures. + + Args: + fill_level_target_profile (FRBCFillLevelTargetProfile): The fill level target profile to be translated and sent. + """ + + soc_minima, soc_maxima = translate_fill_level_target_profile( + fill_level_target_profile, + resolution=RESOLUTION, + ) + + duration = str(pd.Timedelta(RESOLUTION) * len(soc_maxima)) + + # POST SOC Minima measurements to FlexMeasures + await self._fm_client.post_measurements( + sensor_id=self._soc_minima_sensor_id, + start=fill_level_target_profile.start_time, + values=soc_minima, + unit=POWER_UNIT, + duration=duration, + ) + + # POST SOC Maxima measurements to FlexMeasures + await self._fm_client.post_measurements( + sensor_id=self._soc_maxima_sensor_id, + start=fill_level_target_profile.start_time, + values=soc_maxima, + unit=POWER_UNIT, + duration=duration, + ) diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py new file mode 100644 index 00000000..f97c1a23 --- /dev/null +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -0,0 +1,193 @@ +# flake8: noqa + +from datetime import timedelta + +import numpy as np +import pandas as pd +from s2python.frbc import ( + FRBCFillLevelTargetProfile, + FRBCLeakageBehaviour, + FRBCUsageForecast, +) + + +def leakage_behaviour_to_storage_efficieny( + message: FRBCLeakageBehaviour, resolution=timedelta(minutes=15) +) -> float: + """ + Convert a FRBC.LeakeageBehaviour message into a FlexMeasures compatible storage efficiency. + + Definitions: + + LeakageBehaviour: how fast the momentary fill level will decrease per second + due to leakage within the given range of the fill level. This is defined as a function of the + fill level. + + Storage Efficiency: percentage of the storage that remains after one time period. + + Example: + + { + ..., + "elements" : [ + { + "fill_level_range" : {"start_of_range" : 0, "end_of_range" : 5}, + "leakage_rate" : 0 + }, + { + "fill_level_range" : {"start_of_range" : 5, "end_of_range" : 95}, + "leakage_rate" : 1/3600 + } + { + "fill_level_range" : {"start_of_range" : 95, "end_of_range" : 100}, + "leakage_rate" : 2/3600 + } + ] + } + + """ + + last_element = message.elements[-1] + return ( + 1 + - (resolution / timedelta(seconds=1)) + * last_element.leakage_rate + / last_element.fill_level_range.end_of_range + ) + + +def unevenly_ts_to_evenly( + start: pd.Timestamp, + values: list[float], + durations: list[pd.Timedelta], + target_resolution: str, + strategy="mean", +) -> pd.Series: + """ + Convert unevenly spaced time series data into evenly spaced data. + + The function will: + - Floor the start time to align with the target resolution. + - Ceil the end time to align with the target resolution. + - Interpolate and resample the data based on the chosen aggregation strategy. + + Args: + start (pd.Timestamp): The starting timestamp of the time series data. + values (list[float]): The list of values for each time period. + durations (list[pd.Timedelta]): The list of durations for each value. + target_resolution (str): The target time resolution for resampling. + strategy (str): Aggregation strategy ("mean", "min", "max", etc.) for resampling. + + Returns: + pd.Series: A Pandas Series with evenly spaced timestamps and interpolated values. + """ + + # Calculate the time from the absolute start of each event + deltas = pd.TimedeltaIndex(np.cumsum([timedelta(0)] + durations)) + + # Ceil the end time to align with the target resolution + end = pd.Timestamp(start + deltas[-1]).ceil(target_resolution) + + # Floor the start time to align with the target resolution + start = start.floor(target_resolution) + + # Create an index for the time series based on the start time and deltas + index = start + deltas + + # Make a copy of the values list and append a NaN to handle the end boundary + values = values.copy() + values.append(np.nan) + series = pd.Series(values, index) + + # Reindex the series with a regular time grid and forward-fill missing values + series = series.reindex( + pd.date_range( + start=start, + end=end, + freq=min(min(durations), pd.Timedelta(target_resolution)), + inclusive="left", + ) + ).ffill() + + # Resample the series to the target resolution using the specified aggregation strategy and forward-fill + series = series.resample(target_resolution).agg(strategy).ffill() + + return series + + +def translate_usage_forecast_to_fm( + usage_forecast: FRBCUsageForecast, + resolution: str = "1h", +) -> pd.Series: + """ + Translate a FRBC.UsageForecast into a FlexMeasures compatible format with evenly spaced data. + + Args: + usage_forecast (FRBCUsageForecast): The usage forecast message with start time and elements. + resolution (str): The target time resolution for resampling (e.g., "1h"). + + Returns: + pd.Series: A Pandas Series with evenly spaced timestamps and usage forecast values. + """ + + start = pd.Timestamp(usage_forecast.start_time) + + durations = [element.duration.to_timedelta() for element in usage_forecast.elements] + values = [element.usage_rate_expected for element in usage_forecast.elements] + + return unevenly_ts_to_evenly( + start=start, + values=values, + durations=durations, + target_resolution=resolution, + strategy="mean", + ) + + +def translate_fill_level_target_profile( + fill_level_target_profile: FRBCFillLevelTargetProfile, resolution: str = "1h" +) -> tuple[pd.Series, pd.Series]: + """ + Translate a FRBC.FillLevelTargetProfile into SOC minima and maxima compatible with FlexMeasures. + + Args: + fill_level_target_profile (FRBCFillLevelTargetProfile): The target profile message with start time and elements. + resolution (str): The target time resolution for resampling (e.g., "1h"). + + Returns: + tuple[pd.Series, pd.Series]: A tuple containing SOC minima and maxima as Pandas Series. + """ + + start = pd.Timestamp(fill_level_target_profile.start_time) + + durations = [ + element.duration.to_timedelta() + for element in fill_level_target_profile.elements + ] + + soc_minima_values = [ + element.fill_level_range.start_of_range + for element in fill_level_target_profile.elements + ] + soc_maxima_values = [ + element.fill_level_range.end_of_range + for element in fill_level_target_profile.elements + ] + + soc_minima = unevenly_ts_to_evenly( + start=start, + values=soc_minima_values, + durations=durations, + target_resolution=resolution, + strategy="min", + ) + + soc_maxima = unevenly_ts_to_evenly( + start=start, + values=soc_maxima_values, + durations=durations, + target_resolution=resolution, + strategy="max", + ) + + return soc_minima, soc_maxima diff --git a/src/flexmeasures_client/s2/wrapper.py b/src/flexmeasures_client/s2/wrapper.py new file mode 100644 index 00000000..0a4d9c06 --- /dev/null +++ b/src/flexmeasures_client/s2/wrapper.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from pydantic import BaseModel, Field +from s2python.message import S2Message + + +class MetaData(BaseModel): + dt: datetime + + +class S2Wrapper(BaseModel): + message: S2Message = Field(discriminator="message_type") + metadata: MetaData diff --git a/tests/conftest.py b/tests/conftest.py index e0657022..aca532f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,121 @@ -""" - Dummy conftest.py for flexmeasures_client. - - If you don't know what this is for, just leave it empty. - Read more about conftest.py under: - - https://docs.pytest.org/en/stable/fixture.html - - https://docs.pytest.org/en/stable/writing_plugins.html -""" +from __future__ import annotations + +from datetime import datetime + +import pytest +from s2python.common import ( + Commodity, + CommodityQuantity, + ControlType, + Duration, + EnergyManagementRole, + Handshake, + NumberRange, + PowerRange, + ResourceManagerDetails, + Role, + RoleType, +) +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCSystemDescription, +) + +from flexmeasures_client.s2.utils import get_unique_id + + +@pytest.fixture(scope="session") +def frbc_system_description(): + ######## + # FRBC # + ######## + + thp_operation_mode_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=80), + fill_rate=NumberRange(start_of_range=0, end_of_range=2), + power_ranges=[ + PowerRange( + start_of_range=10, + end_of_range=1000, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], + ) + + thp_operation_mode = FRBCOperationMode( + id="tarnoc-operation-mode", + elements=[thp_operation_mode_element], + abnormal_condition_only=False, + ) + + nes_operation_mode_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + fill_rate=NumberRange(start_of_range=0, end_of_range=1), + power_ranges=[ + PowerRange( + start_of_range=10, + end_of_range=1000, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], + ) + + nes_operation_mode = FRBCOperationMode( + id="nestore-operation-mode", + elements=[nes_operation_mode_element], + abnormal_condition_only=False, + ) + + actuator = FRBCActuatorDescription( + id="id-of-the-actuator", + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=[thp_operation_mode, nes_operation_mode], + transitions=[], + timers=[], + ) + + storage = FRBCStorageDescription( + provides_leakage_behaviour=True, + provides_fill_level_target_profile=True, + provides_usage_forecast=True, + fill_level_range=NumberRange(start_of_range=0, end_of_range=1), + ) + + system_description_message = FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1), + actuators=[actuator], + storage=storage, + ) + + return system_description_message + + +@pytest.fixture(scope="session") +def resource_manager_details(): + return ResourceManagerDetails( + message_id=get_unique_id(), + resource_id=get_unique_id(), + roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], + instruction_processing_delay=Duration(__root__=1.0), + available_control_types=[ + ControlType.FILL_RATE_BASED_CONTROL, + ControlType.NO_SELECTION, + ], + provides_forecast=True, + provides_power_measurement_types=[ + CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC + ], + ) + + +@pytest.fixture(scope="session") +def rm_handshake(): + return Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.RM, + supported_protocol_versions=["1.0.0"], + ) diff --git a/tests/test_s2_coordinator.py b/tests/test_cem.py similarity index 53% rename from tests/test_s2_coordinator.py rename to tests/test_cem.py index 9903e4ac..56122f62 100644 --- a/tests/test_s2_coordinator.py +++ b/tests/test_cem.py @@ -1,38 +1,14 @@ from __future__ import annotations -from datetime import datetime - import pytest -from s2python.common import ( - Commodity, - CommodityQuantity, - ControlType, - Duration, - EnergyManagementRole, - Handshake, - NumberRange, - PowerRange, - ReceptionStatus, - ReceptionStatusValues, - ResourceManagerDetails, - Role, - RoleType, -) -from s2python.frbc import ( - FRBCActuatorDescription, - FRBCOperationMode, - FRBCOperationModeElement, - FRBCStorageDescription, - FRBCSystemDescription, -) +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC import FRBCTest -from flexmeasures_client.s2.utils import get_unique_id @pytest.mark.asyncio -async def test_cem(): # TODO: move into different test functions +async def test_handshake(rm_handshake): cem = CEM(fm_client=None) frbc = FRBCTest() @@ -42,18 +18,14 @@ async def test_cem(): # TODO: move into different test functions # Handshake # ############# - handshake_message = Handshake( - message_id=get_unique_id(), - role=EnergyManagementRole.RM, - supported_protocol_versions=["0.1.0"], - ) - - await cem.handle_message(handshake_message) + # RM sends HandShake + await cem.handle_message(rm_handshake) assert ( cem._sending_queue.qsize() == 1 ) # check that message is put to the outgoing queue + # CEM response response = await cem.get_message() assert ( @@ -63,42 +35,74 @@ async def test_cem(): # TODO: move into different test functions response["selected_protocol_version"] == "0.1.0" ), "CEM selected protocol version should be supported by the Resource Manager" + +@pytest.mark.asyncio +async def test_resource_manager_details(resource_manager_details, rm_handshake): + cem = CEM(fm_client=None) + frbc = FRBCTest() + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + + assert ( + cem._sending_queue.qsize() == 1 + ) # check that message is put to the outgoing queue + + response = await cem.get_message() + ########################## # ResourceManagerDetails # ########################## - resource_manager_details_message = ResourceManagerDetails( - message_id=get_unique_id(), - resource_id=get_unique_id(), - roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], - instruction_processing_delay=Duration(__root__=1.0), - available_control_types=[ - ControlType.FILL_RATE_BASED_CONTROL, - ControlType.NO_SELECTION, - ], - provides_forecast=True, - provides_power_measurement_types=[ - CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - ], - ) - - await cem.handle_message(resource_manager_details_message) + # RM sends ResourceManagerDetails + await cem.handle_message(resource_manager_details) response = await cem.get_message() + # CEM response is ReceptionStatus with an OK status assert response["message_type"] == "ReceptionStatus" assert response["status"] == "OK" + assert ( - cem._resource_manager_details == resource_manager_details_message + cem._resource_manager_details == resource_manager_details ), "CEM should store the resource_manager_details" assert cem.control_type == ControlType.NO_SELECTION, ( "CEM control type should switch to ControlType.NO_SELECTION," "independently of the original type" ) + +@pytest.mark.asyncio +async def test_activate_control_type( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + frbc = FRBCTest() + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + ######################### # Activate control type # ######################### + # CEM sends a request to change te control type await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) message = await cem.get_message() @@ -117,59 +121,55 @@ async def test_cem(): # TODO: move into different test functions cem.control_type == ControlType.FILL_RATE_BASED_CONTROL ), "after a positive ResponseStatus, the status changes from NO_SELECTION to FRBC" - ######## - # FRBC # - ######## - operation_mode_element = FRBCOperationModeElement( - fill_level_range=NumberRange(start_of_range=0, end_of_range=1), - fill_rate=NumberRange(start_of_range=0, end_of_range=1), - power_ranges=[ - PowerRange( - start_of_range=10, - end_of_range=1000, - commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, - ) - ], - ) +@pytest.mark.asyncio +async def test_messages_route_to_control_type_handler( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + frbc = FRBCTest() - operation_mode = FRBCOperationMode( - id=get_unique_id(), - elements=[operation_mode_element], - abnormal_condition_only=False, - ) + cem.register_control_type(frbc) - actuator = FRBCActuatorDescription( - id=get_unique_id(), - supported_commodities=[Commodity.ELECTRICITY], - operation_modes=[operation_mode], - transitions=[], - timers=[], - ) + ############# + # Handshake # + ############# - storage = FRBCStorageDescription( - provides_leakage_behaviour=False, - provides_fill_level_target_profile=False, - provides_usage_forecast=False, - fill_level_range=NumberRange(start_of_range=0, end_of_range=1), - ) + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### - system_description_message = FRBCSystemDescription( - message_id=get_unique_id(), - valid_from=datetime.now(), - actuators=[actuator], - storage=storage, + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + message = await cem.get_message() + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK ) - await cem.handle_message(system_description_message) + await cem.handle_message(response) + + ######## + # FRBC # + ######## + + await cem.handle_message(frbc_system_description) response = await cem.get_message() # checking that FRBC handler is being called assert ( cem._control_types_handlers[ ControlType.FILL_RATE_BASED_CONTROL - ]._system_description_history[str(system_description_message.message_id)] - == system_description_message + ]._system_description_history[str(frbc_system_description.message_id)] + == frbc_system_description ), ( "the FRBC.SystemDescription message should be stored" "in the frbc.system_description_history variable" diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py new file mode 100644 index 00000000..30d898c0 --- /dev/null +++ b/tests/test_frbc_tunes.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import numpy as np +import pandas as pd +import pytest +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, +) + + +@pytest.fixture(scope="function") +async def setup_cem(resource_manager_details, rm_handshake): + fm_client = AsyncMock(FlexMeasuresClient) + cem = CEM(fm_client=fm_client) + frbc = FillRateBasedControlTUNES( + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + timezone="UTC", + schedule_duration=timedelta(hours=12), + max_size=100, + valid_from_shift=timedelta(days=1), + ) + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### + + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + message = await cem.get_message() + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK + ) + + await cem.handle_message(response) + + return cem, fm_client + + +@pytest.fixture(scope="function") +async def cem_in_frbc_control_type(setup_cem, frbc_system_description): + cem, fm_client = await setup_cem + + ######## + # FRBC # + ######## + + await cem.handle_message(frbc_system_description) + await cem.get_message() + + return cem, fm_client + + +@pytest.mark.asyncio +async def test_system_description(cem_in_frbc_control_type, frbc_system_description): + cem, fm_client = await cem_in_frbc_control_type + + ######## + # FRBC # + ######## + + await cem.handle_message(frbc_system_description) + frbc = cem._control_types_handlers[cem.control_type] + + tasks = get_pending_tasks() + + # check that we are sending the conversion efficiencies + await tasks["send_conversion_efficiencies"] + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + CONVERSION_EFFICIENCY_DURATION, + RESOLUTION, + ) + + N_SAMPLES = int( + pd.Timedelta(CONVERSION_EFFICIENCY_DURATION) / pd.Timedelta(RESOLUTION) + ) + + # first call of post_measurements which corresponds to the THP efficiency + first_call = fm_client.post_measurements.call_args_list[0][1] + first_call_expected = { + "sensor_id": frbc._thp_efficiency_sensor_id, + "start": datetime(2024, 1, 1), + "values": [0.2] * N_SAMPLES, + "unit": "%", + "duration": "PT24H", + } + for key in first_call.keys(): + assert first_call[key] == first_call_expected[key] + + # second call of post_measurements which corresponds to the NES efficiency + second_call = fm_client.post_measurements.call_args_list[1][1] + + second_call_expected = { + "sensor_id": frbc._nes_efficiency_sensor_id, + "start": datetime(2024, 1, 1), + "values": [0.1] * N_SAMPLES, + "unit": "%", + "duration": "PT24H", + } + for key in second_call.keys(): + assert second_call[key] == second_call_expected[key] + + await cem.close() + get_pending_tasks() + + +def get_pending_tasks(): + pending = asyncio.all_tasks() + + tasks = {} + + # get all pending tasks + for task in pending: + func_name = task.get_coro().cr_code.co_name + tasks[func_name] = task + + return tasks + + +@pytest.mark.asyncio +async def test_fill_level_target_profile(cem_in_frbc_control_type): + cem, fm_client = await cem_in_frbc_control_type + + fill_level_target_profile = { + "start_time": "2024-01-01T00:00:00+01:00", + "message_type": "FRBC.FillLevelTargetProfile", + "message_id": "a-valid-id", + "elements": [ + { + "duration": 1e3 * 3600, + "fill_level_range": {"start_of_range": 0, "end_of_range": 100}, + }, + { + "duration": 1e3 * 2 * 3600, + "fill_level_range": {"start_of_range": 10, "end_of_range": 90}, + }, + { + "duration": 1e3 * 3 * 3600, + "fill_level_range": {"start_of_range": 20, "end_of_range": 80}, + }, + ], + } + + await cem.handle_message(fill_level_target_profile) + + tasks = get_pending_tasks() + + # clear mock state because it contains previous such as + # the ones used to process the system description + fm_client.reset_mock() + + # wait for the task send_fill_level_target_profile to finish + await tasks["send_fill_level_target_profile"] + + start = datetime(2024, 1, 1, 0, 0, tzinfo=timezone(timedelta(seconds=3600))) + + first_call = fm_client.post_measurements.call_args_list[0][1] + assert first_call["sensor_id"] == 2 + assert first_call["start"] == start + assert np.isclose(first_call["values"].values, [0] * 4 + [10] * 8 + [20] * 12).all() + + second_call = fm_client.post_measurements.call_args_list[1][1] + assert second_call["sensor_id"] == 3 + assert second_call["start"] == start + assert np.isclose( + second_call["values"].values, [100] * 4 + [90] * 8 + [80] * 12 + ).all() + + await cem.close() + get_pending_tasks() + + +@pytest.mark.asyncio +async def test_fill_rate_relay(cem_in_frbc_control_type): + cem, fm_client = await cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + + actuator_status = { + "active_operation_mode_id": "tarnoc-operation-mode", + "actuator_id": "id-of-the-actuator", + "message_type": "FRBC.ActuatorStatus", + "message_id": "a-valid-id", + "operation_mode_factor": 0.0, + } + + await cem.handle_message(actuator_status) + + tasks = get_pending_tasks() + + # clear mock state because it contains previous such as + # the ones used to process the system description + fm_client.reset_mock() + + # wait for the task send_actuator_status to finish + await tasks["send_actuator_status"] + + first_call = fm_client.post_measurements.call_args_list[0][1] + assert first_call["sensor_id"] == frbc._thp_fill_rate_sensor_id + + second_call = fm_client.post_measurements.call_args_list[1][1] + assert second_call["sensor_id"] == frbc._fill_rate_sensor_id + + # Switch operation mode to Nestore + actuator_status["active_operation_mode_id"] = "nestore-operation-mode" + + await cem.handle_message(actuator_status) + tasks = get_pending_tasks() + + # clear mock state because it contains previous such as + # the ones used to process the system description + fm_client.reset_mock() + + # wait for the task send_actuator_status to finish + await tasks["send_actuator_status"] + + first_call = fm_client.post_measurements.call_args_list[0][1] + assert first_call["sensor_id"] == frbc._nes_fill_rate_sensor_id + + second_call = fm_client.post_measurements.call_args_list[1][1] + assert second_call["sensor_id"] == frbc._fill_rate_sensor_id + + await cem.close() + get_pending_tasks() + + +@pytest.mark.asyncio +async def test_trigger_schedule(cem_in_frbc_control_type): + cem, fm_client = await cem_in_frbc_control_type + # frbc = cem._control_types_handlers[cem.control_type] + + tasks = get_pending_tasks() + + assert tasks["trigger_schedule_task"]._state == "PENDING" + await cem.close() + + tasks = get_pending_tasks() + + assert tasks["trigger_schedule_task"]._state == "PENDING" + + await cem.close() + get_pending_tasks() diff --git a/tests/test_s2_models.py b/tests/test_s2_models.py new file mode 100644 index 00000000..7c57ce26 --- /dev/null +++ b/tests/test_s2_models.py @@ -0,0 +1,31 @@ +from flexmeasures_client.s2.utils import get_unique_id +from flexmeasures_client.s2.wrapper import S2Wrapper + + +def test_simple_model(): + wrapped_message = { + "message": { + "message_id": get_unique_id(), + "resource_id": get_unique_id(), + "roles": [{"role": "ENERGY_STORAGE", "commodity": "ELECTRICITY"}], + "instruction_processing_delay": 1.0, + "available_control_types": ["FILL_RATE_BASED_CONTROL", "NO_SELECTION"], + "provides_forecast": True, + "provides_power_measurement_types": ["ELECTRIC.POWER.3_PHASE_SYMMETRIC"], + "message_type": "ResourceManagerDetails", + }, + "metadata": {"dt": "2023-01-01T00:00:00"}, + } + + S2Wrapper.validate(wrapped_message) + + wrapped_message_2 = { + "message": { + "message_id": get_unique_id(), + "message_type": "Handshake", + "role": "CEM", + }, + "metadata": {"dt": "2024-01-01T00:00:00"}, + } + + S2Wrapper.validate(wrapped_message_2) diff --git a/tests/test_s2_translations.py b/tests/test_s2_translations.py new file mode 100644 index 00000000..e634375e --- /dev/null +++ b/tests/test_s2_translations.py @@ -0,0 +1,78 @@ +import pytest +from s2python.frbc import FRBCUsageForecast + +from flexmeasures_client.s2.control_types.translations import ( + translate_usage_forecast_to_fm, +) +from flexmeasures_client.s2.utils import get_unique_id + + +@pytest.mark.parametrize( + "start, resolution, values", + [ + ("2024-01-01T00:00:00+01:00", "1h", [100, 100]), + ("2024-01-01T00:00:00+01:00", "15min", [100] * 4 * 2), + ("2024-01-01T00:30:00+01:00", "1h", [100, 100, 100]), + ], +) +def test_resampling_one_block(start, resolution, values): + message = { + "elements": [ + {"duration": 2 * 3600 * 1e3, "usage_rate_expected": 100}, + ], + "message_id": get_unique_id(), + "message_type": "FRBC.UsageForecast", + "start_time": start, + } + + usage_forecast = FRBCUsageForecast.from_dict(message) + + s = translate_usage_forecast_to_fm(usage_forecast, resolution=resolution) + assert all(abs(s.values - values) < 1e-5) + + +@pytest.mark.parametrize( + "start, resolution, values", + [ + ("2024-01-01T00:00:00+01:00", "1h", [100, 200, 200, 350, 450, 600]), + ("2024-01-01T00:45:00+01:00", "1h", [100, 200, 200, 350, 450, 600, 600]), + ( + "2024-01-01T00:00:00+01:00", + "30min", + [100] * 2 + [200] * 2 * 2 + [300] * 1 + [400] * 2 + [500] * 1 + [600] * 1, + ), + ( + "2024-01-01T00:00:00+01:00", + "15min", + [100] * 4 + [200] * 4 * 2 + [300] * 2 + [400] * 4 + [500] * 2 + [600] * 2, + ), + ], +) +def test_usage_forecast(start, resolution, values): + """ + - 100 for 1h + - 200 for 2h + - 300 for 30min + - 400 for 1h + - 500 for 30min + - 600 for 30min + + """ + message = { + "elements": [ + {"duration": 3600 * 1e3, "usage_rate_expected": 100}, + {"duration": 2 * 3600 * 1e3, "usage_rate_expected": 200}, + {"duration": 0.5 * 3600 * 1e3, "usage_rate_expected": 300}, + {"duration": 3600 * 1e3, "usage_rate_expected": 400}, + {"duration": 0.5 * 3600 * 1e3, "usage_rate_expected": 500}, + {"duration": 0.5 * 3600 * 1e3, "usage_rate_expected": 600}, + ], + "message_id": get_unique_id(), + "message_type": "FRBC.UsageForecast", + "start_time": start, + } + + usage_forecast = FRBCUsageForecast.from_dict(message) + + s = translate_usage_forecast_to_fm(usage_forecast, resolution=resolution) + assert all(abs(s.values - values) < 1e-5) From 3b3ca0b1dff279599460467cf83ab74c96e50260 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 7 Aug 2024 16:56:51 +0200 Subject: [PATCH 02/79] make session optional Signed-off-by: Victor Garcia Reolid --- src/flexmeasures_client/client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index f8875ed0..3b6e47c8 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -7,7 +7,7 @@ import socket from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any +from typing import Any, cast import async_timeout import pandas as pd @@ -55,9 +55,11 @@ class FlexMeasuresClient: polling_timeout: float = POLLING_TIMEOUT # seconds request_timeout: float = REQUEST_TIMEOUT # seconds polling_interval: float = POLLING_INTERVAL # seconds + session: ClientSession | None = None def __post_init__(self): - self.session: ClientSession = ClientSession() + if self.session is None: + self.session = ClientSession() if not re.match(r".+\@.+\..+", self.email): raise EmailValidationError( @@ -89,7 +91,7 @@ def __post_init__(self): async def close(self): """Function to close FlexMeasuresClient session when all requests are done""" - await self.session.close() + await cast(ClientSession, self.session).close() async def request( self, @@ -184,7 +186,7 @@ async def request_once( logging.debug("=" * 14) """Sends a single request to FlexMeasures and checks the response""" - response = await self.session.request( + response = await cast(ClientSession, self.session).request( method=method, url=url, params=params, From 473ca06377e79208a2018204c011bdbafd66cb30 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 30 Sep 2024 12:41:58 +0200 Subject: [PATCH 03/79] docs: fix typo --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5913504c..059e8803 100644 --- a/README.rst +++ b/README.rst @@ -46,7 +46,9 @@ As the Flexmeasures Client is still in active development and on version 0.1 it Getting Started =============== -To get started using the FlexMeasures Client package first an account needs to be registered with a FlexMeasures instance or a local FlexMeasures instance needs to be created. Registring a to a FlexMeasures instance can be done through `Seita BV `_. To create a local instance of FlexMeasures follow the `FlexMeasures documentation `_. +To get started using the FlexMeasures Client package first an account needs to be registered with a FlexMeasures instance or a local FlexMeasures instance needs to be created. +Registering a to a FlexMeasures instance can be done through `Seita BV `_. +To create a local instance of FlexMeasures follow the `FlexMeasures documentation `_. In this example we are connecting to ``localhost:5000``, To connect to a different host add the host in the initialization of the client. From 8f8bbaba6c60e47d56b33c4ac5ed536f00378f51 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Oct 2024 21:43:05 +0100 Subject: [PATCH 04/79] docs: fix typo --- .../s2/control_types/FRBC/frbc_tunes.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index a31fd177..07004723 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -10,7 +10,7 @@ import pandas as pd import pydantic import pytz -from s2python.common import NumberRange, ReceptionStatusValues +from s2python.common import NumberRange, ReceptionStatusValues, ReceptionStatus from s2python.frbc import ( FRBCActuatorStatus, FRBCFillLevelTargetProfile, @@ -29,7 +29,8 @@ from flexmeasures_client.s2.utils import get_reception_status, get_unique_id RESOLUTION = "15min" -POWER_UNIT = "MWh" +ENERGY_UNIT = "MWh" +POWER_UNIT = "MW" DIMENSIONLESS = "dimensionless" PERCENTAGE = "%" TASK_PERIOD_SECONDS = 2 @@ -50,7 +51,7 @@ class FillRateBasedControlTUNES(FRBC): _usage_forecast_sensor_id: int | None _soc_minima_sensor_id: int | None _soc_maxima_sensor_id: int | None - _rm_discharge_sensor_id: int | None + # _rm_discharge_sensor_id: int | None def __init__( self, @@ -99,13 +100,21 @@ def now(self): return self._timezone.localize(datetime.now()) async def send_storage_status(self, status: FRBCStorageStatus): - await self._fm_client.post_measurements( - self._fill_level_sensor_id, - start=self.now(), - values=[status.present_fill_level], - unit=POWER_UNIT, - duration=timedelta(minutes=0), # INSTANTANEOUS - ) + try: + await self._fm_client.post_measurements( + self._fill_level_sensor_id, + start=self.now(), + values=[status.present_fill_level], + unit=POWER_UNIT, + duration=timedelta(minutes=15), # INSTANTANEOUS + ) + except Exception as e: + response = ReceptionStatus( + subject_message_id=status.get("message_id"), + status=ReceptionStatusValues.PERMANENT_ERROR, + ) + await self._sending_queue.put(response) + async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor @@ -142,7 +151,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): start=dt, values=[fill_rate], unit=POWER_UNIT, - duration=timedelta(0), + duration=timedelta(minutes=15), ) # Send data to the sensor of the input fill_rate to the storage device @@ -151,7 +160,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): start=dt, values=[fill_rate], unit=POWER_UNIT, - duration=timedelta(0), + duration=timedelta(minutes=15), ) async def start_trigger_schedule(self): @@ -326,6 +335,8 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): Args: usage_forecast (FRBCUsageForecast): The usage forecast to be translated and sent. """ + start_time = usage_forecast.start_time + # todo: floor to RESOLUTION usage_forecast = translate_usage_forecast_to_fm( usage_forecast, RESOLUTION, strategy="mean" @@ -333,8 +344,8 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): await self._fm_client.post_measurements( sensor_id=self._usage_forecast_sensor_id, - start=usage_forecast.start_time, - values=usage_forecast, + start=start_time, + values=usage_forecast.tolist(), unit=POWER_UNIT, duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), ) From 8f7d36941168f84e5b151ac4904616e2b028de50 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:49:20 +0100 Subject: [PATCH 05/79] fix: pin pandas --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d5e3fe93..d60818bd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,7 @@ python_requires >= 3.9 install_requires = importlib-metadata; python_version<"3.8" aiohttp<=3.9.1 - pandas + pandas==2.2.1 pydantic>=1.10.8,<2.0 s2-python @ git+ssh://git@github.com/flexiblepower/s2-python@victor async_timeout From 768f4fdd9f47bae9686273d938d7eaad190b774e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:49:44 +0100 Subject: [PATCH 06/79] docs: comment on tests --- tests/test_frbc_tunes.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 30d898c0..2b98b1a7 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -203,6 +203,11 @@ async def test_fill_level_target_profile(cem_in_frbc_control_type): @pytest.mark.asyncio async def test_fill_rate_relay(cem_in_frbc_control_type): + """Check whether the fill rate from the Tarnoc or Nestor is relayed + to the overall heating system's fill rate sensor, and the fill rate sensor ID + corresponds correctly to the Tarnoc fill rate sensor or the Nestor fill rate sensor. + """ + cem, fm_client = await cem_in_frbc_control_type frbc = cem._control_types_handlers[cem.control_type] @@ -251,11 +256,24 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert second_call["sensor_id"] == frbc._fill_rate_sensor_id await cem.close() - get_pending_tasks() @pytest.mark.asyncio async def test_trigger_schedule(cem_in_frbc_control_type): + """Work in progress. + + # todo: add steps + + Steps + + 1. Check whether the task starts and stops + 2. Check call arguments + 3. Check queue for results mocking the results of FM client + + # todo consider splitting up test + S2 2 FM: converging system description to flex config + FM 2 S2: schedules to instructions + """ cem, fm_client = await cem_in_frbc_control_type # frbc = cem._control_types_handlers[cem.control_type] From bd9fcdf28870f52d7d419da97154738c4cf9c44f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:50:03 +0100 Subject: [PATCH 07/79] feature: expose translation strategy further up the chain of methods --- src/flexmeasures_client/s2/control_types/translations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index f97c1a23..db22bcb4 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -118,6 +118,7 @@ def unevenly_ts_to_evenly( def translate_usage_forecast_to_fm( usage_forecast: FRBCUsageForecast, resolution: str = "1h", + strategy: str = "mean", ) -> pd.Series: """ Translate a FRBC.UsageForecast into a FlexMeasures compatible format with evenly spaced data. @@ -140,7 +141,7 @@ def translate_usage_forecast_to_fm( values=values, durations=durations, target_resolution=resolution, - strategy="mean", + strategy=strategy, ) From f32d392018c1cd0b6d071470d79eb674d04c5dc0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:55:18 +0100 Subject: [PATCH 08/79] style: black --- src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 07004723..4c6a31fb 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -115,7 +115,6 @@ async def send_storage_status(self, status: FRBCStorageStatus): ) await self._sending_queue.put(response) - async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor system_description: FRBCSystemDescription = list( From 050eb7debf7b61d5e0cec5485370d78bcf1407b2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:57:26 +0100 Subject: [PATCH 09/79] style: update black --- .pre-commit-config.yaml | 2 +- src/flexmeasures_client/s2/__init__.py | 12 ++++++------ src/flexmeasures_client/s2/cem.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d409690..89455e67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.8.0 hooks: - id: black language_version: python3 diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index bee45f37..66117f5a 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -40,15 +40,15 @@ def wrap(*args, **kwargs): # TODO: implement function __hash__ in ID that returns # the value of __root__, this way we would be able to use # the ID as key directly - self.incoming_messages[ - get_message_id(incoming_message) - ] = incoming_message + self.incoming_messages[get_message_id(incoming_message)] = ( + incoming_message + ) outgoing_message = func(self, incoming_message) - self.outgoing_messages[ - get_message_id(outgoing_message) - ] = outgoing_message + self.outgoing_messages[get_message_id(outgoing_message)] = ( + outgoing_message + ) return outgoing_message diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 0d5c9f5a..8d40280a 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -96,9 +96,9 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): control_type_handler._sending_queue = self._sending_queue # store control_type_handler - self._control_types_handlers[ - control_type_handler._control_type - ] = control_type_handler + self._control_types_handlers[control_type_handler._control_type] = ( + control_type_handler + ) async def handle_message(self, message: Dict | pydantic.BaseModel | str): """ From f920294d146f0ab2ba172bf0ac1ea2d34329e2b3 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 5 Nov 2024 11:58:53 +0100 Subject: [PATCH 10/79] apply pre-commit --- src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 4c6a31fb..e4b64ce3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -10,7 +10,7 @@ import pandas as pd import pydantic import pytz -from s2python.common import NumberRange, ReceptionStatusValues, ReceptionStatus +from s2python.common import NumberRange, ReceptionStatus, ReceptionStatusValues from s2python.frbc import ( FRBCActuatorStatus, FRBCFillLevelTargetProfile, From 6f78a2d5adfc44e64684b1a7e64d8c9499f1c657 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 5 Nov 2024 12:00:37 +0100 Subject: [PATCH 11/79] per-comit on setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8a3f4305..3696d25f 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ PyScaffold helps you to put up the scaffold of your new Python project. Learn more under: https://pyscaffold.org/ """ + from setuptools import setup if __name__ == "__main__": From 3c14ef7369df89a5522828d466582298311f8a9e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 12:17:13 +0100 Subject: [PATCH 12/79] chore: use (dev) released s2-python version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d60818bd..0c0cf282 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas==2.2.1 pydantic>=1.10.8,<2.0 - s2-python @ git+ssh://git@github.com/flexiblepower/s2-python@victor + s2-python==v0.3.0.dev1 async_timeout From f89419865acd4712e7ec38dbfe9354cdd837dc21 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Dec 2024 14:02:06 +0100 Subject: [PATCH 13/79] fix: update s2-python Signed-off-by: F.N. Claessen --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index c6aead6b..9dacc851 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas==2.2.1 pydantic>=1.10.8,<2.0 - s2-python==0.2.0.dev2 + s2-python==0.1.3 async_timeout [options.packages.find] From 5f1ea21f52ef5b5a4391a0e3953ea09b93e8eb9f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Dec 2024 11:31:33 +0100 Subject: [PATCH 14/79] fix: relax Pandas constraint Signed-off-by: F.N. Claessen --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9dacc851..ab03748c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,7 @@ python_requires >= 3.9 install_requires = importlib-metadata; python_version<"3.8" aiohttp<=3.9.1 - pandas==2.2.1 + pandas>=2.1.4 pydantic>=1.10.8,<2.0 s2-python==0.1.3 async_timeout From 5f61680db5019c4cd68852929e929ef76f13b019 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Dec 2024 11:38:42 +0100 Subject: [PATCH 15/79] fix: fetch whole history, incl. previous dev tags Signed-off-by: F.N. Claessen --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53ce7d0d..2a3af4a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,7 @@ jobs: id-token: write steps: - uses: actions/checkout@v3 + with: {fetch-depth: 0} # deep clone for setuptools-scm - uses: actions/setup-python@v4 with: {python-version: "3.11"} - name: Retrieve pre-built distribution files From 43fe6ac86f12b869c9ecfbd7fc8d54cc13d572da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 2 Oct 2024 17:15:28 +0200 Subject: [PATCH 16/79] rename start_session() to more fitting ensure_session(); fix black/flake8/mypy issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index dc173b7d..2f9a0d7f 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -91,7 +91,7 @@ def __post_init__(self): self.determine_port() def determine_port(self): - parts = self.host.split(':') + parts = self.host.split(":") if len(parts) > 1: if self.port is not None: raise WrongHostError( @@ -100,7 +100,7 @@ def determine_port(self): self.host = parts[0] self.port = int(parts[1]) elif self.port is None: - self.port = 443 if self.scheme == 'https' else 80 + self.port = 443 if self.scheme == "https" else 80 async def close(self): """Function to close FlexMeasuresClient session when all requests are done""" @@ -129,7 +129,7 @@ async def request( """ # noqa: E501 url = self.build_url(uri, path=path) - self.start_session() + self.ensure_session() polling_step = 0 # reset this counter once when starting polling # we allow retrying once if we include authentication headers @@ -199,7 +199,8 @@ async def request_once( logging.debug("=" * 14) """Sends a single request to FlexMeasures and checks the response""" - response = await self.session.request( + self.ensure_session() + response = await self.session.request( # type: ignore method=method, url=url, params=params, @@ -224,7 +225,7 @@ async def request_once( ) return response, polling_step, reauth_once, url - def start_session(self): + def ensure_session(self): """If there is no session, start one""" if self.session is None: self.session = ClientSession() @@ -240,7 +241,9 @@ async def get_headers(self, include_auth: bool) -> dict: def build_url(self, uri: str, path: str = path) -> URL: """Build url for request""" - url = URL.build(scheme=self.scheme, host=self.host, port=self.port, path=path).join( + url = URL.build( + scheme=self.scheme, host=self.host, port=self.port, path=path + ).join( URL(uri), ) return url From 50f889e9051a4b3f75b8a423472132295fa4d1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 2 Oct 2024 17:45:31 +0200 Subject: [PATCH 17/79] do not require session on client.init; fix tests for recent port configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning Signed-off-by: Vlad Iftime --- README.rst | 16 ++++++++++++++++ src/flexmeasures_client/client.py | 2 -- tests/test_client.py | 11 ++++++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index c5c9bb7b..d89eb743 100644 --- a/README.rst +++ b/README.rst @@ -133,9 +133,24 @@ The schedule returns a Pandas ``DataFrame`` that can be used to regulate the fle .. _pyscaffold-notes: + Making Changes & Contributing ============================= +.. note: Read more details in CONTRIBUTING.rst + +Install the project locally (in a virtual environment of your choice):: + + pip install -e + + +Running tests locally is crucial as well. Staying close to the CI workflow:: + + pip install tox + tox -e clean,build + tox -- -rFEx --durations 10 --color yes + + This project uses `pre-commit`_, please make sure to install it before making any changes:: @@ -151,6 +166,7 @@ Don't forget to tell your contributors to also install and use pre-commit. .. _pre-commit: https://pre-commit.com/ + =================== S2 Protocol =================== diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 2f9a0d7f..0360d38e 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -59,8 +59,6 @@ class FlexMeasuresClient: session: ClientSession | None = None def __post_init__(self): - if self.session is None: - self.session = ClientSession() if not re.match(r".+\@.+\..+", self.email): raise EmailValidationError( f"{self.email} is not an email address format string" diff --git a/tests/test_client.py b/tests/test_client.py index ac842497..d7f774db 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -15,7 +15,7 @@ @pytest.mark.parametrize( - "ssl, host, api_version, email, password, asserted_ssl, asserted_host, asserted_version , asserted_scheme", # noqa: E501 + "ssl, host, api_version, email, password, asserted_ssl, asserted_host, asserted_port, asserted_version, asserted_scheme", # noqa: E501 [ ( False, @@ -24,7 +24,8 @@ "test@test.test", "password", False, - "localhost:5000", + "localhost", + 5000, "v3_0", "http", ), @@ -36,6 +37,7 @@ "password", True, "test_host.test", + 443, "v3_0", "https", ), @@ -46,7 +48,8 @@ "test@test.test", "password", True, - "localhost:5000", + "localhost", + 5000, "v3_0", "https", ), @@ -60,6 +63,7 @@ def test__init__( password, asserted_ssl, asserted_host, + asserted_port, asserted_version, asserted_scheme, ): @@ -72,6 +76,7 @@ def test__init__( "email": email, "access_token": None, "host": asserted_host, + "port": asserted_port, "scheme": asserted_scheme, "ssl": asserted_ssl, "api_version": asserted_version, From 36ce4f61fa933204424be463f3f29df5799c7ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Wed, 2 Oct 2024 18:31:40 +0200 Subject: [PATCH 18/79] add deployment by tags to developer docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning Signed-off-by: Vlad Iftime --- README.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.rst b/README.rst index d89eb743..ab8a6e3b 100644 --- a/README.rst +++ b/README.rst @@ -167,6 +167,16 @@ Don't forget to tell your contributors to also install and use pre-commit. .. _pre-commit: https://pre-commit.com/ +New releases on Pypi are made by adding a tag and pushing it:: + + git tag -s -a vX.Y.Z -m "Short summary" + git push --tags + +(of course you need the permissions to do so) + +See releases in GitHub Actions at https://github.com/FlexMeasures/flexmeasures-client/deployments/release + + =================== S2 Protocol =================== From 133d2b93373f4bb8514c1a5221e3e68cbdbe7eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Tue, 8 Oct 2024 23:06:05 +0200 Subject: [PATCH 19/79] Move comments in Readme examples to end of lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning Signed-off-by: Vlad Iftime --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index ab8a6e3b..2d4fd955 100644 --- a/README.rst +++ b/README.rst @@ -86,10 +86,10 @@ Trigger and retrieve a schedule:: sensor_id=, # int start="2023-03-26T10:00+02:00", # iso datetime duration="PT12H", # iso timedelta - flex_context= {"consumption-price-sensor": , # int}, + flex_context= {"consumption-price-sensor": }, # int flex-model= { "soc-unit": "kWh", - "soc-at-start": 50, # soc_units (kWh) + "soc-at-start": 50, # in soc_units (kWh) "soc-max": 400, "soc-min": 20, "soc-targets": [ @@ -106,7 +106,7 @@ Trigger a schedule:: sensor_id=, # int start="2023-03-26T10:00+02:00", # iso datetime duration="PT12H", # iso timedelta - flex_context= {"consumption-price-sensor": , # int}, + flex_context= {"consumption-price-sensor": }, # int flex-model= { "soc-unit": "kWh", "soc-at-start": 50, # soc_units (kWh) From 24bf44bfb41193de11ed1e8dfe08ef1846d4fd88 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:08:33 +0100 Subject: [PATCH 20/79] chore: pin s2-python to dev release that uses Pydantic 1 (#89) chore: pin s2-python to dev release that uses Pydantic 1 (because HA doesn't support Pydantic 2 yet) Signed-off-by: Vlad Iftime --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 03e935f2..c586ddba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp pandas pydantic>=1.10.8,<2.0 - s2-python + s2-python==0.2.0.dev2 async_timeout [options.packages.find] From 42266aabea1a064bbe08913c7cc7246e32210aa5 Mon Sep 17 00:00:00 2001 From: VladIftime <49650168+VladIftime@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:52:11 +0100 Subject: [PATCH 21/79] Update README.rst Signed-off-by: VladIftime <49650168+VladIftime@users.noreply.github.com> Signed-off-by: Vlad Iftime --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2d4fd955..3785061d 100644 --- a/README.rst +++ b/README.rst @@ -141,7 +141,7 @@ Making Changes & Contributing Install the project locally (in a virtual environment of your choice):: - pip install -e + pip install -e . Running tests locally is crucial as well. Staying close to the CI workflow:: From d00a88253e071d9583741ff4f25a028a1bc07ad6 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Tue, 10 Dec 2024 21:01:21 +0100 Subject: [PATCH 22/79] client.get_accout() Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 27 +++++++++++ src/flexmeasures_client/s2/cem.py | 9 ++-- .../s2/script/demo_setup.py | 4 +- .../s2/script/websockets_client.py | 45 +++++++++++++------ .../s2/script/websockets_server.py | 10 +++-- tests/test_client.py | 16 +++++++ tests/test_s2_coordinator.py | 2 +- 7 files changed, 89 insertions(+), 24 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 0360d38e..c3f403c7 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -269,6 +269,7 @@ async def post_measurements( prior: str | None = None, ): """ + This function raises a ContentTypeError when the response is not a dictionary. Post sensor data for the given time range. This function raises a ValueError when an unhandled status code is returned """ @@ -321,6 +322,32 @@ async def get_schedule( ) return schedule + async def get_account(self) -> list[dict]: + """Get the account of the current user. + + :returns: account as dictionary, for example: + { + 'id': 1, + 'name': 'FlexMeasures', + + } + """ + + account_data, status = await self.request(uri="accounts", method="GET") + check_for_status(status, 200) + + # Return just the 'account_roles','id' and 'name' fields + account_data = [ + { + "account_roles": account["account_roles"], + "id": account["id"], + "name": account["name"], + } + for account in account_data + ] + + return account_data + async def get_assets(self) -> list[dict]: """Get all the assets available to the current user. diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index c64bb828..56384dea 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -42,7 +42,10 @@ class CEM(Handler): _sending_queue: Queue[pydantic.BaseModel] def __init__( - self, fm_client: FlexMeasuresClient, logger: Logger | None = None + self, + sensor_id: int, + fm_client: FlexMeasuresClient, + logger: Logger | None = None, ) -> None: """ Customer Energy Manager (CEM) @@ -223,9 +226,7 @@ def handle_handshake(self, message: Handshake): def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message - if ( - not self._control_type - ): # initializing. TODO: check if sending resource_manager_details + if not self._control_type: # TODO: check if sending resource_manager_details # resets control type self._control_type = ControlType.NO_SELECTION diff --git a/src/flexmeasures_client/s2/script/demo_setup.py b/src/flexmeasures_client/s2/script/demo_setup.py index 3524c037..6c3fb430 100644 --- a/src/flexmeasures_client/s2/script/demo_setup.py +++ b/src/flexmeasures_client/s2/script/demo_setup.py @@ -3,8 +3,8 @@ from flexmeasures_client.client import FlexMeasuresClient client = FlexMeasuresClient( - email="admin@admin.com", - password="admin", + email="toy-user@flexmeasures.io", + password="toy-password", host="localhost:5000", ) diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index cf3f0cc3..9773cae3 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -3,34 +3,51 @@ import aiohttp import pytz - -from flexmeasures_client.s2.python_s2_protocol.common.messages import ( - Handshake, - ReceptionStatus, - ReceptionStatusValues, - ResourceManagerDetails, -) -from flexmeasures_client.s2.python_s2_protocol.common.schemas import ( +from s2python.common import ( Commodity, CommodityQuantity, ControlType, Duration, EnergyManagementRole, + Handshake, NumberRange, PowerRange, + ReceptionStatus, + ReceptionStatusValues, + ResourceManagerDetails, Role, RoleType, ) -from flexmeasures_client.s2.python_s2_protocol.FRBC.messages import ( - FRBCStorageStatus, - FRBCSystemDescription, -) -from flexmeasures_client.s2.python_s2_protocol.FRBC.schemas import ( +from s2python.frbc import ( FRBCActuatorDescription, FRBCOperationMode, FRBCOperationModeElement, FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, ) + +# from flexmeasures_client.s2.python_s2_protocol.common.schemas import ( +# Commodity, +# CommodityQuantity, +# ControlType, +# Duration, +# EnergyManagementRole, +# NumberRange, +# PowerRange, +# Role, +# RoleType, +# ) +# from flexmeasures_client.s2.python_s2_protocol.FRBC.messages import ( +# FRBCStorageStatus, +# FRBCSystemDescription, +# ) +# from flexmeasures_client.s2.python_s2_protocol.FRBC.schemas import ( +# FRBCActuatorDescription, +# FRBCOperationMode, +# FRBCOperationModeElement, +# FRBCStorageDescription, +# ) from flexmeasures_client.s2.utils import get_unique_id @@ -59,7 +76,7 @@ async def main_s2(): roles=[ Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) ], - instruction_processing_delay=Duration(__root__=1.0), + instruction_processing_delay=Duration(__root__=1), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 8591f6a8..c65f9373 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -3,11 +3,13 @@ import aiohttp from aiohttp import web +from s2python.common import ControlType from flexmeasures_client.client import FlexMeasuresClient from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple -from flexmeasures_client.s2.python_s2_protocol.common.schemas import ControlType + +# from flexmeasures_client.s2.python_s2_protocol.common.schemas import ControlType async def rm_details_watchdog(ws, cem: CEM): @@ -71,8 +73,10 @@ async def websocket_handler(request): fm_client = FlexMeasuresClient("toy-password", "toy-user@flexmeasures.io") - cem = CEM(sensor_id=1, fm_client=fm_client) - frbc = FRBCSimple(power_sensor_id=1, price_sensor_id=2) + cem = CEM(fm_client=fm_client) + frbc = FRBCSimple( + power_sensor_id=1, price_sensor_id=2, soc_sensor_id=3, rm_discharge_sensor_id=4 + ) cem.register_control_type(frbc) # create "parallel" tasks for the message producer and consumer diff --git a/tests/test_client.py b/tests/test_client.py index d7f774db..4d7dd24d 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -495,6 +495,22 @@ async def test_trigger_and_get_schedule() -> None: assert schedule["values"] == [2.15, 3, 2] +@pytest.mark.asyncio +async def test_get_account() -> None: + with aioresponses() as m: + m.get( + "http://localhost:5000/api/v3_0/accounts", + status=200, + payload=[{"name": "Toy Account"}], + ) + flexmeasures_client = FlexMeasuresClient( + email="toy-user@flexmeasures.io", + password="toy-password", + ) + account = await flexmeasures_client.get_account() + assert account["name"] == "Toy Account" + + @pytest.mark.asyncio async def test_get_sensor_data() -> None: with aioresponses() as m: diff --git a/tests/test_s2_coordinator.py b/tests/test_s2_coordinator.py index 9903e4ac..cfb1c0f0 100644 --- a/tests/test_s2_coordinator.py +++ b/tests/test_s2_coordinator.py @@ -33,7 +33,7 @@ @pytest.mark.asyncio async def test_cem(): # TODO: move into different test functions - cem = CEM(fm_client=None) + cem = CEM(sensor_id=1, fm_client=None) frbc = FRBCTest() cem.register_control_type(frbc) From e62f9910372ccd3f6f2102c0ead2fbc54febd3c1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Dec 2024 21:50:28 +0100 Subject: [PATCH 23/79] fix: mock user authentication by setting a dummy access_token Signed-off-by: F.N. Claessen Signed-off-by: Vlad Iftime --- tests/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_client.py b/tests/test_client.py index 4d7dd24d..766dc7fe 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -507,6 +507,7 @@ async def test_get_account() -> None: email="toy-user@flexmeasures.io", password="toy-password", ) + flexmeasures_client.access_token = "test-token" account = await flexmeasures_client.get_account() assert account["name"] == "Toy Account" From 9fbefd4d54f8640a15cd8f8077e1c0dc73fac15c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Dec 2024 22:11:25 +0100 Subject: [PATCH 24/79] feat: lacking a dedicated API endpoint, we fetch all accessible users first, then pick out ourselves, locate our account ID, and then fetch our organisation account info (two API calls needed, I'm afraid) Signed-off-by: F.N. Claessen Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 36 +++++++++++++-------------- tests/test_client.py | 41 ++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index c3f403c7..6fd024fc 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -322,31 +322,31 @@ async def get_schedule( ) return schedule - async def get_account(self) -> list[dict]: - """Get the account of the current user. + async def get_account(self) -> dict | None: + """Get the organisation account of the current user. - :returns: account as dictionary, for example: + :returns: organisation account as dictionary, for example: { - 'id': 1, - 'name': 'FlexMeasures', - + "id": 1, + "name": "Positive Design", } """ - account_data, status = await self.request(uri="accounts", method="GET") + users, status = await self.request(uri="users", method="GET") check_for_status(status, 200) - # Return just the 'account_roles','id' and 'name' fields - account_data = [ - { - "account_roles": account["account_roles"], - "id": account["id"], - "name": account["name"], - } - for account in account_data - ] - - return account_data + account_id = None + for user in users: + if user["email"] == self.email: + account_id = user["account_id"] + if account_id is None: + raise NotImplementedError(f"User does not seem to belong to account, which should not be possible.") + account, status = await self.request( + uri=f"accounts/{account_id}", + method="GET", + ) + check_for_status(status, 200) + return account async def get_assets(self) -> list[dict]: """Get all the assets available to the current user. diff --git a/tests/test_client.py b/tests/test_client.py index 766dc7fe..7853e3e9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -499,17 +499,50 @@ async def test_trigger_and_get_schedule() -> None: async def test_get_account() -> None: with aioresponses() as m: m.get( - "http://localhost:5000/api/v3_0/accounts", + "http://localhost:5000/api/v3_0/users", status=200, - payload=[{"name": "Toy Account"}], + payload=[ + { + "account_id": 1, + "active": True, + "email": "toy-user@flexmeasures.io", + "id": 39, + "username": "toy-user", + }, + { + "account_id": 1, + "active": True, + "email": "toy-colleague@flexmeasures.io", + "id": 40, + "username": "toy-colleague", + }, + { + "account_id": 2, + "active": True, + "email": "toy-client@flexmeasures.io", + "id": 41, + "username": "toy-client", + }, + ] + ) + m.get( + "http://localhost:5000/api/v3_0/accounts/1", + status=200, + payload={ + "id": 1, + "name": "Positive Design", + } ) flexmeasures_client = FlexMeasuresClient( + host="localhost", + port=5000, email="toy-user@flexmeasures.io", password="toy-password", ) flexmeasures_client.access_token = "test-token" - account = await flexmeasures_client.get_account() - assert account["name"] == "Toy Account" + account = await flexmeasures_client.get_account() + assert account["id"] == 1 + assert account["name"] == "Positive Design" @pytest.mark.asyncio From adc97f4187310ab4b540d7eeea593a6203dc3eb8 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Fri, 13 Dec 2024 14:48:45 +0100 Subject: [PATCH 25/79] PPBC Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 10 +++- .../s2/control_types/PPBC/__init__.py | 4 ++ .../s2/control_types/PPBC/ppbc_simple.py | 52 +++++++++++++++++++ .../s2/control_types/PPBC/utils.py | 0 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 src/flexmeasures_client/s2/control_types/PPBC/__init__.py create mode 100644 src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py create mode 100644 src/flexmeasures_client/s2/control_types/PPBC/utils.py diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 6fd024fc..956c1730 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -340,11 +340,19 @@ async def get_account(self) -> dict | None: if user["email"] == self.email: account_id = user["account_id"] if account_id is None: - raise NotImplementedError(f"User does not seem to belong to account, which should not be possible.") + raise NotImplementedError( + "User does not seem to belong to account, which should not be possible." + ) + # Force account to be a dictionary + account, status = await self.request( uri=f"accounts/{account_id}", method="GET", ) + if not isinstance(account, dict): + raise ContentTypeError( + f"Expected an account dictionary, but got {type(account)}", + ) check_for_status(status, 200) return account diff --git a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py new file mode 100644 index 00000000..2c989303 --- /dev/null +++ b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py @@ -0,0 +1,4 @@ +# from s2python.ppbc import PPBCScheduleInstruction + +# from flexmeasures_client.s2.control_types import ControlTypeHandler +# from flexmeasures_client.s2.utils import get_reception_status, get_unique_id diff --git a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py new file mode 100644 index 00000000..2e98863c --- /dev/null +++ b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py @@ -0,0 +1,52 @@ +""" +This module contains the PPBC simple control type. +""" + +from datetime import datetime, timedelta + +import pytz +from s2python.ppbc import PPBCScheduleInstruction + +from flexmeasures_client.s2.control_types.PPBC import PPBC + + +class PPBCSimple(PPBC): + _power_sensor_id: int + _price_sensor_id: int + _schedule_duration: timedelta + _valid_from_shift: timedelta + + def __init__( + self, + power_sensor_id: int, + price_sensor_id: int, + timezone: str = "UTC", + schedule_duration: timedelta = timedelta(hours=12), + max_size: int = 100, + valid_from_shift: timedelta = timedelta(days=1), + ) -> None: + super().__init__(max_size) + self._power_sensor_id = power_sensor_id + self._price_sensor_id = price_sensor_id + self._schedule_duration = schedule_duration + self._timezone = pytz.timezone(timezone) + + # delay the start of the schedule from the time `valid_from` + # of the PPBC.SystemDescritption. + self._valid_from_shift = valid_from_shift + + def now(self): + return self._timezone.localize(datetime.now()) + + async def send_schedule_instruction(self, instruction: PPBCScheduleInstruction): + await self._fm_client.post_schedule( + self._power_sensor_id, + start=self.now(), + values=instruction.power_values, + unit="MW", + duration=self._schedule_duration, + price_sensor_id=self._price_sensor_id, + price_values=instruction.price_values, + price_unit="EUR/MWh", + valid_from=self.now() + self._valid_from_shift, + ) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/utils.py b/src/flexmeasures_client/s2/control_types/PPBC/utils.py new file mode 100644 index 00000000..e69de29b From b33b406f2624f5243fd032dbb4a6ae53bd634d79 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Fri, 13 Dec 2024 14:55:17 +0100 Subject: [PATCH 26/79] applied black formatting Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 956c1730..c2b6dd0e 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -351,7 +351,7 @@ async def get_account(self) -> dict | None: ) if not isinstance(account, dict): raise ContentTypeError( - f"Expected an account dictionary, but got {type(account)}", + f"Expected an account dictionary! but got {type(account)}", ) check_for_status(status, 200) return account From fd341092e6b0eade5ca552ed5b7d488335af03d5 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Fri, 13 Dec 2024 14:57:33 +0100 Subject: [PATCH 27/79] applied black formatting test_client.py Signed-off-by: Vlad Iftime --- tests/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 7853e3e9..6ffbf487 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -523,7 +523,7 @@ async def test_get_account() -> None: "id": 41, "username": "toy-client", }, - ] + ], ) m.get( "http://localhost:5000/api/v3_0/accounts/1", @@ -531,7 +531,7 @@ async def test_get_account() -> None: payload={ "id": 1, "name": "Positive Design", - } + }, ) flexmeasures_client = FlexMeasuresClient( host="localhost", From 9b65284350dfa6b822fb62a1c7d69fe0f1d55b2d Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Tue, 17 Dec 2024 10:39:41 +0200 Subject: [PATCH 28/79] Adding PPBC importer and CEM test Signed-off-by: Vlad Iftime --- .../s2/control_types/FRBC/__init__.py | 2 + .../s2/control_types/PPBC/__init__.py | 44 +++++- tests/test_s2_coordinator.py | 146 +++++++++++++++++- 3 files changed, 185 insertions(+), 7 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index e61e61a9..da150b18 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -41,6 +41,8 @@ def __init__(self, max_size: int = 100) -> None: self._fill_level_target_profile_history = SizeLimitOrderedDict( max_size=max_size ) + + # ! ASK: Why are these not being used? self._leakage_behaviour_history = SizeLimitOrderedDict(max_size=max_size) self._usage_forecast_history = SizeLimitOrderedDict(max_size=max_size) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py index 2c989303..b2f05836 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py @@ -1,4 +1,44 @@ -# from s2python.ppbc import PPBCScheduleInstruction +import pydantic +from s2python.common import ControlType, ReceptionStatusValues +from s2python.ppbc import PPBCEndInterruptionInstruction, PPBCScheduleInstruction + +from flexmeasures_client.s2 import SizeLimitOrderedDict, register +from flexmeasures_client.s2.control_types import ControlTypeHandler +from flexmeasures_client.s2.utils import get_reception_status -# from flexmeasures_client.s2.control_types import ControlTypeHandler # from flexmeasures_client.s2.utils import get_reception_status, get_unique_id + + +class PPBC(ControlTypeHandler): + _control_type = ControlType.POWER_PROFILE_BASED_CONTROL + + _schedule_instruction_history: SizeLimitOrderedDict[str, PPBCScheduleInstruction] + _end_interruption_instruction_history: SizeLimitOrderedDict[ + str, PPBCEndInterruptionInstruction + ] + + def __init__(self, max_size: int = 100) -> None: + super().__init__(max_size) + + self._schedule_instruction_history = SizeLimitOrderedDict(max_size=max_size) + self._end_interruption_instruction_history = SizeLimitOrderedDict( + max_size=max_size + ) + + @register(PPBCScheduleInstruction) + def handle_schedule_instruction( + self, message: PPBCScheduleInstruction + ) -> pydantic.BaseModel: + schedule_instruction_id = str(message.message_id) + self._schedule_instruction_history[schedule_instruction_id] = message + return get_reception_status(message, status=ReceptionStatusValues.OK) + + @register(PPBCEndInterruptionInstruction) + def handle_end_interruption_instruction( + self, message: PPBCEndInterruptionInstruction + ) -> pydantic.BaseModel: + end_interruption_instruction_id = str(message.message_id) + self._end_interruption_instruction_history[ + end_interruption_instruction_id + ] = message + return get_reception_status(message, status=ReceptionStatusValues.OK) diff --git a/tests/test_s2_coordinator.py b/tests/test_s2_coordinator.py index cfb1c0f0..c4a48b63 100644 --- a/tests/test_s2_coordinator.py +++ b/tests/test_s2_coordinator.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timezone import pytest from s2python.common import ( @@ -28,15 +28,18 @@ from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC import FRBCTest +from flexmeasures_client.s2.control_types.PPBC import PPBC from flexmeasures_client.s2.utils import get_unique_id @pytest.mark.asyncio -async def test_cem(): # TODO: move into different test functions +async def test_cem_frbc(): # TODO: move into different test functions cem = CEM(sensor_id=1, fm_client=None) frbc = FRBCTest() - + ppbc = PPBC() cem.register_control_type(frbc) + cem.register_control_type(ppbc) + # Show that this point is reached ############# # Handshake # @@ -71,9 +74,10 @@ async def test_cem(): # TODO: move into different test functions message_id=get_unique_id(), resource_id=get_unique_id(), roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], - instruction_processing_delay=Duration(__root__=1.0), + instruction_processing_delay=Duration(1), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, + ControlType.POWER_PROFILE_BASED_CONTROL, ControlType.NO_SELECTION, ], provides_forecast=True, @@ -156,7 +160,7 @@ async def test_cem(): # TODO: move into different test functions system_description_message = FRBCSystemDescription( message_id=get_unique_id(), - valid_from=datetime.now(), + valid_from=datetime.now(timezone.utc), actuators=[actuator], storage=storage, ) @@ -199,3 +203,135 @@ async def test_cem(): # TODO: move into different test functions ControlType.FILL_RATE_BASED_CONTROL ].success_callbacks ), "success callback should be deleted" + + +# @pytest.mark.asyncio +# async def test_cem_ppbc(): # TODO: move into different test functions +# cem = CEM(sensor_id=1, fm_client=None) +# ppbc = PPBC() +# cem.register_control_type(ppbc) +# # Show that this point is reached + +# ############# +# # Handshake # +# ############# + +# handshake_message = Handshake( +# message_id=get_unique_id(), +# role=EnergyManagementRole.RM, +# supported_protocol_versions=["0.1.0"], +# ) + +# await cem.handle_message(handshake_message) + +# assert ( +# cem._sending_queue.qsize() == 1 +# ) # check that message is put to the outgoing queue + +# response = await cem.get_message() + +# assert ( +# response["message_type"] == "HandshakeResponse" +# ), "response message_type should be HandshakeResponse" +# assert ( +# response["selected_protocol_version"] == "0.1.0" +# ), "CEM selected protocol version should be supported by the Resource Manager" + +# ########################## +# # ResourceManagerDetails # +# ########################## + +# resource_manager_details_message = ResourceManagerDetails( +# message_id=get_unique_id(), +# resource_id=get_unique_id(), +# roles=[Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY)], +# instruction_processing_delay=Duration(1), +# available_control_types=[ +# ControlType.FILL_RATE_BASED_CONTROL, +# ControlType.POWER_PROFILE_BASED_CONTROL, +# ControlType.NO_SELECTION, +# ], +# provides_forecast=True, +# provides_power_measurement_types=[ +# CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC +# ], +# ) + +# await cem.handle_message(resource_manager_details_message) +# response = await cem.get_message() + +# assert response["message_type"] == "ReceptionStatus" +# assert response["status"] == "OK" +# assert ( +# cem._resource_manager_details == resource_manager_details_message +# ), "CEM should store the resource_manager_details" +# assert cem.control_type == ControlType.NO_SELECTION, ( +# "CEM control type should switch to ControlType.NO_SELECTION," +# "independently of the original type" +# ) + +# ######################### +# # Activate control type # +# ######################### + +# await cem.activate_control_type(ControlType.POWER_PROFILE_BASED_CONTROL) +# message = await cem.get_message() + +# assert cem.control_type == ControlType.NO_SELECTION, ( +# "the control type should still be NO_SELECTION (rather than PPBC)," +# " because the RM has not yet confirmed FRBC activation" +# ) + +# response = ReceptionStatus( +# subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK +# ) + +# await cem.handle_message(response) + +# assert ( +# cem.control_type == ControlType.POWER_PROFILE_BASED_CONTROL +# ), "after a positive ResponseStatus, the status changes from NO_SELECTION to FRBC" + +# ######## +# # PPBC # +# ######## + + +# await cem.handle_message(system_description_message) +# response = await cem.get_message() + +# # checking that FRBC handler is being called +# assert ( +# cem._control_types_handlers[ +# ControlType.POWER_PROFILE_BASED_CONTROL +# ]._system_description_history[str(system_description_message.message_id)] +# == system_description_message +# ), ( +# "the FRBC.SystemDescription message should be stored" +# "in the frbc.system_description_history variable" +# ) + +# # change of control type is not performed in case that the RM answers +# # with a negative response +# await cem.activate_control_type(ControlType.NO_SELECTION) +# response = await cem.get_message() +# assert ( +# cem._control_type == ControlType.FILL_RATE_BASED_CONTROL +# ), "control type should not change, confirmation still pending" + +# await cem.handle_message( +# ReceptionStatus( +# subject_message_id=response.get("message_id"), +# status=ReceptionStatusValues.INVALID_CONTENT, +# ) +# ) + +# assert ( +# cem._control_type == ControlType.FILL_RATE_BASED_CONTROL +# ), "control type should not change, confirmation state is not 'OK'" +# assert ( +# response.get("message_id") +# not in cem._control_types_handlers[ +# ControlType.FILL_RATE_BASED_CONTROL +# ].success_callbacks +# ), "success callback should be deleted" From e5e26478497d7f9fa18b7622c995cd230df0281f Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 7 Aug 2024 15:56:26 +0200 Subject: [PATCH 29/79] feat: create FillRateBasedControlTUNES which implements an ad-hoc FRBC control type to work only with TUNES RM (for now). Signed-off-by: Victor Garcia Reolid Signed-off-by: Vlad Iftime --- README.rst | 6 + setup.cfg | 3 +- src/flexmeasures_client/client.py | 2 + src/flexmeasures_client/s2/__init__.py | 2 + src/flexmeasures_client/s2/cem.py | 9 +- .../s2/control_types/FRBC/__init__.py | 86 +++- .../s2/control_types/FRBC/frbc_simple.py | 17 - .../s2/control_types/FRBC/frbc_tunes.py | 375 ++++++++++++++++++ .../s2/control_types/translations.py | 193 +++++++++ src/flexmeasures_client/s2/wrapper.py | 13 + tests/conftest.py | 129 +++++- tests/test_cem.py | 201 ++++++++++ tests/test_frbc_tunes.py | 272 +++++++++++++ tests/test_s2_coordinator.py | 337 ---------------- tests/test_s2_models.py | 31 ++ tests/test_s2_translations.py | 78 ++++ 16 files changed, 1372 insertions(+), 382 deletions(-) create mode 100644 src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py create mode 100644 src/flexmeasures_client/s2/control_types/translations.py create mode 100644 src/flexmeasures_client/s2/wrapper.py create mode 100644 tests/test_cem.py create mode 100644 tests/test_frbc_tunes.py delete mode 100644 tests/test_s2_coordinator.py create mode 100644 tests/test_s2_models.py create mode 100644 tests/test_s2_translations.py diff --git a/README.rst b/README.rst index 3785061d..4de3975e 100644 --- a/README.rst +++ b/README.rst @@ -130,6 +130,12 @@ The schedule returns a Pandas ``DataFrame`` that can be used to regulate the fle +Development +============== + +If you want to develop this package it's necessary to install testing requirements:: + + pip install -e ".[testing]" .. _pyscaffold-notes: diff --git a/setup.cfg b/setup.cfg index c586ddba..33edbddb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,12 +49,13 @@ python_requires >= 3.9 # For more information, check out https://semver.org/. install_requires = importlib-metadata; python_version<"3.8" - aiohttp + aiohttp<=3.9.1 pandas pydantic>=1.10.8,<2.0 s2-python==0.2.0.dev2 async_timeout + [options.packages.find] where = src exclude = diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index c2b6dd0e..5e97ce10 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -59,6 +59,8 @@ class FlexMeasuresClient: session: ClientSession | None = None def __post_init__(self): + self.session: ClientSession = ClientSession() + if not re.match(r".+\@.+\..+", self.email): raise EmailValidationError( f"{self.email} is not an email address format string" diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index 42f68135..bee45f37 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -80,6 +80,8 @@ class Handler: outgoing_messages_status: SizeLimitOrderedDict + background_tasks: set + def __init__(self, max_size: int = 100) -> None: """ Handler diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 56384dea..d9a1021c 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -66,9 +66,13 @@ def __init__( def supports_control_type(self, control_type: ControlType): return control_type in self._resource_manager_details.available_control_types - def close(self): + async def close(self): self._is_closed = True + for control_type, handler in self._control_types_handlers.items(): + print(control_type, handler) + await handler.close() + def is_closed(self): return self._is_closed @@ -274,3 +278,6 @@ def handle_revoke_object(self, message: RevokeObject): ) return get_reception_status(message, ReceptionStatusValues.OK) + + async def send_message(self, message): + await self._sending_queue.put(message) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index da150b18..2b51a75b 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -32,6 +32,7 @@ class FRBC(ControlTypeHandler): _timer_status_history: SizeLimitOrderedDict[str, FRBCTimerStatus] _actuator_status_history: SizeLimitOrderedDict[str, FRBCActuatorStatus] _storage_status_history: SizeLimitOrderedDict[str, FRBCStorageStatus] + background_tasks: set def __init__(self, max_size: int = 100) -> None: super().__init__(max_size) @@ -53,6 +54,7 @@ def __init__(self, max_size: int = 100) -> None: self._system_description_history = SizeLimitOrderedDict(max_size=max_size) self._leakage_behaviour_history = SizeLimitOrderedDict(max_size=max_size) self._usage_forecast_history = SizeLimitOrderedDict(max_size=max_size) + self.background_tasks = set() @register(FRBCSystemDescription) def handle_system_description( @@ -64,15 +66,25 @@ def handle_system_description( self._system_description_history[system_description_id] = message # schedule trigger_schedule to run soon concurrently - asyncio.create_task(self.trigger_schedule(system_description_id)) - + task = asyncio.create_task(self.trigger_schedule(system_description_id)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) - async def send_storage_status(self, status: FRBCStorageStatus): - raise NotImplementedError() + @register(FRBCUsageForecast) + def handle_usage_forecast(self, message: FRBCUsageForecast) -> pydantic.BaseModel: + message_id = str(message.message_id) - async def send_actuator_status(self, status: FRBCActuatorStatus): - raise NotImplementedError() + self._usage_forecast_history[message_id] = message + + task = asyncio.create_task(self.send_usage_forecast(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCStorageStatus) def handle_storage_status(self, message: FRBCStorageStatus) -> pydantic.BaseModel: @@ -80,8 +92,11 @@ def handle_storage_status(self, message: FRBCStorageStatus) -> pydantic.BaseMode self._storage_status_history[message_id] = message - asyncio.create_task(self.send_storage_status(message)) - + task = asyncio.create_task(self.send_storage_status(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCActuatorStatus) @@ -90,29 +105,64 @@ def handle_actuator_status(self, message: FRBCActuatorStatus) -> pydantic.BaseMo self._actuator_status_history[message_id] = message - asyncio.create_task(self.send_actuator_status(message)) - + task = asyncio.create_task(self.send_actuator_status(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCLeakageBehaviour) def handle_leakage_behaviour( self, message: FRBCLeakageBehaviour ) -> pydantic.BaseModel: - # return get_reception_status(message, status=ReceptionStatusValues.OK) - raise NotImplementedError() + message_id = str(message.message_id) - @register(FRBCUsageForecast) - def handle_usage_forecast(self, message: FRBCUsageForecast) -> pydantic.BaseModel: - # return get_reception_status(message, status=ReceptionStatusValues.OK) - raise NotImplementedError() + self._leakage_behaviour_history[message_id] = message - async def trigger_schedule(self, system_description_id: str): - raise NotImplementedError() + task = asyncio.create_task(self.send_leakage_behaviour(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) + + @register(FRBCFillLevelTargetProfile) + def handle_fill_level_target_profile( + self, message: FRBCFillLevelTargetProfile + ) -> pydantic.BaseModel: + message_id = str(message.message_id) + + self._fill_level_target_profile_history[message_id] = message + + task = asyncio.create_task(self.send_fill_level_target_profile(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCTimerStatus) def handle_frbc_timer_status(self, message: FRBCTimerStatus) -> pydantic.BaseModel: return get_reception_status(message, status=ReceptionStatusValues.OK) + async def send_storage_status(self, status: FRBCStorageStatus): + raise NotImplementedError() + + async def send_actuator_status(self, status: FRBCActuatorStatus): + raise NotImplementedError() + + async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): + raise NotImplementedError() + + async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): + raise NotImplementedError() + + async def send_fill_level_target_profile( + self, fill_level_target_profile: FRBCFillLevelTargetProfile + ): + raise NotImplementedError() + class FRBCTest(FRBC): """Dummy class to simulate the triggering of a schedule.""" diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 85acb557..0b023b73 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -75,23 +75,6 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - # await self._fm_client.post_measurements( - # self._soc_sensor_id - # ) - - # system_description = self.find_system_description_from_actuator() - - # if system_description is None: - # return - - # #for a - # if system_description is not None: - - # self._system_description_history[] - # status.active_operation_mode_id - # status.actuator_id - # status.operation_mode_factor - async def trigger_schedule(self, system_description_id: str): """Translates S2 System Description into FM API calls""" diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py new file mode 100644 index 00000000..a31fd177 --- /dev/null +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -0,0 +1,375 @@ +# flake8: noqa +""" +This control type is in a very EXPERIMENTAL stage. +Used it at your own risk :) +""" + +import asyncio +from datetime import datetime, timedelta + +import pandas as pd +import pydantic +import pytz +from s2python.common import NumberRange, ReceptionStatusValues +from s2python.frbc import ( + FRBCActuatorStatus, + FRBCFillLevelTargetProfile, + FRBCInstruction, + FRBCStorageStatus, + FRBCSystemDescription, + FRBCUsageForecast, +) + +from flexmeasures_client.s2 import register +from flexmeasures_client.s2.control_types.FRBC import FRBC +from flexmeasures_client.s2.control_types.translations import ( + translate_fill_level_target_profile, + translate_usage_forecast_to_fm, +) +from flexmeasures_client.s2.utils import get_reception_status, get_unique_id + +RESOLUTION = "15min" +POWER_UNIT = "MWh" +DIMENSIONLESS = "dimensionless" +PERCENTAGE = "%" +TASK_PERIOD_SECONDS = 2 +CONVERSION_EFFICIENCY_DURATION = "PT24H" + + +class FillRateBasedControlTUNES(FRBC): + _fill_level_sensor_id: int | None + + _fill_rate_sensor_id: int | None + _thp_fill_rate_sensor_id: int | None + _thp_efficiency_sensor_id: int | None + _nes_fill_rate_sensor_id: int | None + _nes_efficiency_sensor_id: int | None + + _schedule_duration: timedelta + + _usage_forecast_sensor_id: int | None + _soc_minima_sensor_id: int | None + _soc_maxima_sensor_id: int | None + _rm_discharge_sensor_id: int | None + + def __init__( + self, + soc_minima_sensor_id: int | None = None, + soc_maxima_sensor_id: int | None = None, + fill_level_sensor_id: int | None = None, + usage_forecast_sensor_id: int | None = None, + thp_fill_rate_sensor_id: int | None = None, + thp_efficiency_sensor_id: int | None = None, + nes_fill_rate_sensor_id: int | None = None, + nes_efficiency_sensor_id: int | None = None, + fill_rate_sensor_id: int | None = None, + rm_discharge_sensor_id: int | None = None, + timezone: str = "UTC", + schedule_duration: timedelta = timedelta(hours=12), + max_size: int = 100, + valid_from_shift: timedelta = timedelta(days=1), + **kwargs + ) -> None: + super().__init__(max_size) + + self._fill_level_sensor_id = fill_level_sensor_id + + self._fill_rate_sensor_id = fill_rate_sensor_id + self._thp_fill_rate_sensor_id = thp_fill_rate_sensor_id + self._thp_efficiency_sensor_id = thp_efficiency_sensor_id + self._nes_fill_rate_sensor_id = nes_fill_rate_sensor_id + self._nes_efficiency_sensor_id = nes_efficiency_sensor_id + + self._schedule_duration = schedule_duration + + self._usage_forecast_sensor_id = usage_forecast_sensor_id + self._soc_minima_sensor_id = soc_minima_sensor_id + self._soc_maxima_sensor_id = soc_maxima_sensor_id + self._rm_discharge_sensor_id = rm_discharge_sensor_id + + self._timezone = pytz.timezone(timezone) + + # delay the start of the schedule from the time `valid_from` + # of the FRBC.SystemDescritption + self._valid_from_shift = valid_from_shift + + self._active_recurring_schedule = False + + def now(self): + return self._timezone.localize(datetime.now()) + + async def send_storage_status(self, status: FRBCStorageStatus): + await self._fm_client.post_measurements( + self._fill_level_sensor_id, + start=self.now(), + values=[status.present_fill_level], + unit=POWER_UNIT, + duration=timedelta(minutes=0), # INSTANTANEOUS + ) + + async def send_actuator_status(self, status: FRBCActuatorStatus): + factor = status.operation_mode_factor + system_description: FRBCSystemDescription = list( + self._system_description_history.values() + )[-1] + + # find the active FRBCOperationMode + for op_pos, operation_mode in enumerate( + system_description.actuators[0].operation_modes + ): + if operation_mode.id == status.active_operation_mode_id: + break + + dt = status.transition_timestamp # self.now() + + # Assume that THP is op_pos = 0 and NES = op_pos = 1. + # TODO: should we rely on a sensor_id? For example, "nes-actuator-mode", "thp-actuator-mode" + if op_pos == 0: + active_operation_mode_fill_rate_sensor_id = self._thp_fill_rate_sensor_id + else: + active_operation_mode_fill_rate_sensor_id = self._nes_fill_rate_sensor_id + + # Operation Mode Factor to fill rate + fill_rate = operation_mode.elements[0].fill_rate + fill_rate = ( + fill_rate.start_of_range + + (fill_rate.end_of_range - fill_rate.start_of_range) * factor + ) + + # Send data to the sensor of the fill rate corresponding to the active operation mode + await self._fm_client.post_measurements( + sensor_id=active_operation_mode_fill_rate_sensor_id, + start=dt, + values=[fill_rate], + unit=POWER_UNIT, + duration=timedelta(0), + ) + + # Send data to the sensor of the input fill_rate to the storage device + await self._fm_client.post_measurements( + sensor_id=self._fill_rate_sensor_id, + start=dt, + values=[fill_rate], + unit=POWER_UNIT, + duration=timedelta(0), + ) + + async def start_trigger_schedule(self): + """ + Start a recurring task to create new schedules. + + This function ensures that the scheduling task is started only once. + """ + + if not self._active_recurring_schedule: + self._active_recurring_schedule = True + self._recurrent_task = asyncio.create_task(self.trigger_schedule_task()) + self.background_tasks.add( + self._recurrent_task + ) # important to avoid a task disappearing mid-execution. + self._recurrent_task.add_done_callback(self.background_tasks.discard) + + async def stop_trigger_schedule(self): + """ + Stop the recurring task that creates new schedules. + + This function ensures that the scheduling task is stopped gracefully. + """ + + if self._active_recurring_schedule: + self._active_recurring_schedule = False + self._recurrent_task.cancel() + + async def trigger_schedule_task(self): + """ + Recurring task to trigger the schedule creation process. + + This task runs continuously while the active recurring schedule is enabled. + """ + + while self._active_recurring_schedule: + await self.trigger_scehdule() + await asyncio.sleep(TASK_PERIOD_SECONDS) + + async def trigger_scehdule(self): + """ + Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager + """ + + # Retrieve the latest system description from history + system_description: FRBCSystemDescription = list( + self._system_description_history.values() + )[-1] + + actuator = system_description.actuators[0] + fill_level_range: NumberRange = system_description.storage.fill_level_range + + # get SOC Max and Min to be sent on the Flex Model + soc_min = fill_level_range.end_of_range + soc_max = fill_level_range.start_of_range + + operation_mode = actuator.operation_modes[0] + operation_mode_factor = 0.1 + + # TODO: 1) Call FlexMeasures + # TODO: 2) Select with which actuator to send the instruction + # TODO: 3) Create operation_mode_factor from power (we have a function for that) + + instruction = FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=actuator.id, + operation_mode=operation_mode.id, # Based on the expeted fill_level, select the best actuator (most efficient) to fulfill a certain fill_rate + operation_mode_factor=operation_mode_factor, + execution_time=self.now(), + abnormal_condition=False, + ) + + # Put the instruction in the sending queue + await self._sending_queue.put(instruction) + + @register(FRBCSystemDescription) + def handle_system_description( + self, message: FRBCSystemDescription + ) -> pydantic.BaseModel: + """ + Handle FRBC.SystemDescription messages. + + Process: + 1) Store system_description message for later. + 2) Send conversion efficiencies (COP) to FlexMeasures. + 3) Start a recurring tasks to trigger the scehduler. + """ + + system_description_id = str(message.message_id) + + # store system_description message for later + self._system_description_history[system_description_id] = message + + # send conversion efficiencies + task = asyncio.create_task(self.send_conversion_efficiencies(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + + # schedule trigger_schedule to run soon concurrently + task = asyncio.create_task(self.start_trigger_schedule()) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + + return get_reception_status(message, status=ReceptionStatusValues.OK) + + async def send_conversion_efficiencies( + self, system_description: FRBCSystemDescription + ): + """ + Send conversion efficiencies to FlexMeasures. + + Args: + system_description (FRBCSystemDescription): The system description containing actuator details. + """ + + start = system_description.valid_from + actuator = system_description.actuators[0] + + # Calculate the number of samples based on the conversion efficiency duration + N_SAMPLES = int( + pd.Timedelta(CONVERSION_EFFICIENCY_DURATION) / pd.Timedelta(RESOLUTION) + ) + + thp_op_mode_element = actuator.operation_modes[0].elements[-1] + nes_op_mode_element = actuator.operation_modes[1].elements[-1] + + # THP efficiencies: Calculate and post measurements for THP efficiencies + await self._fm_client.post_measurements( + sensor_id=self._thp_efficiency_sensor_id, + start=start, + values=[ + 100 + * thp_op_mode_element.fill_rate.end_of_range + / thp_op_mode_element.power_ranges[0].end_of_range + ] + * N_SAMPLES, + unit=PERCENTAGE, + duration=CONVERSION_EFFICIENCY_DURATION, + ) + + # NES efficiencies: Calculate and post measurements for NES efficiencies + await self._fm_client.post_measurements( + sensor_id=self._nes_efficiency_sensor_id, + start=start, + values=[ + 100 + * nes_op_mode_element.fill_rate.end_of_range + / nes_op_mode_element.power_ranges[0].end_of_range + ] + * N_SAMPLES, + unit=PERCENTAGE, + duration=CONVERSION_EFFICIENCY_DURATION, + ) + + async def close(self): + """ + Closing procedure: + 1) Stop recurrent task + """ + + await self.stop_trigger_schedule() + + async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): + """ + Send FRBC.UsageForecast to FlexMeasures. + + Args: + usage_forecast (FRBCUsageForecast): The usage forecast to be translated and sent. + """ + + usage_forecast = translate_usage_forecast_to_fm( + usage_forecast, RESOLUTION, strategy="mean" + ) + + await self._fm_client.post_measurements( + sensor_id=self._usage_forecast_sensor_id, + start=usage_forecast.start_time, + values=usage_forecast, + unit=POWER_UNIT, + duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), + ) + + async def send_fill_level_target_profile( + self, fill_level_target_profile: FRBCFillLevelTargetProfile + ): + """ + Send FRBC.FillLevelTargetProfile to FlexMeasures. + + Args: + fill_level_target_profile (FRBCFillLevelTargetProfile): The fill level target profile to be translated and sent. + """ + + soc_minima, soc_maxima = translate_fill_level_target_profile( + fill_level_target_profile, + resolution=RESOLUTION, + ) + + duration = str(pd.Timedelta(RESOLUTION) * len(soc_maxima)) + + # POST SOC Minima measurements to FlexMeasures + await self._fm_client.post_measurements( + sensor_id=self._soc_minima_sensor_id, + start=fill_level_target_profile.start_time, + values=soc_minima, + unit=POWER_UNIT, + duration=duration, + ) + + # POST SOC Maxima measurements to FlexMeasures + await self._fm_client.post_measurements( + sensor_id=self._soc_maxima_sensor_id, + start=fill_level_target_profile.start_time, + values=soc_maxima, + unit=POWER_UNIT, + duration=duration, + ) diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py new file mode 100644 index 00000000..f97c1a23 --- /dev/null +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -0,0 +1,193 @@ +# flake8: noqa + +from datetime import timedelta + +import numpy as np +import pandas as pd +from s2python.frbc import ( + FRBCFillLevelTargetProfile, + FRBCLeakageBehaviour, + FRBCUsageForecast, +) + + +def leakage_behaviour_to_storage_efficieny( + message: FRBCLeakageBehaviour, resolution=timedelta(minutes=15) +) -> float: + """ + Convert a FRBC.LeakeageBehaviour message into a FlexMeasures compatible storage efficiency. + + Definitions: + + LeakageBehaviour: how fast the momentary fill level will decrease per second + due to leakage within the given range of the fill level. This is defined as a function of the + fill level. + + Storage Efficiency: percentage of the storage that remains after one time period. + + Example: + + { + ..., + "elements" : [ + { + "fill_level_range" : {"start_of_range" : 0, "end_of_range" : 5}, + "leakage_rate" : 0 + }, + { + "fill_level_range" : {"start_of_range" : 5, "end_of_range" : 95}, + "leakage_rate" : 1/3600 + } + { + "fill_level_range" : {"start_of_range" : 95, "end_of_range" : 100}, + "leakage_rate" : 2/3600 + } + ] + } + + """ + + last_element = message.elements[-1] + return ( + 1 + - (resolution / timedelta(seconds=1)) + * last_element.leakage_rate + / last_element.fill_level_range.end_of_range + ) + + +def unevenly_ts_to_evenly( + start: pd.Timestamp, + values: list[float], + durations: list[pd.Timedelta], + target_resolution: str, + strategy="mean", +) -> pd.Series: + """ + Convert unevenly spaced time series data into evenly spaced data. + + The function will: + - Floor the start time to align with the target resolution. + - Ceil the end time to align with the target resolution. + - Interpolate and resample the data based on the chosen aggregation strategy. + + Args: + start (pd.Timestamp): The starting timestamp of the time series data. + values (list[float]): The list of values for each time period. + durations (list[pd.Timedelta]): The list of durations for each value. + target_resolution (str): The target time resolution for resampling. + strategy (str): Aggregation strategy ("mean", "min", "max", etc.) for resampling. + + Returns: + pd.Series: A Pandas Series with evenly spaced timestamps and interpolated values. + """ + + # Calculate the time from the absolute start of each event + deltas = pd.TimedeltaIndex(np.cumsum([timedelta(0)] + durations)) + + # Ceil the end time to align with the target resolution + end = pd.Timestamp(start + deltas[-1]).ceil(target_resolution) + + # Floor the start time to align with the target resolution + start = start.floor(target_resolution) + + # Create an index for the time series based on the start time and deltas + index = start + deltas + + # Make a copy of the values list and append a NaN to handle the end boundary + values = values.copy() + values.append(np.nan) + series = pd.Series(values, index) + + # Reindex the series with a regular time grid and forward-fill missing values + series = series.reindex( + pd.date_range( + start=start, + end=end, + freq=min(min(durations), pd.Timedelta(target_resolution)), + inclusive="left", + ) + ).ffill() + + # Resample the series to the target resolution using the specified aggregation strategy and forward-fill + series = series.resample(target_resolution).agg(strategy).ffill() + + return series + + +def translate_usage_forecast_to_fm( + usage_forecast: FRBCUsageForecast, + resolution: str = "1h", +) -> pd.Series: + """ + Translate a FRBC.UsageForecast into a FlexMeasures compatible format with evenly spaced data. + + Args: + usage_forecast (FRBCUsageForecast): The usage forecast message with start time and elements. + resolution (str): The target time resolution for resampling (e.g., "1h"). + + Returns: + pd.Series: A Pandas Series with evenly spaced timestamps and usage forecast values. + """ + + start = pd.Timestamp(usage_forecast.start_time) + + durations = [element.duration.to_timedelta() for element in usage_forecast.elements] + values = [element.usage_rate_expected for element in usage_forecast.elements] + + return unevenly_ts_to_evenly( + start=start, + values=values, + durations=durations, + target_resolution=resolution, + strategy="mean", + ) + + +def translate_fill_level_target_profile( + fill_level_target_profile: FRBCFillLevelTargetProfile, resolution: str = "1h" +) -> tuple[pd.Series, pd.Series]: + """ + Translate a FRBC.FillLevelTargetProfile into SOC minima and maxima compatible with FlexMeasures. + + Args: + fill_level_target_profile (FRBCFillLevelTargetProfile): The target profile message with start time and elements. + resolution (str): The target time resolution for resampling (e.g., "1h"). + + Returns: + tuple[pd.Series, pd.Series]: A tuple containing SOC minima and maxima as Pandas Series. + """ + + start = pd.Timestamp(fill_level_target_profile.start_time) + + durations = [ + element.duration.to_timedelta() + for element in fill_level_target_profile.elements + ] + + soc_minima_values = [ + element.fill_level_range.start_of_range + for element in fill_level_target_profile.elements + ] + soc_maxima_values = [ + element.fill_level_range.end_of_range + for element in fill_level_target_profile.elements + ] + + soc_minima = unevenly_ts_to_evenly( + start=start, + values=soc_minima_values, + durations=durations, + target_resolution=resolution, + strategy="min", + ) + + soc_maxima = unevenly_ts_to_evenly( + start=start, + values=soc_maxima_values, + durations=durations, + target_resolution=resolution, + strategy="max", + ) + + return soc_minima, soc_maxima diff --git a/src/flexmeasures_client/s2/wrapper.py b/src/flexmeasures_client/s2/wrapper.py new file mode 100644 index 00000000..0a4d9c06 --- /dev/null +++ b/src/flexmeasures_client/s2/wrapper.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from pydantic import BaseModel, Field +from s2python.message import S2Message + + +class MetaData(BaseModel): + dt: datetime + + +class S2Wrapper(BaseModel): + message: S2Message = Field(discriminator="message_type") + metadata: MetaData diff --git a/tests/conftest.py b/tests/conftest.py index e0657022..aca532f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,121 @@ -""" - Dummy conftest.py for flexmeasures_client. - - If you don't know what this is for, just leave it empty. - Read more about conftest.py under: - - https://docs.pytest.org/en/stable/fixture.html - - https://docs.pytest.org/en/stable/writing_plugins.html -""" +from __future__ import annotations + +from datetime import datetime + +import pytest +from s2python.common import ( + Commodity, + CommodityQuantity, + ControlType, + Duration, + EnergyManagementRole, + Handshake, + NumberRange, + PowerRange, + ResourceManagerDetails, + Role, + RoleType, +) +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCSystemDescription, +) + +from flexmeasures_client.s2.utils import get_unique_id + + +@pytest.fixture(scope="session") +def frbc_system_description(): + ######## + # FRBC # + ######## + + thp_operation_mode_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=80), + fill_rate=NumberRange(start_of_range=0, end_of_range=2), + power_ranges=[ + PowerRange( + start_of_range=10, + end_of_range=1000, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], + ) + + thp_operation_mode = FRBCOperationMode( + id="tarnoc-operation-mode", + elements=[thp_operation_mode_element], + abnormal_condition_only=False, + ) + + nes_operation_mode_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + fill_rate=NumberRange(start_of_range=0, end_of_range=1), + power_ranges=[ + PowerRange( + start_of_range=10, + end_of_range=1000, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], + ) + + nes_operation_mode = FRBCOperationMode( + id="nestore-operation-mode", + elements=[nes_operation_mode_element], + abnormal_condition_only=False, + ) + + actuator = FRBCActuatorDescription( + id="id-of-the-actuator", + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=[thp_operation_mode, nes_operation_mode], + transitions=[], + timers=[], + ) + + storage = FRBCStorageDescription( + provides_leakage_behaviour=True, + provides_fill_level_target_profile=True, + provides_usage_forecast=True, + fill_level_range=NumberRange(start_of_range=0, end_of_range=1), + ) + + system_description_message = FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1), + actuators=[actuator], + storage=storage, + ) + + return system_description_message + + +@pytest.fixture(scope="session") +def resource_manager_details(): + return ResourceManagerDetails( + message_id=get_unique_id(), + resource_id=get_unique_id(), + roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], + instruction_processing_delay=Duration(__root__=1.0), + available_control_types=[ + ControlType.FILL_RATE_BASED_CONTROL, + ControlType.NO_SELECTION, + ], + provides_forecast=True, + provides_power_measurement_types=[ + CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC + ], + ) + + +@pytest.fixture(scope="session") +def rm_handshake(): + return Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.RM, + supported_protocol_versions=["1.0.0"], + ) diff --git a/tests/test_cem.py b/tests/test_cem.py new file mode 100644 index 00000000..56122f62 --- /dev/null +++ b/tests/test_cem.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import pytest +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues + +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types.FRBC import FRBCTest + + +@pytest.mark.asyncio +async def test_handshake(rm_handshake): + cem = CEM(fm_client=None) + frbc = FRBCTest() + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + # RM sends HandShake + await cem.handle_message(rm_handshake) + + assert ( + cem._sending_queue.qsize() == 1 + ) # check that message is put to the outgoing queue + + # CEM response + response = await cem.get_message() + + assert ( + response["message_type"] == "HandshakeResponse" + ), "response message_type should be HandshakeResponse" + assert ( + response["selected_protocol_version"] == "0.1.0" + ), "CEM selected protocol version should be supported by the Resource Manager" + + +@pytest.mark.asyncio +async def test_resource_manager_details(resource_manager_details, rm_handshake): + cem = CEM(fm_client=None) + frbc = FRBCTest() + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + + assert ( + cem._sending_queue.qsize() == 1 + ) # check that message is put to the outgoing queue + + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + + # RM sends ResourceManagerDetails + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + + # CEM response is ReceptionStatus with an OK status + assert response["message_type"] == "ReceptionStatus" + assert response["status"] == "OK" + + assert ( + cem._resource_manager_details == resource_manager_details + ), "CEM should store the resource_manager_details" + assert cem.control_type == ControlType.NO_SELECTION, ( + "CEM control type should switch to ControlType.NO_SELECTION," + "independently of the original type" + ) + + +@pytest.mark.asyncio +async def test_activate_control_type( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + frbc = FRBCTest() + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### + + # CEM sends a request to change te control type + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + message = await cem.get_message() + + assert cem.control_type == ControlType.NO_SELECTION, ( + "the control type should still be NO_SELECTION (rather than FRBC)," + " because the RM has not yet confirmed FRBC activation" + ) + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK + ) + + await cem.handle_message(response) + + assert ( + cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + ), "after a positive ResponseStatus, the status changes from NO_SELECTION to FRBC" + + +@pytest.mark.asyncio +async def test_messages_route_to_control_type_handler( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + frbc = FRBCTest() + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### + + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + message = await cem.get_message() + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK + ) + + await cem.handle_message(response) + + ######## + # FRBC # + ######## + + await cem.handle_message(frbc_system_description) + response = await cem.get_message() + + # checking that FRBC handler is being called + assert ( + cem._control_types_handlers[ + ControlType.FILL_RATE_BASED_CONTROL + ]._system_description_history[str(frbc_system_description.message_id)] + == frbc_system_description + ), ( + "the FRBC.SystemDescription message should be stored" + "in the frbc.system_description_history variable" + ) + + # change of control type is not performed in case that the RM answers + # with a negative response + await cem.activate_control_type(ControlType.NO_SELECTION) + response = await cem.get_message() + assert ( + cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + ), "control type should not change, confirmation still pending" + + await cem.handle_message( + ReceptionStatus( + subject_message_id=response.get("message_id"), + status=ReceptionStatusValues.INVALID_CONTENT, + ) + ) + + assert ( + cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + ), "control type should not change, confirmation state is not 'OK'" + assert ( + response.get("message_id") + not in cem._control_types_handlers[ + ControlType.FILL_RATE_BASED_CONTROL + ].success_callbacks + ), "success callback should be deleted" diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py new file mode 100644 index 00000000..30d898c0 --- /dev/null +++ b/tests/test_frbc_tunes.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import numpy as np +import pandas as pd +import pytest +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, +) + + +@pytest.fixture(scope="function") +async def setup_cem(resource_manager_details, rm_handshake): + fm_client = AsyncMock(FlexMeasuresClient) + cem = CEM(fm_client=fm_client) + frbc = FillRateBasedControlTUNES( + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + timezone="UTC", + schedule_duration=timedelta(hours=12), + max_size=100, + valid_from_shift=timedelta(days=1), + ) + + cem.register_control_type(frbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### + + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + message = await cem.get_message() + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK + ) + + await cem.handle_message(response) + + return cem, fm_client + + +@pytest.fixture(scope="function") +async def cem_in_frbc_control_type(setup_cem, frbc_system_description): + cem, fm_client = await setup_cem + + ######## + # FRBC # + ######## + + await cem.handle_message(frbc_system_description) + await cem.get_message() + + return cem, fm_client + + +@pytest.mark.asyncio +async def test_system_description(cem_in_frbc_control_type, frbc_system_description): + cem, fm_client = await cem_in_frbc_control_type + + ######## + # FRBC # + ######## + + await cem.handle_message(frbc_system_description) + frbc = cem._control_types_handlers[cem.control_type] + + tasks = get_pending_tasks() + + # check that we are sending the conversion efficiencies + await tasks["send_conversion_efficiencies"] + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + CONVERSION_EFFICIENCY_DURATION, + RESOLUTION, + ) + + N_SAMPLES = int( + pd.Timedelta(CONVERSION_EFFICIENCY_DURATION) / pd.Timedelta(RESOLUTION) + ) + + # first call of post_measurements which corresponds to the THP efficiency + first_call = fm_client.post_measurements.call_args_list[0][1] + first_call_expected = { + "sensor_id": frbc._thp_efficiency_sensor_id, + "start": datetime(2024, 1, 1), + "values": [0.2] * N_SAMPLES, + "unit": "%", + "duration": "PT24H", + } + for key in first_call.keys(): + assert first_call[key] == first_call_expected[key] + + # second call of post_measurements which corresponds to the NES efficiency + second_call = fm_client.post_measurements.call_args_list[1][1] + + second_call_expected = { + "sensor_id": frbc._nes_efficiency_sensor_id, + "start": datetime(2024, 1, 1), + "values": [0.1] * N_SAMPLES, + "unit": "%", + "duration": "PT24H", + } + for key in second_call.keys(): + assert second_call[key] == second_call_expected[key] + + await cem.close() + get_pending_tasks() + + +def get_pending_tasks(): + pending = asyncio.all_tasks() + + tasks = {} + + # get all pending tasks + for task in pending: + func_name = task.get_coro().cr_code.co_name + tasks[func_name] = task + + return tasks + + +@pytest.mark.asyncio +async def test_fill_level_target_profile(cem_in_frbc_control_type): + cem, fm_client = await cem_in_frbc_control_type + + fill_level_target_profile = { + "start_time": "2024-01-01T00:00:00+01:00", + "message_type": "FRBC.FillLevelTargetProfile", + "message_id": "a-valid-id", + "elements": [ + { + "duration": 1e3 * 3600, + "fill_level_range": {"start_of_range": 0, "end_of_range": 100}, + }, + { + "duration": 1e3 * 2 * 3600, + "fill_level_range": {"start_of_range": 10, "end_of_range": 90}, + }, + { + "duration": 1e3 * 3 * 3600, + "fill_level_range": {"start_of_range": 20, "end_of_range": 80}, + }, + ], + } + + await cem.handle_message(fill_level_target_profile) + + tasks = get_pending_tasks() + + # clear mock state because it contains previous such as + # the ones used to process the system description + fm_client.reset_mock() + + # wait for the task send_fill_level_target_profile to finish + await tasks["send_fill_level_target_profile"] + + start = datetime(2024, 1, 1, 0, 0, tzinfo=timezone(timedelta(seconds=3600))) + + first_call = fm_client.post_measurements.call_args_list[0][1] + assert first_call["sensor_id"] == 2 + assert first_call["start"] == start + assert np.isclose(first_call["values"].values, [0] * 4 + [10] * 8 + [20] * 12).all() + + second_call = fm_client.post_measurements.call_args_list[1][1] + assert second_call["sensor_id"] == 3 + assert second_call["start"] == start + assert np.isclose( + second_call["values"].values, [100] * 4 + [90] * 8 + [80] * 12 + ).all() + + await cem.close() + get_pending_tasks() + + +@pytest.mark.asyncio +async def test_fill_rate_relay(cem_in_frbc_control_type): + cem, fm_client = await cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + + actuator_status = { + "active_operation_mode_id": "tarnoc-operation-mode", + "actuator_id": "id-of-the-actuator", + "message_type": "FRBC.ActuatorStatus", + "message_id": "a-valid-id", + "operation_mode_factor": 0.0, + } + + await cem.handle_message(actuator_status) + + tasks = get_pending_tasks() + + # clear mock state because it contains previous such as + # the ones used to process the system description + fm_client.reset_mock() + + # wait for the task send_actuator_status to finish + await tasks["send_actuator_status"] + + first_call = fm_client.post_measurements.call_args_list[0][1] + assert first_call["sensor_id"] == frbc._thp_fill_rate_sensor_id + + second_call = fm_client.post_measurements.call_args_list[1][1] + assert second_call["sensor_id"] == frbc._fill_rate_sensor_id + + # Switch operation mode to Nestore + actuator_status["active_operation_mode_id"] = "nestore-operation-mode" + + await cem.handle_message(actuator_status) + tasks = get_pending_tasks() + + # clear mock state because it contains previous such as + # the ones used to process the system description + fm_client.reset_mock() + + # wait for the task send_actuator_status to finish + await tasks["send_actuator_status"] + + first_call = fm_client.post_measurements.call_args_list[0][1] + assert first_call["sensor_id"] == frbc._nes_fill_rate_sensor_id + + second_call = fm_client.post_measurements.call_args_list[1][1] + assert second_call["sensor_id"] == frbc._fill_rate_sensor_id + + await cem.close() + get_pending_tasks() + + +@pytest.mark.asyncio +async def test_trigger_schedule(cem_in_frbc_control_type): + cem, fm_client = await cem_in_frbc_control_type + # frbc = cem._control_types_handlers[cem.control_type] + + tasks = get_pending_tasks() + + assert tasks["trigger_schedule_task"]._state == "PENDING" + await cem.close() + + tasks = get_pending_tasks() + + assert tasks["trigger_schedule_task"]._state == "PENDING" + + await cem.close() + get_pending_tasks() diff --git a/tests/test_s2_coordinator.py b/tests/test_s2_coordinator.py deleted file mode 100644 index c4a48b63..00000000 --- a/tests/test_s2_coordinator.py +++ /dev/null @@ -1,337 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timezone - -import pytest -from s2python.common import ( - Commodity, - CommodityQuantity, - ControlType, - Duration, - EnergyManagementRole, - Handshake, - NumberRange, - PowerRange, - ReceptionStatus, - ReceptionStatusValues, - ResourceManagerDetails, - Role, - RoleType, -) -from s2python.frbc import ( - FRBCActuatorDescription, - FRBCOperationMode, - FRBCOperationModeElement, - FRBCStorageDescription, - FRBCSystemDescription, -) - -from flexmeasures_client.s2.cem import CEM -from flexmeasures_client.s2.control_types.FRBC import FRBCTest -from flexmeasures_client.s2.control_types.PPBC import PPBC -from flexmeasures_client.s2.utils import get_unique_id - - -@pytest.mark.asyncio -async def test_cem_frbc(): # TODO: move into different test functions - cem = CEM(sensor_id=1, fm_client=None) - frbc = FRBCTest() - ppbc = PPBC() - cem.register_control_type(frbc) - cem.register_control_type(ppbc) - # Show that this point is reached - - ############# - # Handshake # - ############# - - handshake_message = Handshake( - message_id=get_unique_id(), - role=EnergyManagementRole.RM, - supported_protocol_versions=["0.1.0"], - ) - - await cem.handle_message(handshake_message) - - assert ( - cem._sending_queue.qsize() == 1 - ) # check that message is put to the outgoing queue - - response = await cem.get_message() - - assert ( - response["message_type"] == "HandshakeResponse" - ), "response message_type should be HandshakeResponse" - assert ( - response["selected_protocol_version"] == "0.1.0" - ), "CEM selected protocol version should be supported by the Resource Manager" - - ########################## - # ResourceManagerDetails # - ########################## - - resource_manager_details_message = ResourceManagerDetails( - message_id=get_unique_id(), - resource_id=get_unique_id(), - roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], - instruction_processing_delay=Duration(1), - available_control_types=[ - ControlType.FILL_RATE_BASED_CONTROL, - ControlType.POWER_PROFILE_BASED_CONTROL, - ControlType.NO_SELECTION, - ], - provides_forecast=True, - provides_power_measurement_types=[ - CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - ], - ) - - await cem.handle_message(resource_manager_details_message) - response = await cem.get_message() - - assert response["message_type"] == "ReceptionStatus" - assert response["status"] == "OK" - assert ( - cem._resource_manager_details == resource_manager_details_message - ), "CEM should store the resource_manager_details" - assert cem.control_type == ControlType.NO_SELECTION, ( - "CEM control type should switch to ControlType.NO_SELECTION," - "independently of the original type" - ) - - ######################### - # Activate control type # - ######################### - - await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) - message = await cem.get_message() - - assert cem.control_type == ControlType.NO_SELECTION, ( - "the control type should still be NO_SELECTION (rather than FRBC)," - " because the RM has not yet confirmed FRBC activation" - ) - - response = ReceptionStatus( - subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK - ) - - await cem.handle_message(response) - - assert ( - cem.control_type == ControlType.FILL_RATE_BASED_CONTROL - ), "after a positive ResponseStatus, the status changes from NO_SELECTION to FRBC" - - ######## - # FRBC # - ######## - - operation_mode_element = FRBCOperationModeElement( - fill_level_range=NumberRange(start_of_range=0, end_of_range=1), - fill_rate=NumberRange(start_of_range=0, end_of_range=1), - power_ranges=[ - PowerRange( - start_of_range=10, - end_of_range=1000, - commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, - ) - ], - ) - - operation_mode = FRBCOperationMode( - id=get_unique_id(), - elements=[operation_mode_element], - abnormal_condition_only=False, - ) - - actuator = FRBCActuatorDescription( - id=get_unique_id(), - supported_commodities=[Commodity.ELECTRICITY], - operation_modes=[operation_mode], - transitions=[], - timers=[], - ) - - storage = FRBCStorageDescription( - provides_leakage_behaviour=False, - provides_fill_level_target_profile=False, - provides_usage_forecast=False, - fill_level_range=NumberRange(start_of_range=0, end_of_range=1), - ) - - system_description_message = FRBCSystemDescription( - message_id=get_unique_id(), - valid_from=datetime.now(timezone.utc), - actuators=[actuator], - storage=storage, - ) - - await cem.handle_message(system_description_message) - response = await cem.get_message() - - # checking that FRBC handler is being called - assert ( - cem._control_types_handlers[ - ControlType.FILL_RATE_BASED_CONTROL - ]._system_description_history[str(system_description_message.message_id)] - == system_description_message - ), ( - "the FRBC.SystemDescription message should be stored" - "in the frbc.system_description_history variable" - ) - - # change of control type is not performed in case that the RM answers - # with a negative response - await cem.activate_control_type(ControlType.NO_SELECTION) - response = await cem.get_message() - assert ( - cem._control_type == ControlType.FILL_RATE_BASED_CONTROL - ), "control type should not change, confirmation still pending" - - await cem.handle_message( - ReceptionStatus( - subject_message_id=response.get("message_id"), - status=ReceptionStatusValues.INVALID_CONTENT, - ) - ) - - assert ( - cem._control_type == ControlType.FILL_RATE_BASED_CONTROL - ), "control type should not change, confirmation state is not 'OK'" - assert ( - response.get("message_id") - not in cem._control_types_handlers[ - ControlType.FILL_RATE_BASED_CONTROL - ].success_callbacks - ), "success callback should be deleted" - - -# @pytest.mark.asyncio -# async def test_cem_ppbc(): # TODO: move into different test functions -# cem = CEM(sensor_id=1, fm_client=None) -# ppbc = PPBC() -# cem.register_control_type(ppbc) -# # Show that this point is reached - -# ############# -# # Handshake # -# ############# - -# handshake_message = Handshake( -# message_id=get_unique_id(), -# role=EnergyManagementRole.RM, -# supported_protocol_versions=["0.1.0"], -# ) - -# await cem.handle_message(handshake_message) - -# assert ( -# cem._sending_queue.qsize() == 1 -# ) # check that message is put to the outgoing queue - -# response = await cem.get_message() - -# assert ( -# response["message_type"] == "HandshakeResponse" -# ), "response message_type should be HandshakeResponse" -# assert ( -# response["selected_protocol_version"] == "0.1.0" -# ), "CEM selected protocol version should be supported by the Resource Manager" - -# ########################## -# # ResourceManagerDetails # -# ########################## - -# resource_manager_details_message = ResourceManagerDetails( -# message_id=get_unique_id(), -# resource_id=get_unique_id(), -# roles=[Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY)], -# instruction_processing_delay=Duration(1), -# available_control_types=[ -# ControlType.FILL_RATE_BASED_CONTROL, -# ControlType.POWER_PROFILE_BASED_CONTROL, -# ControlType.NO_SELECTION, -# ], -# provides_forecast=True, -# provides_power_measurement_types=[ -# CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC -# ], -# ) - -# await cem.handle_message(resource_manager_details_message) -# response = await cem.get_message() - -# assert response["message_type"] == "ReceptionStatus" -# assert response["status"] == "OK" -# assert ( -# cem._resource_manager_details == resource_manager_details_message -# ), "CEM should store the resource_manager_details" -# assert cem.control_type == ControlType.NO_SELECTION, ( -# "CEM control type should switch to ControlType.NO_SELECTION," -# "independently of the original type" -# ) - -# ######################### -# # Activate control type # -# ######################### - -# await cem.activate_control_type(ControlType.POWER_PROFILE_BASED_CONTROL) -# message = await cem.get_message() - -# assert cem.control_type == ControlType.NO_SELECTION, ( -# "the control type should still be NO_SELECTION (rather than PPBC)," -# " because the RM has not yet confirmed FRBC activation" -# ) - -# response = ReceptionStatus( -# subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK -# ) - -# await cem.handle_message(response) - -# assert ( -# cem.control_type == ControlType.POWER_PROFILE_BASED_CONTROL -# ), "after a positive ResponseStatus, the status changes from NO_SELECTION to FRBC" - -# ######## -# # PPBC # -# ######## - - -# await cem.handle_message(system_description_message) -# response = await cem.get_message() - -# # checking that FRBC handler is being called -# assert ( -# cem._control_types_handlers[ -# ControlType.POWER_PROFILE_BASED_CONTROL -# ]._system_description_history[str(system_description_message.message_id)] -# == system_description_message -# ), ( -# "the FRBC.SystemDescription message should be stored" -# "in the frbc.system_description_history variable" -# ) - -# # change of control type is not performed in case that the RM answers -# # with a negative response -# await cem.activate_control_type(ControlType.NO_SELECTION) -# response = await cem.get_message() -# assert ( -# cem._control_type == ControlType.FILL_RATE_BASED_CONTROL -# ), "control type should not change, confirmation still pending" - -# await cem.handle_message( -# ReceptionStatus( -# subject_message_id=response.get("message_id"), -# status=ReceptionStatusValues.INVALID_CONTENT, -# ) -# ) - -# assert ( -# cem._control_type == ControlType.FILL_RATE_BASED_CONTROL -# ), "control type should not change, confirmation state is not 'OK'" -# assert ( -# response.get("message_id") -# not in cem._control_types_handlers[ -# ControlType.FILL_RATE_BASED_CONTROL -# ].success_callbacks -# ), "success callback should be deleted" diff --git a/tests/test_s2_models.py b/tests/test_s2_models.py new file mode 100644 index 00000000..7c57ce26 --- /dev/null +++ b/tests/test_s2_models.py @@ -0,0 +1,31 @@ +from flexmeasures_client.s2.utils import get_unique_id +from flexmeasures_client.s2.wrapper import S2Wrapper + + +def test_simple_model(): + wrapped_message = { + "message": { + "message_id": get_unique_id(), + "resource_id": get_unique_id(), + "roles": [{"role": "ENERGY_STORAGE", "commodity": "ELECTRICITY"}], + "instruction_processing_delay": 1.0, + "available_control_types": ["FILL_RATE_BASED_CONTROL", "NO_SELECTION"], + "provides_forecast": True, + "provides_power_measurement_types": ["ELECTRIC.POWER.3_PHASE_SYMMETRIC"], + "message_type": "ResourceManagerDetails", + }, + "metadata": {"dt": "2023-01-01T00:00:00"}, + } + + S2Wrapper.validate(wrapped_message) + + wrapped_message_2 = { + "message": { + "message_id": get_unique_id(), + "message_type": "Handshake", + "role": "CEM", + }, + "metadata": {"dt": "2024-01-01T00:00:00"}, + } + + S2Wrapper.validate(wrapped_message_2) diff --git a/tests/test_s2_translations.py b/tests/test_s2_translations.py new file mode 100644 index 00000000..e634375e --- /dev/null +++ b/tests/test_s2_translations.py @@ -0,0 +1,78 @@ +import pytest +from s2python.frbc import FRBCUsageForecast + +from flexmeasures_client.s2.control_types.translations import ( + translate_usage_forecast_to_fm, +) +from flexmeasures_client.s2.utils import get_unique_id + + +@pytest.mark.parametrize( + "start, resolution, values", + [ + ("2024-01-01T00:00:00+01:00", "1h", [100, 100]), + ("2024-01-01T00:00:00+01:00", "15min", [100] * 4 * 2), + ("2024-01-01T00:30:00+01:00", "1h", [100, 100, 100]), + ], +) +def test_resampling_one_block(start, resolution, values): + message = { + "elements": [ + {"duration": 2 * 3600 * 1e3, "usage_rate_expected": 100}, + ], + "message_id": get_unique_id(), + "message_type": "FRBC.UsageForecast", + "start_time": start, + } + + usage_forecast = FRBCUsageForecast.from_dict(message) + + s = translate_usage_forecast_to_fm(usage_forecast, resolution=resolution) + assert all(abs(s.values - values) < 1e-5) + + +@pytest.mark.parametrize( + "start, resolution, values", + [ + ("2024-01-01T00:00:00+01:00", "1h", [100, 200, 200, 350, 450, 600]), + ("2024-01-01T00:45:00+01:00", "1h", [100, 200, 200, 350, 450, 600, 600]), + ( + "2024-01-01T00:00:00+01:00", + "30min", + [100] * 2 + [200] * 2 * 2 + [300] * 1 + [400] * 2 + [500] * 1 + [600] * 1, + ), + ( + "2024-01-01T00:00:00+01:00", + "15min", + [100] * 4 + [200] * 4 * 2 + [300] * 2 + [400] * 4 + [500] * 2 + [600] * 2, + ), + ], +) +def test_usage_forecast(start, resolution, values): + """ + - 100 for 1h + - 200 for 2h + - 300 for 30min + - 400 for 1h + - 500 for 30min + - 600 for 30min + + """ + message = { + "elements": [ + {"duration": 3600 * 1e3, "usage_rate_expected": 100}, + {"duration": 2 * 3600 * 1e3, "usage_rate_expected": 200}, + {"duration": 0.5 * 3600 * 1e3, "usage_rate_expected": 300}, + {"duration": 3600 * 1e3, "usage_rate_expected": 400}, + {"duration": 0.5 * 3600 * 1e3, "usage_rate_expected": 500}, + {"duration": 0.5 * 3600 * 1e3, "usage_rate_expected": 600}, + ], + "message_id": get_unique_id(), + "message_type": "FRBC.UsageForecast", + "start_time": start, + } + + usage_forecast = FRBCUsageForecast.from_dict(message) + + s = translate_usage_forecast_to_fm(usage_forecast, resolution=resolution) + assert all(abs(s.values - values) < 1e-5) From dac5f6a44429b5717237146e80a8d57f3353cb0c Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Wed, 7 Aug 2024 16:56:51 +0200 Subject: [PATCH 30/79] make session optional Signed-off-by: Victor Garcia Reolid Signed-off-by: Vlad Iftime --- public-key.asc | 52 +++++++++++++++++++++++++++++++ src/flexmeasures_client/client.py | 11 +++++-- 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 public-key.asc diff --git a/public-key.asc b/public-key.asc new file mode 100644 index 00000000..1b9f3d25 --- /dev/null +++ b/public-key.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGdpc0QBEADSlQSDYaojMIMDcUjAWVuu3XVivJTB76LVhIN2nqpIW47uMdGv +NjnhNdjPHCamX8C6zT+o8HdeY4aSfLnTIaVO1EzgO3twPJsg9YII2DGZdmZy5y7x +aJjDgZVab58rNhvRanLfklt4HYqlsd07eJmSxYmQZgxpsd0a83hig6RIkK/N5Sv4 +DTXtdokzdurOZNeihXSV3n17YJiGgI4OHWY4BiLyXVzVZw8F81rPQvF8hWWd3iqG +IE/c4IiQz2pXq9tLSsg9BnOdLhaN2LzIde5Ya6NsQX0M56HD5OYr6MBei3e6x34a +K/TddKMCpvlzydwHeT54UXLXMdcxptOA4warLMgIcoFN77voSQZcdpojTDn9HFbp +pb7sO18O9OW7SJmO9OeZPMQ/VIBruQOVbN4OVBUiVMDVwV45iAU5MIGpZETAOgVR +/c7wBecLCgFG1MWVw37wCQiZfb6nki/1P//Q+LiUN6su/Bu5TrkMAQT/NsLOG4eZ +1VfNAzfBlYPXVK3Kxlqv7eXv5qLuCz1JcwDPyyKej8BoapqtDiFAZoJD/p+Z8LYX ++p1sLGc8iMw7lQ3dPJAMjB2lyTCuMgGxY0t3qh3TccMqzVOgYXahmDPB9I6yyorN +p4u9ZDag37KwxcK5+weHUR310BwLqjDLRfpwZv91eNJ/OiLTfAgo7dP//wARAQAB +tDRWbGFkIElmdGltZSAoR1BHIGZvciBTZWl0YSkgPHZsYWRpZnRpbWU2MEBnbWFp +bC5jb20+iQJRBBMBCAA7FiEEj0ibEKJxZRRLRO1EXx20V18bO9oFAmdpc0QCGwMF +CwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQXx20V18bO9oVhxAAmK6sulZE +ho3C6vtZRMFFT50QTnDeLPI8dF//KATnhYeirKl6+1suS9jqWOaByaQg1Zu6mO94 +9datsFapL8eGeWue5ED1RPfTmnrefxPh/BvGIsr8IM+kJ1l74hKoqhIp+exPcaed +30JVQnw14ODaxDOT693/bOfSO5CUr+7tf8wdksyicJ7HG66beE1rY5s0kvTP2uI4 +kJy9z0qIHYtK+w0J6l7Vyyv62Fv5/QSGTrKcTvikr0OWTF9ysj6UksCwsApY8u1w +eVJ3n4WounXrCGYyhB2nYpP24MN54GuHmaALL0SSZhpjL5FPUmLb3I+Ta5syi+LE +GVAoUX5/bXjC2Q/l8Mc7zEYD5SWqAfRvRd3aH1hJOiriEJmJrL+x4zv3bdJc6dUY +8QEMN7fCjm+hk1zXNtkU8BFMMB0iBhWvwGxsh6cLJ/qBJ/h5cPvD4n9p+NEtLLml +s8SkFRl8yrIOKLCN68V6PjmAWpZ3GFuPO4FtFi9dNoBlZfbkVhsJhTCdXbYbNGPr +YvSlflfiEyh23jSa3HITvVfnIaYAf21WCOFz0FnvKwK40e9pbR8GXTAatCnbyPEq +8xnARdWxZPLiNlqVBt/1XqBt7azDiCWT8yhu/oZAxMHCT9x95vZqiGjE9JEQUf2b +wUGshBrtLIxQAx6wth/vpGISiTz4zTHVLta5Ag0EZ2lzRAEQAKkZotjNDX0U7uTU +V/E45b+iuMDxluN0+gYnIRMYiVTsWM6WkAEj5n3QO2BwfjieW9RLDWx6E/naNxV2 +KljSGzWG0DnS+7IHtmZOxKm16YWreW+Ojlnf8XvdsNm6BjKi+UgRsA3nPvgyVvoT +3wr7gNQNayRZUOIGXDZno3NZ5y80Ds8eOKdNOHBpUBxZzS4lyIS+6kqfjMZL7v2o +mxgy1B+WMnLoXyc54mmVblcpysUkNICtR9jLiQKZyZY/PjMfkFihrzPTG0hsH6Gq +QCjACMGRR1TxbH0BNFuvtJyt6WhAV2kHq/TFt5G46It/2qugJAHP8F5pjIvwcbWR +3he73xwX4KJmQ1/iJgE3uLK921K6/uCkcCrp88UWWooFuFV0visyVyRH3Lou7Ade +5nkkJEcgO7aYkjaKo142ZSoBC6Wzgn7RgN611ScmzE+2CCbRPTsufBnRy1ULBetF +Ny+M82jQN8FtnKb7nYHS452WPmhvq/L7xE60/0vdbD8W15uqTp2HUZdnINj+GYLs +Yv82ERH2is4Wd6Ow8qzBTuNTxsC4IYuWKqOi6/dpMhp7GUvcUdQwQajcKQvrgLC5 +fRC1MKJzUFFutRFzeWm4HB6Ng1gwO+a+aA4VNORQRgmwTFRnYNtaZ5x2mn6sSzX8 +p8I57V1GpYzZUl/QuB/biqD3zFLJABEBAAGJAjYEGAEIACAWIQSPSJsQonFlFEtE +7URfHbRXXxs72gUCZ2lzRAIbDAAKCRBfHbRXXxs72umuD/sE4xP41Vd2fJ0T4pll +RylfuPTSvRQrAK8Ggr3woWiZl9lxpoeES/6wNAYs2wU29o5AlfB903qOOsr/qHOf +6cf8I2kii1YEtpyYGnqd3S0J1u54CixPH5vL8Vfq1L/0VFFJYl0u/24xpnNtKqaO +m/qcCewILibuhU1M4KLhou0G2E3NYWH+TUHCoSqdP8HBquPFxKs035yz2/hkBhjJ +i6G7rTqA0x5cFf3DN3ge0FIDwRVmJvJlBKK8U320B3npN8/al64dg7uYhC049NrR +DYcEWp9boOKKlq/ebF6St0sKq1MrvMJxD6K6SVxYy9mbxYbK0CoGoAHPW63XthR7 +7RuRktBizUvXy0cbXN272f+odb0Nj7OW+WG9CbC08gQQzFkixpxLDKTZw60XJcJd +diO0JVIGQV7OCgbV2XFCG+qv9YVu1Fgin6xyriKtcSxfPIAFmxND8oboaLW9K46E +dbkv9CHgtkGKPwlKk+B95k08qLXxId8lWt0PpnGLF0c8iqfGI13K2rAIAg2rUy0A +CyLJX1+u1kDMPW2gCLzvH8GbjiL+LKBolCNYvJ3jwrJ9ADZ2wuCmjkUNa9WRyixz +0anf0nE+KjD68X4hkOuXPDUNQe+GpC8WY7viVDE/x5biFLjnC9AZCMxJrxJbqlFk +qp/mh7JdgqneCZYG2eleA28z9w== +=8gvm +-----END PGP PUBLIC KEY BLOCK----- diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 5e97ce10..be14020c 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -7,7 +7,7 @@ import socket from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Any +from typing import Any, cast import async_timeout import pandas as pd @@ -59,7 +59,8 @@ class FlexMeasuresClient: session: ClientSession | None = None def __post_init__(self): - self.session: ClientSession = ClientSession() + if self.session is None: + self.session = ClientSession() if not re.match(r".+\@.+\..+", self.email): raise EmailValidationError( @@ -104,7 +105,7 @@ def determine_port(self): async def close(self): """Function to close FlexMeasuresClient session when all requests are done""" - await self.session.close() + await cast(ClientSession, self.session).close() async def request( self, @@ -199,8 +200,12 @@ async def request_once( logging.debug("=" * 14) """Sends a single request to FlexMeasures and checks the response""" +<<<<<<< HEAD self.ensure_session() response = await self.session.request( # type: ignore +======= + response = await cast(ClientSession, self.session).request( +>>>>>>> 3b3ca0b (make session optional) method=method, url=url, params=params, From d90c5f91f8927260b656e0f46844be9cc13eabbf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 30 Sep 2024 12:41:58 +0200 Subject: [PATCH 31/79] docs: fix typo Signed-off-by: Vlad Iftime --- README.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4de3975e..94814ab4 100644 --- a/README.rst +++ b/README.rst @@ -46,9 +46,11 @@ The Flexmeasures Client package provides functionality for authentication, asset Getting Started =============== -To get started with the FlexMeasures Client package, first an account needs to be registered with a FlexMeasures instance. -To create a local instance of FlexMeasures, follow the `FlexMeasures documentation `_. -Registering to a hosted FlexMeasures instance instead can be done through `Seita BV `_. +To get started using the FlexMeasures Client package first an account needs to be registered with a FlexMeasures instance or a local FlexMeasures instance needs to be created. +Registering a to a FlexMeasures instance can be done through `Seita BV `_. +To create a local instance of FlexMeasures follow the `FlexMeasures documentation `_. + +In this example we are connecting to ``localhost:5000``, To connect to a different host add the host in the initialization of the client. Install using ``pip``:: From d32efd2de42984bdf1a3100e7be7103d6c0625ed Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 29 Oct 2024 21:43:05 +0100 Subject: [PATCH 32/79] docs: fix typo Signed-off-by: Vlad Iftime --- .../s2/control_types/FRBC/frbc_tunes.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index a31fd177..07004723 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -10,7 +10,7 @@ import pandas as pd import pydantic import pytz -from s2python.common import NumberRange, ReceptionStatusValues +from s2python.common import NumberRange, ReceptionStatusValues, ReceptionStatus from s2python.frbc import ( FRBCActuatorStatus, FRBCFillLevelTargetProfile, @@ -29,7 +29,8 @@ from flexmeasures_client.s2.utils import get_reception_status, get_unique_id RESOLUTION = "15min" -POWER_UNIT = "MWh" +ENERGY_UNIT = "MWh" +POWER_UNIT = "MW" DIMENSIONLESS = "dimensionless" PERCENTAGE = "%" TASK_PERIOD_SECONDS = 2 @@ -50,7 +51,7 @@ class FillRateBasedControlTUNES(FRBC): _usage_forecast_sensor_id: int | None _soc_minima_sensor_id: int | None _soc_maxima_sensor_id: int | None - _rm_discharge_sensor_id: int | None + # _rm_discharge_sensor_id: int | None def __init__( self, @@ -99,13 +100,21 @@ def now(self): return self._timezone.localize(datetime.now()) async def send_storage_status(self, status: FRBCStorageStatus): - await self._fm_client.post_measurements( - self._fill_level_sensor_id, - start=self.now(), - values=[status.present_fill_level], - unit=POWER_UNIT, - duration=timedelta(minutes=0), # INSTANTANEOUS - ) + try: + await self._fm_client.post_measurements( + self._fill_level_sensor_id, + start=self.now(), + values=[status.present_fill_level], + unit=POWER_UNIT, + duration=timedelta(minutes=15), # INSTANTANEOUS + ) + except Exception as e: + response = ReceptionStatus( + subject_message_id=status.get("message_id"), + status=ReceptionStatusValues.PERMANENT_ERROR, + ) + await self._sending_queue.put(response) + async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor @@ -142,7 +151,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): start=dt, values=[fill_rate], unit=POWER_UNIT, - duration=timedelta(0), + duration=timedelta(minutes=15), ) # Send data to the sensor of the input fill_rate to the storage device @@ -151,7 +160,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): start=dt, values=[fill_rate], unit=POWER_UNIT, - duration=timedelta(0), + duration=timedelta(minutes=15), ) async def start_trigger_schedule(self): @@ -326,6 +335,8 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): Args: usage_forecast (FRBCUsageForecast): The usage forecast to be translated and sent. """ + start_time = usage_forecast.start_time + # todo: floor to RESOLUTION usage_forecast = translate_usage_forecast_to_fm( usage_forecast, RESOLUTION, strategy="mean" @@ -333,8 +344,8 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): await self._fm_client.post_measurements( sensor_id=self._usage_forecast_sensor_id, - start=usage_forecast.start_time, - values=usage_forecast, + start=start_time, + values=usage_forecast.tolist(), unit=POWER_UNIT, duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), ) From 963ea53c21044a82bb81b49710c2d2d212cdb21b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:49:20 +0100 Subject: [PATCH 33/79] fix: pin pandas Signed-off-by: Vlad Iftime --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 33edbddb..51f36fa9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,7 @@ python_requires >= 3.9 install_requires = importlib-metadata; python_version<"3.8" aiohttp<=3.9.1 - pandas + pandas==2.2.1 pydantic>=1.10.8,<2.0 s2-python==0.2.0.dev2 async_timeout From 1cdc0718a97062e202b06b1c7ac759cc89fb4f36 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:49:44 +0100 Subject: [PATCH 34/79] docs: comment on tests Signed-off-by: Vlad Iftime --- tests/test_frbc_tunes.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 30d898c0..2b98b1a7 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -203,6 +203,11 @@ async def test_fill_level_target_profile(cem_in_frbc_control_type): @pytest.mark.asyncio async def test_fill_rate_relay(cem_in_frbc_control_type): + """Check whether the fill rate from the Tarnoc or Nestor is relayed + to the overall heating system's fill rate sensor, and the fill rate sensor ID + corresponds correctly to the Tarnoc fill rate sensor or the Nestor fill rate sensor. + """ + cem, fm_client = await cem_in_frbc_control_type frbc = cem._control_types_handlers[cem.control_type] @@ -251,11 +256,24 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert second_call["sensor_id"] == frbc._fill_rate_sensor_id await cem.close() - get_pending_tasks() @pytest.mark.asyncio async def test_trigger_schedule(cem_in_frbc_control_type): + """Work in progress. + + # todo: add steps + + Steps + + 1. Check whether the task starts and stops + 2. Check call arguments + 3. Check queue for results mocking the results of FM client + + # todo consider splitting up test + S2 2 FM: converging system description to flex config + FM 2 S2: schedules to instructions + """ cem, fm_client = await cem_in_frbc_control_type # frbc = cem._control_types_handlers[cem.control_type] From a29adbc7b3a048d61c7600845ba5a43361c70319 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:50:03 +0100 Subject: [PATCH 35/79] feature: expose translation strategy further up the chain of methods Signed-off-by: Vlad Iftime --- src/flexmeasures_client/s2/control_types/translations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index f97c1a23..db22bcb4 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -118,6 +118,7 @@ def unevenly_ts_to_evenly( def translate_usage_forecast_to_fm( usage_forecast: FRBCUsageForecast, resolution: str = "1h", + strategy: str = "mean", ) -> pd.Series: """ Translate a FRBC.UsageForecast into a FlexMeasures compatible format with evenly spaced data. @@ -140,7 +141,7 @@ def translate_usage_forecast_to_fm( values=values, durations=durations, target_resolution=resolution, - strategy="mean", + strategy=strategy, ) From 6a01983065deeb4d419f45b771a0063bb7d09261 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:55:18 +0100 Subject: [PATCH 36/79] style: black Signed-off-by: Vlad Iftime --- src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 07004723..4c6a31fb 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -115,7 +115,6 @@ async def send_storage_status(self, status: FRBCStorageStatus): ) await self._sending_queue.put(response) - async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor system_description: FRBCSystemDescription = list( From cdac191e0e710d20c86986a64a68e3246b079b70 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 11:57:26 +0100 Subject: [PATCH 37/79] style: update black Signed-off-by: Vlad Iftime --- .pre-commit-config.yaml | 2 +- src/flexmeasures_client/s2/__init__.py | 12 ++++++------ src/flexmeasures_client/s2/cem.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d409690..89455e67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.8.0 hooks: - id: black language_version: python3 diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index bee45f37..66117f5a 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -40,15 +40,15 @@ def wrap(*args, **kwargs): # TODO: implement function __hash__ in ID that returns # the value of __root__, this way we would be able to use # the ID as key directly - self.incoming_messages[ - get_message_id(incoming_message) - ] = incoming_message + self.incoming_messages[get_message_id(incoming_message)] = ( + incoming_message + ) outgoing_message = func(self, incoming_message) - self.outgoing_messages[ - get_message_id(outgoing_message) - ] = outgoing_message + self.outgoing_messages[get_message_id(outgoing_message)] = ( + outgoing_message + ) return outgoing_message diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index d9a1021c..d7285e80 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -99,9 +99,9 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): control_type_handler._sending_queue = self._sending_queue # store control_type_handler - self._control_types_handlers[ - control_type_handler._control_type - ] = control_type_handler + self._control_types_handlers[control_type_handler._control_type] = ( + control_type_handler + ) async def handle_message(self, message: Dict | pydantic.BaseModel | str): """ From 29138fb95f36dc28299fbebb1d2c8c9689f75f84 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 5 Nov 2024 11:58:53 +0100 Subject: [PATCH 38/79] apply pre-commit Signed-off-by: Vlad Iftime --- src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 4c6a31fb..e4b64ce3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -10,7 +10,7 @@ import pandas as pd import pydantic import pytz -from s2python.common import NumberRange, ReceptionStatusValues, ReceptionStatus +from s2python.common import NumberRange, ReceptionStatus, ReceptionStatusValues from s2python.frbc import ( FRBCActuatorStatus, FRBCFillLevelTargetProfile, From 5bea6b4f6cf46cf2b56956b31f5e4345ed9240a8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 5 Nov 2024 12:17:13 +0100 Subject: [PATCH 39/79] chore: use (dev) released s2-python version Signed-off-by: Vlad Iftime --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 51f36fa9..0c0cf282 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas==2.2.1 pydantic>=1.10.8,<2.0 - s2-python==0.2.0.dev2 + s2-python==v0.3.0.dev1 async_timeout From 551ff51558b91a4b6fb8c5a947d1371ac6c3465f Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 5 Nov 2024 12:00:37 +0100 Subject: [PATCH 40/79] per-comit on setup Signed-off-by: Vlad Iftime --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8a3f4305..3696d25f 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ PyScaffold helps you to put up the scaffold of your new Python project. Learn more under: https://pyscaffold.org/ """ + from setuptools import setup if __name__ == "__main__": From d94c42b4c98cc2183655ac4efe0c1cc783e2c472 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 6 Dec 2024 14:02:06 +0100 Subject: [PATCH 41/79] fix: update s2-python Signed-off-by: F.N. Claessen Signed-off-by: Vlad Iftime --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0c0cf282..747980f9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas==2.2.1 pydantic>=1.10.8,<2.0 - s2-python==v0.3.0.dev1 + s2-python==0.1.3 async_timeout From ff11ab9fed082049ad90c4efe44183f96846a0a3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 10 Dec 2024 11:31:33 +0100 Subject: [PATCH 42/79] fix: relax Pandas constraint Signed-off-by: F.N. Claessen Signed-off-by: Vlad Iftime --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 747980f9..7e6d9aac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,7 +50,7 @@ python_requires >= 3.9 install_requires = importlib-metadata; python_version<"3.8" aiohttp<=3.9.1 - pandas==2.2.1 + pandas>=2.1.4 pydantic>=1.10.8,<2.0 s2-python==0.1.3 async_timeout From e6f7a44b9744f48929b4271261d5fe4e2cb6566c Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Thu, 19 Dec 2024 00:11:43 +0200 Subject: [PATCH 43/79] Created tests for PPBC. Passed all the tests. One test left. Signed-off-by: Vlad Iftime --- src/flexmeasures_client/s2/cem.py | 2 +- tests/conftest.py | 130 ++++++++++++++++++++++++++++-- tests/test_s2_models.py | 31 ------- 3 files changed, 124 insertions(+), 39 deletions(-) delete mode 100644 tests/test_s2_models.py diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index d7285e80..37617e52 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -43,8 +43,8 @@ class CEM(Handler): def __init__( self, - sensor_id: int, fm_client: FlexMeasuresClient, + sensor_id: int = 1, logger: Logger | None = None, ) -> None: """ diff --git a/tests/conftest.py b/tests/conftest.py index aca532f0..9e36d44a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations -from datetime import datetime +import uuid +from datetime import datetime, timedelta, timezone import pytest from s2python.common import ( @@ -11,6 +12,7 @@ EnergyManagementRole, Handshake, NumberRange, + PowerForecastValue, PowerRange, ResourceManagerDetails, Role, @@ -23,6 +25,12 @@ FRBCStorageDescription, FRBCSystemDescription, ) +from s2python.ppbc import ( + PPBCPowerProfileDefinition, + PPBCPowerSequence, + PPBCPowerSequenceContainer, + PPBCPowerSequenceElement, +) from flexmeasures_client.s2.utils import get_unique_id @@ -46,7 +54,7 @@ def frbc_system_description(): ) thp_operation_mode = FRBCOperationMode( - id="tarnoc-operation-mode", + id=str(uuid.uuid4()), elements=[thp_operation_mode_element], abnormal_condition_only=False, ) @@ -64,13 +72,13 @@ def frbc_system_description(): ) nes_operation_mode = FRBCOperationMode( - id="nestore-operation-mode", + id=str(uuid.uuid4()), elements=[nes_operation_mode_element], abnormal_condition_only=False, ) actuator = FRBCActuatorDescription( - id="id-of-the-actuator", + id=str(uuid.uuid4()), supported_commodities=[Commodity.ELECTRICITY], operation_modes=[thp_operation_mode, nes_operation_mode], transitions=[], @@ -86,7 +94,7 @@ def frbc_system_description(): system_description_message = FRBCSystemDescription( message_id=get_unique_id(), - valid_from=datetime(2024, 1, 1), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), # Attach UTC timezone actuators=[actuator], storage=storage, ) @@ -95,12 +103,100 @@ def frbc_system_description(): @pytest.fixture(scope="session") -def resource_manager_details(): +def ppbc_power_profile_definition(): + forecast1 = PowerForecastValue( + value_expected=100.0, commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1 + ) + forecast2 = PowerForecastValue( + value_expected=200.0, commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1 + ) + forecast3 = PowerForecastValue( + value_expected=300.0, commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1 + ) + + element1 = PPBCPowerSequenceElement( + duration=Duration(1), power_values=[forecast1, forecast2] + ) + element2 = PPBCPowerSequenceElement( + duration=Duration(1), power_values=[forecast2, forecast3, forecast1] + ) + + power_sequence1 = PPBCPowerSequence( + id=uuid.uuid4(), + elements=[element1, element2], + is_interruptible=False, + max_pause_before=Duration(0), + abnormal_condition_only=False, + ) + + power_sequence2 = PPBCPowerSequence( + id=uuid.uuid4(), + elements=[element2, element1], + is_interruptible=True, + max_pause_before=Duration(0), + abnormal_condition_only=True, + ) + + power_sequence3 = PPBCPowerSequence( + id=uuid.uuid4(), + elements=[element2], + is_interruptible=False, + max_pause_before=Duration(10000), + abnormal_condition_only=False, + ) + + power_sequence4 = PPBCPowerSequence( + id=uuid.uuid4(), + elements=[element1], + is_interruptible=True, + max_pause_before=Duration(10000), + abnormal_condition_only=True, + ) + + power_sequence_container1 = PPBCPowerSequenceContainer( + id=uuid.uuid4(), + power_sequences=[ + power_sequence1, + power_sequence2, + ], + ) + + power_sequence_container2 = PPBCPowerSequenceContainer( + id=uuid.uuid4(), + power_sequences=[ + power_sequence3, + ], + ) + + power_sequence_container3 = PPBCPowerSequenceContainer( + id=uuid.uuid4(), + power_sequences=[ + power_sequence4, + ], + ) + + power_profile_definition = PPBCPowerProfileDefinition( + message_id=uuid.uuid4(), + id=uuid.uuid4(), + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc) + timedelta(hours=4), + power_sequences_containers=[ + power_sequence_container1, + power_sequence_container2, + power_sequence_container3, + ], + ) + + return power_profile_definition + + +@pytest.fixture(scope="session") +def resource_manager_details_frbc(): return ResourceManagerDetails( message_id=get_unique_id(), resource_id=get_unique_id(), roles=[Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY)], - instruction_processing_delay=Duration(__root__=1.0), + instruction_processing_delay=Duration(1), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, @@ -112,6 +208,26 @@ def resource_manager_details(): ) +@pytest.fixture(scope="session") +def resource_manager_details_ppbc(): + return ResourceManagerDetails( + message_id=get_unique_id(), + resource_id=get_unique_id(), + roles=[Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY)], + instruction_processing_delay=Duration(1), + available_control_types=[ + ControlType.POWER_PROFILE_BASED_CONTROL, + ControlType.NO_SELECTION, + ], + provides_forecast=True, + provides_power_measurement_types=[ + CommodityQuantity.ELECTRIC_POWER_L1, + CommodityQuantity.ELECTRIC_POWER_L2, + CommodityQuantity.ELECTRIC_POWER_L3, + ], + ) + + @pytest.fixture(scope="session") def rm_handshake(): return Handshake( diff --git a/tests/test_s2_models.py b/tests/test_s2_models.py deleted file mode 100644 index 7c57ce26..00000000 --- a/tests/test_s2_models.py +++ /dev/null @@ -1,31 +0,0 @@ -from flexmeasures_client.s2.utils import get_unique_id -from flexmeasures_client.s2.wrapper import S2Wrapper - - -def test_simple_model(): - wrapped_message = { - "message": { - "message_id": get_unique_id(), - "resource_id": get_unique_id(), - "roles": [{"role": "ENERGY_STORAGE", "commodity": "ELECTRICITY"}], - "instruction_processing_delay": 1.0, - "available_control_types": ["FILL_RATE_BASED_CONTROL", "NO_SELECTION"], - "provides_forecast": True, - "provides_power_measurement_types": ["ELECTRIC.POWER.3_PHASE_SYMMETRIC"], - "message_type": "ResourceManagerDetails", - }, - "metadata": {"dt": "2023-01-01T00:00:00"}, - } - - S2Wrapper.validate(wrapped_message) - - wrapped_message_2 = { - "message": { - "message_id": get_unique_id(), - "message_type": "Handshake", - "role": "CEM", - }, - "metadata": {"dt": "2024-01-01T00:00:00"}, - } - - S2Wrapper.validate(wrapped_message_2) From f842424d50a7a5f91b3ca6f6962bba94b0ee5ca1 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:44:28 +0100 Subject: [PATCH 44/79] chore: close test clients at the end of each relevant async test (#90) Signed-off-by: Vlad Iftime --- tests/test_client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_client.py b/tests/test_client.py index 6ffbf487..767e1c57 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -170,6 +170,7 @@ async def test_get_access_token() -> None: ssl=False, allow_redirects=False, ) + await flexmeasures_client.close() @pytest.mark.asyncio @@ -213,6 +214,7 @@ async def test_post_measurements() -> None: ssl=False, allow_redirects=False, ) + await flexmeasures_client.close() @pytest.mark.asyncio @@ -276,6 +278,7 @@ async def test_trigger_schedule() -> None: ssl=False, allow_redirects=False, ) + await flexmeasures_client.close() @pytest.mark.asyncio @@ -327,6 +330,7 @@ async def test_get_schedule_polling() -> None: sensor_id=1, schedule_id="some-uuid", duration="PT45M" ) assert schedule["values"] == [2.15, 3, 2] + await flexmeasures_client.close() @pytest.mark.asyncio @@ -355,6 +359,7 @@ async def callback(url, **kwargs): await flexmeasures_client.get_schedule( sensor_id=1, schedule_id="some-uuid", duration="PT45M" ) + await flexmeasures_client.close() @pytest.mark.asyncio @@ -383,6 +388,7 @@ async def test_get_assets() -> None: assets = await flexmeasures_client.get_assets() assert len(assets) == 1 assert assets[0]["account_id"] == 2 + await flexmeasures_client.close() @pytest.mark.asyncio @@ -413,6 +419,7 @@ async def test_get_sensors() -> None: sensors[0]["entity_address"] == "ea1.1000-01.required-but-unused-field:fm1.2" ) + await flexmeasures_client.close() @pytest.mark.asyncio @@ -432,6 +439,7 @@ async def test_get_sensors2() -> None: ConnectionError, match="Error occurred while communicating with the API." ): await flexmeasures_client.get_sensors() + await flexmeasures_client.close() @pytest.mark.asyncio @@ -493,6 +501,7 @@ async def test_trigger_and_get_schedule() -> None: flex_model={}, ) assert schedule["values"] == [2.15, 3, 2] + await flexmeasures_client.close() @pytest.mark.asyncio @@ -580,6 +589,7 @@ async def test_get_sensor_data() -> None: resolution=resolution, ) assert sensor_data["values"] == [8.5, 8.5, 8.5] + await flexmeasures_client.close() @pytest.mark.asyncio @@ -614,6 +624,7 @@ async def test_reauth_with_access_token() -> None: json=None, allow_redirects=False, ) + await flexmeasures_client.close() @pytest.mark.parametrize( @@ -649,6 +660,7 @@ async def test_reauth_wrong_cred(email, password, payload, error) -> None: with pytest.raises(ValueError, match=error): await flexmeasures_client.get_sensors() + await flexmeasures_client.close() @pytest.mark.asyncio @@ -677,6 +689,7 @@ async def test_update_sensor(): ssl=False, allow_redirects=False, ) + await flexmeasures_client.close() @pytest.mark.asyncio @@ -705,6 +718,7 @@ async def test_update_assets(): ssl=False, allow_redirects=False, ) + await flexmeasures_client.close() @pytest.mark.asyncio From 38a9f36fbb33ba91df10e88ca1ffc76b414defc6 Mon Sep 17 00:00:00 2001 From: VladIftime <49650168+VladIftime@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:33:44 +0100 Subject: [PATCH 45/79] Get client feature request (#94) * Update README.rst Signed-off-by: VladIftime <49650168+VladIftime@users.noreply.github.com> * client.get_accout() * fix: mock user authentication by setting a dummy access_token Signed-off-by: F.N. Claessen * feat: lacking a dedicated API endpoint, we fetch all accessible users first, then pick out ourselves, locate our account ID, and then fetch our organisation account info (two API calls needed, I'm afraid) Signed-off-by: F.N. Claessen * style: run precommit hooks Signed-off-by: F.N. Claessen * fix: mypy Signed-off-by: F.N. Claessen * remove: redundant exception Signed-off-by: F.N. Claessen * feature: get_user Signed-off-by: F.N. Claessen * docs: describe new features Signed-off-by: F.N. Claessen --------- Signed-off-by: VladIftime <49650168+VladIftime@users.noreply.github.com> Signed-off-by: F.N. Claessen Co-authored-by: Vlad Iftime Co-authored-by: F.N. Claessen Signed-off-by: Vlad Iftime --- README.rst | 5 +++++ src/flexmeasures_client/s2/cem.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 94814ab4..3dd0a8ff 100644 --- a/README.rst +++ b/README.rst @@ -62,6 +62,11 @@ Initialization and authentication:: client = FlexMeasuresClient(host="localhost:5000", ssl=False, email="email@email.com", password="pw") client = FlexMeasuresClient(host="seita.energy", ssl=True, email="email@email.com", password="pw") +Retrieve user and account info:: + + user = await client.get_user() + account = await client.get_account() + Retrieve available assets and sensors:: assets = await client.get_assets() diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 37617e52..d7285e80 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -43,8 +43,8 @@ class CEM(Handler): def __init__( self, + sensor_id: int, fm_client: FlexMeasuresClient, - sensor_id: int = 1, logger: Logger | None = None, ) -> None: """ From bdf068475f290126ede4d1d8fec3dd52140a0018 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 13 Dec 2024 15:37:20 +0100 Subject: [PATCH 46/79] Revert unwanted parts of "Get client feature request (#94)" This partly reverts commit 8b05ea2b81f0271cd4583dc861319b032ad7fe48. Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 1 - src/flexmeasures_client/s2/cem.py | 9 ++-- .../s2/script/demo_setup.py | 4 +- .../s2/script/websockets_client.py | 45 ++++++------------- .../s2/script/websockets_server.py | 10 ++--- 5 files changed, 23 insertions(+), 46 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index be14020c..3550cc8c 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -276,7 +276,6 @@ async def post_measurements( prior: str | None = None, ): """ - This function raises a ContentTypeError when the response is not a dictionary. Post sensor data for the given time range. This function raises a ValueError when an unhandled status code is returned """ diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index d7285e80..8d40280a 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -42,10 +42,7 @@ class CEM(Handler): _sending_queue: Queue[pydantic.BaseModel] def __init__( - self, - sensor_id: int, - fm_client: FlexMeasuresClient, - logger: Logger | None = None, + self, fm_client: FlexMeasuresClient, logger: Logger | None = None ) -> None: """ Customer Energy Manager (CEM) @@ -230,7 +227,9 @@ def handle_handshake(self, message: Handshake): def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message - if not self._control_type: # TODO: check if sending resource_manager_details + if ( + not self._control_type + ): # initializing. TODO: check if sending resource_manager_details # resets control type self._control_type = ControlType.NO_SELECTION diff --git a/src/flexmeasures_client/s2/script/demo_setup.py b/src/flexmeasures_client/s2/script/demo_setup.py index 6c3fb430..3524c037 100644 --- a/src/flexmeasures_client/s2/script/demo_setup.py +++ b/src/flexmeasures_client/s2/script/demo_setup.py @@ -3,8 +3,8 @@ from flexmeasures_client.client import FlexMeasuresClient client = FlexMeasuresClient( - email="toy-user@flexmeasures.io", - password="toy-password", + email="admin@admin.com", + password="admin", host="localhost:5000", ) diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 9773cae3..cf3f0cc3 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -3,51 +3,34 @@ import aiohttp import pytz -from s2python.common import ( + +from flexmeasures_client.s2.python_s2_protocol.common.messages import ( + Handshake, + ReceptionStatus, + ReceptionStatusValues, + ResourceManagerDetails, +) +from flexmeasures_client.s2.python_s2_protocol.common.schemas import ( Commodity, CommodityQuantity, ControlType, Duration, EnergyManagementRole, - Handshake, NumberRange, PowerRange, - ReceptionStatus, - ReceptionStatusValues, - ResourceManagerDetails, Role, RoleType, ) -from s2python.frbc import ( +from flexmeasures_client.s2.python_s2_protocol.FRBC.messages import ( + FRBCStorageStatus, + FRBCSystemDescription, +) +from flexmeasures_client.s2.python_s2_protocol.FRBC.schemas import ( FRBCActuatorDescription, FRBCOperationMode, FRBCOperationModeElement, FRBCStorageDescription, - FRBCStorageStatus, - FRBCSystemDescription, ) - -# from flexmeasures_client.s2.python_s2_protocol.common.schemas import ( -# Commodity, -# CommodityQuantity, -# ControlType, -# Duration, -# EnergyManagementRole, -# NumberRange, -# PowerRange, -# Role, -# RoleType, -# ) -# from flexmeasures_client.s2.python_s2_protocol.FRBC.messages import ( -# FRBCStorageStatus, -# FRBCSystemDescription, -# ) -# from flexmeasures_client.s2.python_s2_protocol.FRBC.schemas import ( -# FRBCActuatorDescription, -# FRBCOperationMode, -# FRBCOperationModeElement, -# FRBCStorageDescription, -# ) from flexmeasures_client.s2.utils import get_unique_id @@ -76,7 +59,7 @@ async def main_s2(): roles=[ Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) ], - instruction_processing_delay=Duration(__root__=1), + instruction_processing_delay=Duration(__root__=1.0), available_control_types=[ ControlType.FILL_RATE_BASED_CONTROL, ControlType.NO_SELECTION, diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index c65f9373..8591f6a8 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -3,13 +3,11 @@ import aiohttp from aiohttp import web -from s2python.common import ControlType from flexmeasures_client.client import FlexMeasuresClient from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple - -# from flexmeasures_client.s2.python_s2_protocol.common.schemas import ControlType +from flexmeasures_client.s2.python_s2_protocol.common.schemas import ControlType async def rm_details_watchdog(ws, cem: CEM): @@ -73,10 +71,8 @@ async def websocket_handler(request): fm_client = FlexMeasuresClient("toy-password", "toy-user@flexmeasures.io") - cem = CEM(fm_client=fm_client) - frbc = FRBCSimple( - power_sensor_id=1, price_sensor_id=2, soc_sensor_id=3, rm_discharge_sensor_id=4 - ) + cem = CEM(sensor_id=1, fm_client=fm_client) + frbc = FRBCSimple(power_sensor_id=1, price_sensor_id=2) cem.register_control_type(frbc) # create "parallel" tasks for the message producer and consumer From 170fe7253e51d824fca2a3940cb762fb12584c18 Mon Sep 17 00:00:00 2001 From: VladIftime <49650168+VladIftime@users.noreply.github.com> Date: Wed, 25 Dec 2024 02:23:28 +0200 Subject: [PATCH 47/79] Update readme (#96) * docs: update README with connection instructions and development set-up push Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Update to Readme about installing for testing and for running on different ports Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Fix PR: flake8 Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Fix PR: Test Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Fix PR: Test. Weird regex check fail Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Fix PR: Test. Weird regex check fail Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Fix PR: Test. Weird regex check fail Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime * Update README: Add instructions for retrieving user and account information Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime --------- Signed-off-by: Vlad Iftime Signed-off-by: Vlad Iftime --- README.rst | 13 +++++++++++++ src/flexmeasures_client/client.py | 8 ++++---- src/flexmeasures_client/response_handling.py | 3 +-- tests/test_client.py | 6 +++--- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 3dd0a8ff..58b36621 100644 --- a/README.rst +++ b/README.rst @@ -52,6 +52,8 @@ To create a local instance of FlexMeasures follow the `FlexMeasures documentatio In this example we are connecting to ``localhost:5000``, To connect to a different host add the host in the initialization of the client. +In this example we are connecting to ``localhost:5000``, To connect to a different host add the host in the initialization of the client. + Install using ``pip``:: pip install flexmeasures-client @@ -113,6 +115,17 @@ Trigger a schedule:: sensor_id=, # int start="2023-03-26T10:00+02:00", # iso datetime duration="PT12H", # iso timedelta + flex_context= {"consumption-price-sensor": , # int}, + flex-model= { + "soc-unit": "kWh", + "soc-at-start": 50, # soc_units (kWh) + "soc-max": 400, + "soc-min": 20, + "soc-targets": [ + {"value": 100, "datetime": "2023-03-03T11:00+02:00"} + ], + } + duration="PT12H", # iso timedelta flex_context= {"consumption-price-sensor": }, # int flex-model= { "soc-unit": "kWh", diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 3550cc8c..81eb3ecc 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -78,14 +78,14 @@ def __post_init__(self): if re.match(r"^http\:\/\/", self.host): host_without_scheme = self.host.removeprefix("http://") raise WrongHostError( - f"http:// should not be included in {self.host}." + f"http: // should not be included in {self.host}. " f"Instead use host={host_without_scheme}" ) if re.match(r"^https\:\/\/", self.host): host_without_scheme = self.host.removeprefix("https://") raise WrongHostError( - f"https:// should not be included in {self.host}." - f"To use https:// set ssl=True and host={host_without_scheme}" + f"https: // should not be included in {self.host}." + f"To use https: // set ssl=True and host={host_without_scheme}" ) if len(self.password) < 1: raise EmptyPasswordError("password cannot be empty") @@ -122,7 +122,7 @@ async def request( Retries if: - the client request timed out (as indicated by the client's self.request_timeout) - the server response indicates a 408 (Request Timeout) status - - the server response indicates a 503 (Service Unavailable) status with a Retry-After response header + - the server response indicates a 503 (Service Unavailable) status with a Retry-After response header. Fails if: - the server response indicated a status code of 400 or higher diff --git a/src/flexmeasures_client/response_handling.py b/src/flexmeasures_client/response_handling.py index bec109a5..98b7fde2 100644 --- a/src/flexmeasures_client/response_handling.py +++ b/src/flexmeasures_client/response_handling.py @@ -47,8 +47,7 @@ async def check_response( status: {status} headers: {headers} payload: {payload}. - Re-authenticating! - """ + Re-authenticating!""" logging.debug(message) await self.get_access_token() reauth_once = False diff --git a/tests/test_client.py b/tests/test_client.py index 767e1c57..0e63e867 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -106,7 +106,7 @@ def test__init__( "password": "test_password", }, WrongHostError, - "http:// should not be included in http://test." "Instead use host=test", + "http: // should not be included in http://test." " Instead use host=test", ), ( { @@ -115,8 +115,8 @@ def test__init__( "password": "test_password", }, WrongHostError, - "https:// should not be included in https://test." - "To use https:// set ssl=True and host=test", + "https: // should not be included in https://test." + "To use https: // set ssl=True and host=test", ), ( { From 4579ba70224023191f68d4f416241cef1081f3a0 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 03:15:24 +0200 Subject: [PATCH 48/79] refactor: simplify assignment in PPBC class Signed-off-by: Vlad Iftime --- src/flexmeasures_client/s2/control_types/PPBC/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py index b2f05836..ce9be0d8 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py @@ -38,7 +38,7 @@ def handle_end_interruption_instruction( self, message: PPBCEndInterruptionInstruction ) -> pydantic.BaseModel: end_interruption_instruction_id = str(message.message_id) - self._end_interruption_instruction_history[ - end_interruption_instruction_id - ] = message + self._end_interruption_instruction_history[end_interruption_instruction_id] = ( + message + ) return get_reception_status(message, status=ReceptionStatusValues.OK) From d8f131f9dd34cf5c89f4f76ce3d10bf92137bb77 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 22:05:25 +0200 Subject: [PATCH 49/79] FRBC individual tests Signed-off-by: Vlad Iftime --- tests/test_cem.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_cem.py b/tests/test_cem.py index 56122f62..e5b7c76d 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -5,15 +5,17 @@ from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC import FRBCTest +from flexmeasures_client.s2.control_types.PPBC import PPBC @pytest.mark.asyncio async def test_handshake(rm_handshake): cem = CEM(fm_client=None) frbc = FRBCTest() + ppbc = PPBC() cem.register_control_type(frbc) - + cem.register_control_type(ppbc) ############# # Handshake # ############# @@ -37,7 +39,9 @@ async def test_handshake(rm_handshake): @pytest.mark.asyncio -async def test_resource_manager_details(resource_manager_details, rm_handshake): +async def test_resource_manager_details_frbc( + resource_manager_details_frbc, rm_handshake +): cem = CEM(fm_client=None) frbc = FRBCTest() @@ -60,7 +64,7 @@ async def test_resource_manager_details(resource_manager_details, rm_handshake): ########################## # RM sends ResourceManagerDetails - await cem.handle_message(resource_manager_details) + await cem.handle_message(resource_manager_details_frbc) response = await cem.get_message() # CEM response is ReceptionStatus with an OK status @@ -68,7 +72,7 @@ async def test_resource_manager_details(resource_manager_details, rm_handshake): assert response["status"] == "OK" assert ( - cem._resource_manager_details == resource_manager_details + cem._resource_manager_details == resource_manager_details_frbc ), "CEM should store the resource_manager_details" assert cem.control_type == ControlType.NO_SELECTION, ( "CEM control type should switch to ControlType.NO_SELECTION," @@ -77,8 +81,8 @@ async def test_resource_manager_details(resource_manager_details, rm_handshake): @pytest.mark.asyncio -async def test_activate_control_type( - frbc_system_description, resource_manager_details, rm_handshake +async def test_activate_control_type_frbc( + frbc_system_description, resource_manager_details_frbc, rm_handshake ): cem = CEM(fm_client=None) frbc = FRBCTest() @@ -95,7 +99,7 @@ async def test_activate_control_type( ########################## # ResourceManagerDetails # ########################## - await cem.handle_message(resource_manager_details) + await cem.handle_message(resource_manager_details_frbc) response = await cem.get_message() ######################### @@ -123,8 +127,8 @@ async def test_activate_control_type( @pytest.mark.asyncio -async def test_messages_route_to_control_type_handler( - frbc_system_description, resource_manager_details, rm_handshake +async def test_messages_route_to_control_type_handler_frbc( + frbc_system_description, resource_manager_details_frbc, rm_handshake ): cem = CEM(fm_client=None) frbc = FRBCTest() @@ -141,7 +145,7 @@ async def test_messages_route_to_control_type_handler( ########################## # ResourceManagerDetails # ########################## - await cem.handle_message(resource_manager_details) + await cem.handle_message(resource_manager_details_frbc) response = await cem.get_message() ######################### From ee9e83d819c0ddf371e89c21ab21ca45e80a0401 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 22:09:38 +0200 Subject: [PATCH 50/79] PPBC RM test Signed-off-by: Vlad Iftime --- tests/test_cem.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_cem.py b/tests/test_cem.py index e5b7c76d..aa3b6a6c 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -38,6 +38,7 @@ async def test_handshake(rm_handshake): ), "CEM selected protocol version should be supported by the Resource Manager" +# FRBC @pytest.mark.asyncio async def test_resource_manager_details_frbc( resource_manager_details_frbc, rm_handshake @@ -203,3 +204,44 @@ async def test_messages_route_to_control_type_handler_frbc( ControlType.FILL_RATE_BASED_CONTROL ].success_callbacks ), "success callback should be deleted" + + +# PPBC +@pytest.mark.asyncio +async def test_resource_manager_details_ppbc( + resource_manager_details_ppbc, rm_handshake +): + cem = CEM(fm_client=None) + ppbc = PPBC() + + cem.register_control_type(ppbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + + assert cem._sending_queue.qsize() == 1 + + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + + # RM sends ResourceManagerDetails + await cem.handle_message(resource_manager_details_ppbc) + response = await cem.get_message() + + # CEM response is ReceptionStatus with an OK status + assert response["message_type"] == "ReceptionStatus" + assert response["status"] == "OK" + + assert ( + cem._resource_manager_details == resource_manager_details_ppbc + ), "CEM should store the resource_manager_details" + assert cem.control_type == ControlType.NO_SELECTION, ( + "CEM control type should switch to ControlType.NO_SELECTION," + "independently of the original type" + ) From faab9d838bba6b0a995ca44087de52f1a72785db Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 22:11:32 +0200 Subject: [PATCH 51/79] PPBC RM test Signed-off-by: Vlad Iftime --- tests/test_cem.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_cem.py b/tests/test_cem.py index aa3b6a6c..cce3c324 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -245,3 +245,49 @@ async def test_resource_manager_details_ppbc( "CEM control type should switch to ControlType.NO_SELECTION," "independently of the original type" ) + + +@pytest.mark.asyncio +async def test_activate_control_type_ppbc( + ppbc_system_description, resource_manager_details_ppbc, rm_handshake +): + cem = CEM(fm_client=None) + ppbc = PPBC() + + cem.register_control_type(ppbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(resource_manager_details_ppbc) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### + + # CEM sends a request to change te control type + await cem.activate_control_type(ControlType.POWER_PROFILE_BASED_CONTROL) + message = await cem.get_message() + + assert cem.control_type == ControlType.NO_SELECTION, ( + "the control type should still be NO_SELECTION (rather than PPBC)," + " because the RM has not yet confirmed PPBC activation" + ) + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK + ) + + await cem.handle_message(response) + + assert ( + cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + ), "after a positive ResponseStatus, the status changes from NO_SELECTION to PPBC" From 63cd6ed5869bde9bda6ab92e7a9843d03564d9dc Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 22:16:26 +0200 Subject: [PATCH 52/79] PPBC activation controll type by RM Signed-off-by: Vlad Iftime --- tests/test_cem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cem.py b/tests/test_cem.py index cce3c324..21916107 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -249,7 +249,7 @@ async def test_resource_manager_details_ppbc( @pytest.mark.asyncio async def test_activate_control_type_ppbc( - ppbc_system_description, resource_manager_details_ppbc, rm_handshake + ppbc_power_profile_definition, resource_manager_details_ppbc, rm_handshake ): cem = CEM(fm_client=None) ppbc = PPBC() @@ -289,5 +289,5 @@ async def test_activate_control_type_ppbc( await cem.handle_message(response) assert ( - cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + cem.control_type == ControlType.POWER_PROFILE_BASED_CONTROL ), "after a positive ResponseStatus, the status changes from NO_SELECTION to PPBC" From e2ab228501701e8460526fd799ab173ed21e92a1 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 22:48:02 +0200 Subject: [PATCH 53/79] PPBC message handlers skeleton Signed-off-by: Vlad Iftime --- .../s2/control_types/PPBC/__init__.py | 62 ++++++++++----- tests/test_cem.py | 79 +++++++++++++++++++ 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py index ce9be0d8..b05e9d81 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py @@ -1,6 +1,8 @@ +import asyncio + import pydantic from s2python.common import ControlType, ReceptionStatusValues -from s2python.ppbc import PPBCEndInterruptionInstruction, PPBCScheduleInstruction +from s2python.ppbc import PPBCPowerProfileDefinition, PPBCPowerProfileStatus from flexmeasures_client.s2 import SizeLimitOrderedDict, register from flexmeasures_client.s2.control_types import ControlTypeHandler @@ -12,33 +14,53 @@ class PPBC(ControlTypeHandler): _control_type = ControlType.POWER_PROFILE_BASED_CONTROL - _schedule_instruction_history: SizeLimitOrderedDict[str, PPBCScheduleInstruction] - _end_interruption_instruction_history: SizeLimitOrderedDict[ - str, PPBCEndInterruptionInstruction + _power_profile_definition_history: SizeLimitOrderedDict[ + str, PPBCPowerProfileDefinition ] + _power_profile_status_history: SizeLimitOrderedDict[str, PPBCPowerProfileStatus] def __init__(self, max_size: int = 100) -> None: super().__init__(max_size) - self._schedule_instruction_history = SizeLimitOrderedDict(max_size=max_size) - self._end_interruption_instruction_history = SizeLimitOrderedDict( - max_size=max_size - ) + self._power_profile_definition_history = SizeLimitOrderedDict(max_size=max_size) + self._power_profile_status_history = SizeLimitOrderedDict(max_size=max_size) - @register(PPBCScheduleInstruction) - def handle_schedule_instruction( - self, message: PPBCScheduleInstruction + @register(PPBCPowerProfileDefinition) + def handle_power_profile_definition( + self, message: PPBCPowerProfileDefinition ) -> pydantic.BaseModel: - schedule_instruction_id = str(message.message_id) - self._schedule_instruction_history[schedule_instruction_id] = message + power_profile_id = str(message.id) + + # Store the power profile definition + self._power_profile_definition_history[power_profile_id] = message + + task = asyncio.create_task(self.send_power_profile_definition(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) - @register(PPBCEndInterruptionInstruction) - def handle_end_interruption_instruction( - self, message: PPBCEndInterruptionInstruction + @register(PPBCPowerProfileStatus) + def handle_power_profile_status( + self, message: PPBCPowerProfileStatus ) -> pydantic.BaseModel: - end_interruption_instruction_id = str(message.message_id) - self._end_interruption_instruction_history[end_interruption_instruction_id] = ( - message - ) + power_profile_status_message_id = str(message.message_id) + + # Store the power profile status + self._power_profile_status_history[power_profile_status_message_id] = message + + task = asyncio.create_task(self.send_power_profile_status(message)) + self.background_tasks.add( + task + ) # important to avoid a task disappearing mid-execution. + task.add_done_callback(self.background_tasks.discard) + return get_reception_status(message, status=ReceptionStatusValues.OK) + + async def send_power_profile_definition(self, message: PPBCPowerProfileDefinition): + raise NotImplementedError() + + async def send_power_profile_status(self, message: PPBCPowerProfileStatus): + raise NotImplementedError() diff --git a/tests/test_cem.py b/tests/test_cem.py index 21916107..1f811ab9 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -291,3 +291,82 @@ async def test_activate_control_type_ppbc( assert ( cem.control_type == ControlType.POWER_PROFILE_BASED_CONTROL ), "after a positive ResponseStatus, the status changes from NO_SELECTION to PPBC" + + +@pytest.mark.asyncio +async def test_messages_route_to_control_type_handler_ppbc( + ppbc_power_profile_definition, resource_manager_details_ppbc, rm_handshake +): + cem = CEM(fm_client=None) + ppbc = PPBC() + + cem.register_control_type(ppbc) + + ############# + # Handshake # + ############# + + await cem.handle_message(rm_handshake) + response = await cem.get_message() + + ########################## + # ResourceManagerDetails # + ########################## + await cem.handle_message(ppbc_power_profile_definition) + response = await cem.get_message() + + ######################### + # Activate control type # + ######################### + + await cem.activate_control_type(ControlType.POWER_PROFILE_BASED_CONTROL) + message = await cem.get_message() + + response = ReceptionStatus( + subject_message_id=message.get("message_id"), status=ReceptionStatusValues.OK + ) + + await cem.handle_message(response) + + ######## + # PPBC # + ######## + + await cem.handle_message(ppbc_power_profile_definition) + response = await cem.get_message() + + # checking that PPBC handler is being called + assert ( + cem._control_types_handlers[ + ControlType.POWER_PROFILE_BASED_CONTROL + ]._system_description_history[str(ppbc_power_profile_definition.message_id)] + == ppbc_power_profile_definition + ), ( + "the PPBC. message should be stored" + "in the frbc.system_description_history variable" + ) + + # change of control type is not performed in case that the RM answers + # with a negative response + await cem.activate_control_type(ControlType.NO_SELECTION) + response = await cem.get_message() + assert ( + cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + ), "control type should not change, confirmation still pending" + + await cem.handle_message( + ReceptionStatus( + subject_message_id=response.get("message_id"), + status=ReceptionStatusValues.INVALID_CONTENT, + ) + ) + + assert ( + cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + ), "control type should not change, confirmation state is not 'OK'" + assert ( + response.get("message_id") + not in cem._control_types_handlers[ + ControlType.FILL_RATE_BASED_CONTROL + ].success_callbacks + ), "success callback should be deleted" From 8a7c23caf08111bbb1bce8d8ee9142e9b4d8c790 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Wed, 25 Dec 2024 23:02:35 +0200 Subject: [PATCH 54/79] PPBC controll type handler rounting implemented Signed-off-by: Vlad Iftime --- .../s2/control_types/FRBC/__init__.py | 1 - .../s2/control_types/PPBC/__init__.py | 3 ++- tests/test_cem.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 2b51a75b..194c095e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -43,7 +43,6 @@ def __init__(self, max_size: int = 100) -> None: max_size=max_size ) - # ! ASK: Why are these not being used? self._leakage_behaviour_history = SizeLimitOrderedDict(max_size=max_size) self._usage_forecast_history = SizeLimitOrderedDict(max_size=max_size) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py index b05e9d81..f32b1e2d 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py @@ -24,12 +24,13 @@ def __init__(self, max_size: int = 100) -> None: self._power_profile_definition_history = SizeLimitOrderedDict(max_size=max_size) self._power_profile_status_history = SizeLimitOrderedDict(max_size=max_size) + self.background_tasks = set() @register(PPBCPowerProfileDefinition) def handle_power_profile_definition( self, message: PPBCPowerProfileDefinition ) -> pydantic.BaseModel: - power_profile_id = str(message.id) + power_profile_id = str(message.message_id) # Store the power profile definition self._power_profile_definition_history[power_profile_id] = message diff --git a/tests/test_cem.py b/tests/test_cem.py index 1f811ab9..0a7c8e17 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -312,7 +312,7 @@ async def test_messages_route_to_control_type_handler_ppbc( ########################## # ResourceManagerDetails # ########################## - await cem.handle_message(ppbc_power_profile_definition) + await cem.handle_message(resource_manager_details_ppbc) response = await cem.get_message() ######################### @@ -339,11 +339,13 @@ async def test_messages_route_to_control_type_handler_ppbc( assert ( cem._control_types_handlers[ ControlType.POWER_PROFILE_BASED_CONTROL - ]._system_description_history[str(ppbc_power_profile_definition.message_id)] + ]._power_profile_definition_history[ + str(ppbc_power_profile_definition.message_id) + ] == ppbc_power_profile_definition ), ( - "the PPBC. message should be stored" - "in the frbc.system_description_history variable" + "the PPBC.power_profile_definition message should be stored" + "in the ppbc._power_profile_definition_history variable" ) # change of control type is not performed in case that the RM answers @@ -351,7 +353,7 @@ async def test_messages_route_to_control_type_handler_ppbc( await cem.activate_control_type(ControlType.NO_SELECTION) response = await cem.get_message() assert ( - cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + cem._control_type == ControlType.POWER_PROFILE_BASED_CONTROL ), "control type should not change, confirmation still pending" await cem.handle_message( @@ -362,11 +364,11 @@ async def test_messages_route_to_control_type_handler_ppbc( ) assert ( - cem._control_type == ControlType.FILL_RATE_BASED_CONTROL + cem._control_type == ControlType.POWER_PROFILE_BASED_CONTROL ), "control type should not change, confirmation state is not 'OK'" assert ( response.get("message_id") not in cem._control_types_handlers[ - ControlType.FILL_RATE_BASED_CONTROL + ControlType.POWER_PROFILE_BASED_CONTROL ].success_callbacks ), "success callback should be deleted" From 7bc201c8d3a6bbf440dc38e5070cdfd5b57d7d35 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Thu, 26 Dec 2024 00:57:18 +0200 Subject: [PATCH 55/79] Removed accidental public key Signed-off-by: Vlad Iftime --- public-key.asc | 52 -------------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 public-key.asc diff --git a/public-key.asc b/public-key.asc deleted file mode 100644 index 1b9f3d25..00000000 --- a/public-key.asc +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBGdpc0QBEADSlQSDYaojMIMDcUjAWVuu3XVivJTB76LVhIN2nqpIW47uMdGv -NjnhNdjPHCamX8C6zT+o8HdeY4aSfLnTIaVO1EzgO3twPJsg9YII2DGZdmZy5y7x -aJjDgZVab58rNhvRanLfklt4HYqlsd07eJmSxYmQZgxpsd0a83hig6RIkK/N5Sv4 -DTXtdokzdurOZNeihXSV3n17YJiGgI4OHWY4BiLyXVzVZw8F81rPQvF8hWWd3iqG -IE/c4IiQz2pXq9tLSsg9BnOdLhaN2LzIde5Ya6NsQX0M56HD5OYr6MBei3e6x34a -K/TddKMCpvlzydwHeT54UXLXMdcxptOA4warLMgIcoFN77voSQZcdpojTDn9HFbp -pb7sO18O9OW7SJmO9OeZPMQ/VIBruQOVbN4OVBUiVMDVwV45iAU5MIGpZETAOgVR -/c7wBecLCgFG1MWVw37wCQiZfb6nki/1P//Q+LiUN6su/Bu5TrkMAQT/NsLOG4eZ -1VfNAzfBlYPXVK3Kxlqv7eXv5qLuCz1JcwDPyyKej8BoapqtDiFAZoJD/p+Z8LYX -+p1sLGc8iMw7lQ3dPJAMjB2lyTCuMgGxY0t3qh3TccMqzVOgYXahmDPB9I6yyorN -p4u9ZDag37KwxcK5+weHUR310BwLqjDLRfpwZv91eNJ/OiLTfAgo7dP//wARAQAB -tDRWbGFkIElmdGltZSAoR1BHIGZvciBTZWl0YSkgPHZsYWRpZnRpbWU2MEBnbWFp -bC5jb20+iQJRBBMBCAA7FiEEj0ibEKJxZRRLRO1EXx20V18bO9oFAmdpc0QCGwMF -CwkIBwICIgIGFQoJCAsCBBYCAwECHgcCF4AACgkQXx20V18bO9oVhxAAmK6sulZE -ho3C6vtZRMFFT50QTnDeLPI8dF//KATnhYeirKl6+1suS9jqWOaByaQg1Zu6mO94 -9datsFapL8eGeWue5ED1RPfTmnrefxPh/BvGIsr8IM+kJ1l74hKoqhIp+exPcaed -30JVQnw14ODaxDOT693/bOfSO5CUr+7tf8wdksyicJ7HG66beE1rY5s0kvTP2uI4 -kJy9z0qIHYtK+w0J6l7Vyyv62Fv5/QSGTrKcTvikr0OWTF9ysj6UksCwsApY8u1w -eVJ3n4WounXrCGYyhB2nYpP24MN54GuHmaALL0SSZhpjL5FPUmLb3I+Ta5syi+LE -GVAoUX5/bXjC2Q/l8Mc7zEYD5SWqAfRvRd3aH1hJOiriEJmJrL+x4zv3bdJc6dUY -8QEMN7fCjm+hk1zXNtkU8BFMMB0iBhWvwGxsh6cLJ/qBJ/h5cPvD4n9p+NEtLLml -s8SkFRl8yrIOKLCN68V6PjmAWpZ3GFuPO4FtFi9dNoBlZfbkVhsJhTCdXbYbNGPr -YvSlflfiEyh23jSa3HITvVfnIaYAf21WCOFz0FnvKwK40e9pbR8GXTAatCnbyPEq -8xnARdWxZPLiNlqVBt/1XqBt7azDiCWT8yhu/oZAxMHCT9x95vZqiGjE9JEQUf2b -wUGshBrtLIxQAx6wth/vpGISiTz4zTHVLta5Ag0EZ2lzRAEQAKkZotjNDX0U7uTU -V/E45b+iuMDxluN0+gYnIRMYiVTsWM6WkAEj5n3QO2BwfjieW9RLDWx6E/naNxV2 -KljSGzWG0DnS+7IHtmZOxKm16YWreW+Ojlnf8XvdsNm6BjKi+UgRsA3nPvgyVvoT -3wr7gNQNayRZUOIGXDZno3NZ5y80Ds8eOKdNOHBpUBxZzS4lyIS+6kqfjMZL7v2o -mxgy1B+WMnLoXyc54mmVblcpysUkNICtR9jLiQKZyZY/PjMfkFihrzPTG0hsH6Gq -QCjACMGRR1TxbH0BNFuvtJyt6WhAV2kHq/TFt5G46It/2qugJAHP8F5pjIvwcbWR -3he73xwX4KJmQ1/iJgE3uLK921K6/uCkcCrp88UWWooFuFV0visyVyRH3Lou7Ade -5nkkJEcgO7aYkjaKo142ZSoBC6Wzgn7RgN611ScmzE+2CCbRPTsufBnRy1ULBetF -Ny+M82jQN8FtnKb7nYHS452WPmhvq/L7xE60/0vdbD8W15uqTp2HUZdnINj+GYLs -Yv82ERH2is4Wd6Ow8qzBTuNTxsC4IYuWKqOi6/dpMhp7GUvcUdQwQajcKQvrgLC5 -fRC1MKJzUFFutRFzeWm4HB6Ng1gwO+a+aA4VNORQRgmwTFRnYNtaZ5x2mn6sSzX8 -p8I57V1GpYzZUl/QuB/biqD3zFLJABEBAAGJAjYEGAEIACAWIQSPSJsQonFlFEtE -7URfHbRXXxs72gUCZ2lzRAIbDAAKCRBfHbRXXxs72umuD/sE4xP41Vd2fJ0T4pll -RylfuPTSvRQrAK8Ggr3woWiZl9lxpoeES/6wNAYs2wU29o5AlfB903qOOsr/qHOf -6cf8I2kii1YEtpyYGnqd3S0J1u54CixPH5vL8Vfq1L/0VFFJYl0u/24xpnNtKqaO -m/qcCewILibuhU1M4KLhou0G2E3NYWH+TUHCoSqdP8HBquPFxKs035yz2/hkBhjJ -i6G7rTqA0x5cFf3DN3ge0FIDwRVmJvJlBKK8U320B3npN8/al64dg7uYhC049NrR -DYcEWp9boOKKlq/ebF6St0sKq1MrvMJxD6K6SVxYy9mbxYbK0CoGoAHPW63XthR7 -7RuRktBizUvXy0cbXN272f+odb0Nj7OW+WG9CbC08gQQzFkixpxLDKTZw60XJcJd -diO0JVIGQV7OCgbV2XFCG+qv9YVu1Fgin6xyriKtcSxfPIAFmxND8oboaLW9K46E -dbkv9CHgtkGKPwlKk+B95k08qLXxId8lWt0PpnGLF0c8iqfGI13K2rAIAg2rUy0A -CyLJX1+u1kDMPW2gCLzvH8GbjiL+LKBolCNYvJ3jwrJ9ADZ2wuCmjkUNa9WRyixz -0anf0nE+KjD68X4hkOuXPDUNQe+GpC8WY7viVDE/x5biFLjnC9AZCMxJrxJbqlFk -qp/mh7JdgqneCZYG2eleA28z9w== -=8gvm ------END PGP PUBLIC KEY BLOCK----- From d6bce53e815244572449d4ae57ad12a809f79a26 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Thu, 2 Jan 2025 11:48:46 +0200 Subject: [PATCH 56/79] "Updated PPBC module in flexmeasures client: improved code readability and fixed minor errors" Signed-off-by: Vlad Iftime --- src/flexmeasures_client/s2/control_types/PPBC/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py index f32b1e2d..93101832 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/__init__.py @@ -24,6 +24,7 @@ def __init__(self, max_size: int = 100) -> None: self._power_profile_definition_history = SizeLimitOrderedDict(max_size=max_size) self._power_profile_status_history = SizeLimitOrderedDict(max_size=max_size) + # Keep track of the tasks that are running asynchronously self.background_tasks = set() @register(PPBCPowerProfileDefinition) @@ -38,7 +39,7 @@ def handle_power_profile_definition( task = asyncio.create_task(self.send_power_profile_definition(message)) self.background_tasks.add( task - ) # important to avoid a task disappearing mid-execution. + ) # Important to avoid a task disappearing mid-execution. task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @@ -55,7 +56,7 @@ def handle_power_profile_status( task = asyncio.create_task(self.send_power_profile_status(message)) self.background_tasks.add( task - ) # important to avoid a task disappearing mid-execution. + ) # Important to avoid a task disappearing mid-execution. task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) From c8d7e268c0519b338b727c38a106a2b9ddd6db85 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Fri, 10 Jan 2025 14:26:27 +0100 Subject: [PATCH 57/79] reset to right commit Signed-off-by: Vlad Iftime --- src/flexmeasures_client/client.py | 1 - tests/conftest.py | 5 ----- tests/test_cem.py | 5 ----- 3 files changed, 11 deletions(-) diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 2a287e46..c15bf9bf 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -203,7 +203,6 @@ async def request_once( self.ensure_session() response = await self.session.request( # type: ignore - method=method, url=url, params=params, diff --git a/tests/conftest.py b/tests/conftest.py index 742f590d..8fe8d083 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,6 @@ import uuid from datetime import datetime, timedelta, timezone - import pytest from s2python.common import ( Commodity, @@ -14,7 +13,6 @@ Handshake, NumberRange, PowerForecastValue, - PowerRange, ResourceManagerDetails, Role, @@ -34,7 +32,6 @@ PPBCPowerSequenceElement, ) - from flexmeasures_client.s2.utils import get_unique_id @@ -58,7 +55,6 @@ def frbc_system_description(): thp_operation_mode = FRBCOperationMode( id=str(uuid.uuid4()), - elements=[thp_operation_mode_element], abnormal_condition_only=False, ) @@ -233,7 +229,6 @@ def resource_manager_details_ppbc(): @pytest.fixture(scope="session") - def rm_handshake(): return Handshake( message_id=get_unique_id(), diff --git a/tests/test_cem.py b/tests/test_cem.py index d3a7d060..d7e7264e 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -8,7 +8,6 @@ from flexmeasures_client.s2.control_types.PPBC import PPBC - @pytest.mark.asyncio async def test_handshake(rm_handshake): cem = CEM(fm_client=None) @@ -78,7 +77,6 @@ async def test_resource_manager_details_frbc( assert ( cem._resource_manager_details == resource_manager_details_frbc - ), "CEM should store the resource_manager_details" assert cem.control_type == ControlType.NO_SELECTION, ( "CEM control type should switch to ControlType.NO_SELECTION," @@ -89,7 +87,6 @@ async def test_resource_manager_details_frbc( @pytest.mark.asyncio async def test_activate_control_type_frbc( frbc_system_description, resource_manager_details_frbc, rm_handshake - ): cem = CEM(fm_client=None) frbc = FRBCTest() @@ -137,7 +134,6 @@ async def test_activate_control_type_frbc( @pytest.mark.asyncio async def test_messages_route_to_control_type_handler_frbc( frbc_system_description, resource_manager_details_frbc, rm_handshake - ): cem = CEM(fm_client=None) frbc = FRBCTest() @@ -381,4 +377,3 @@ async def test_messages_route_to_control_type_handler_ppbc( ControlType.POWER_PROFILE_BASED_CONTROL ].success_callbacks ), "success callback should be deleted" - From ea31300a16ea5f5a49d4468167256f689bad5d57 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 11 Jan 2025 02:55:41 +0100 Subject: [PATCH 58/79] revert: linebreaks Signed-off-by: F.N. Claessen --- setup.cfg | 1 - src/flexmeasures_client/s2/control_types/FRBC/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7e6d9aac..ab03748c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,7 +55,6 @@ install_requires = s2-python==0.1.3 async_timeout - [options.packages.find] where = src exclude = diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 194c095e..ad2b8c39 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -42,7 +42,6 @@ def __init__(self, max_size: int = 100) -> None: self._fill_level_target_profile_history = SizeLimitOrderedDict( max_size=max_size ) - self._leakage_behaviour_history = SizeLimitOrderedDict(max_size=max_size) self._usage_forecast_history = SizeLimitOrderedDict(max_size=max_size) From 79d4d9647f0decc923fa32b39f5cc4738b2228c1 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 11 Jan 2025 03:06:20 +0100 Subject: [PATCH 59/79] dev: try installing s2-python from branch containing required imports Signed-off-by: F.N. Claessen --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ab03748c..81ef3b11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas>=2.1.4 pydantic>=1.10.8,<2.0 - s2-python==0.1.3 + s2-python @ git+ssh://git@github.com/flexiblepower/s2-python@Dev-VladIftime-Kiflin-PPBC async_timeout [options.packages.find] From 2923aa90b8a2cf1159624f6f961cd22dcafc051b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Jan 2025 12:55:27 +0100 Subject: [PATCH 60/79] chore: point to s2-python release with PPBC scaffolding Signed-off-by: F.N. Claessen --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 81ef3b11..5eb077fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas>=2.1.4 pydantic>=1.10.8,<2.0 - s2-python @ git+ssh://git@github.com/flexiblepower/s2-python@Dev-VladIftime-Kiflin-PPBC + s2-python>=0.3.1 async_timeout [options.packages.find] From bb427ad55469adc450513af85cca80c7bcbe60d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Jan 2025 13:01:07 +0100 Subject: [PATCH 61/79] chore: add reason for minimum version Signed-off-by: F.N. Claessen --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5eb077fd..5b8931c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = aiohttp<=3.9.1 pandas>=2.1.4 pydantic>=1.10.8,<2.0 - s2-python>=0.3.1 + s2-python>=0.3.1 # minimum version adding PPBC classes async_timeout [options.packages.find] From a71f1fadbb783c07c7c4b4e9b578203eaf4f5f99 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Jan 2025 13:34:51 +0100 Subject: [PATCH 62/79] dev: unpin pydantic (let it go to v2) Signed-off-by: F.N. Claessen --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5b8931c7..0acf6227 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ install_requires = importlib-metadata; python_version<"3.8" aiohttp<=3.9.1 pandas>=2.1.4 - pydantic>=1.10.8,<2.0 + pydantic>=1.10.8 s2-python>=0.3.1 # minimum version adding PPBC classes async_timeout From 72a0c65d5f800e948054ded1f439c01e5fb84730 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Mon, 13 Jan 2025 16:59:33 +0100 Subject: [PATCH 63/79] Removed irrelevant file --- src/flexmeasures_client/s2/wrapper.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 src/flexmeasures_client/s2/wrapper.py diff --git a/src/flexmeasures_client/s2/wrapper.py b/src/flexmeasures_client/s2/wrapper.py deleted file mode 100644 index 0a4d9c06..00000000 --- a/src/flexmeasures_client/s2/wrapper.py +++ /dev/null @@ -1,13 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel, Field -from s2python.message import S2Message - - -class MetaData(BaseModel): - dt: datetime - - -class S2Wrapper(BaseModel): - message: S2Message = Field(discriminator="message_type") - metadata: MetaData From 1210eec2ac4e6d17912a0c0425ea8982b3a0bff9 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Mon, 13 Jan 2025 17:04:54 +0100 Subject: [PATCH 64/79] Removed irrelevant file --- tests/test_s2_models.py | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 tests/test_s2_models.py diff --git a/tests/test_s2_models.py b/tests/test_s2_models.py deleted file mode 100644 index 7c57ce26..00000000 --- a/tests/test_s2_models.py +++ /dev/null @@ -1,31 +0,0 @@ -from flexmeasures_client.s2.utils import get_unique_id -from flexmeasures_client.s2.wrapper import S2Wrapper - - -def test_simple_model(): - wrapped_message = { - "message": { - "message_id": get_unique_id(), - "resource_id": get_unique_id(), - "roles": [{"role": "ENERGY_STORAGE", "commodity": "ELECTRICITY"}], - "instruction_processing_delay": 1.0, - "available_control_types": ["FILL_RATE_BASED_CONTROL", "NO_SELECTION"], - "provides_forecast": True, - "provides_power_measurement_types": ["ELECTRIC.POWER.3_PHASE_SYMMETRIC"], - "message_type": "ResourceManagerDetails", - }, - "metadata": {"dt": "2023-01-01T00:00:00"}, - } - - S2Wrapper.validate(wrapped_message) - - wrapped_message_2 = { - "message": { - "message_id": get_unique_id(), - "message_type": "Handshake", - "role": "CEM", - }, - "metadata": {"dt": "2024-01-01T00:00:00"}, - } - - S2Wrapper.validate(wrapped_message_2) From 29a09b613aae8f10a519a59da2f3c7008f2fe42a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 17:22:40 +0100 Subject: [PATCH 65/79] fix: test fixture Signed-off-by: F.N. Claessen --- tests/conftest.py | 1 + tests/test_frbc_tunes.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8fe8d083..46e2ff8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -190,6 +190,7 @@ def ppbc_power_profile_definition(): return power_profile_definition +@pytest.fixture(scope="session") def resource_manager_details_frbc(): return ResourceManagerDetails( diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 2b98b1a7..886db1d2 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -17,7 +17,7 @@ @pytest.fixture(scope="function") -async def setup_cem(resource_manager_details, rm_handshake): +async def setup_cem(resource_manager_details_frbc, rm_handshake): fm_client = AsyncMock(FlexMeasuresClient) cem = CEM(fm_client=fm_client) frbc = FillRateBasedControlTUNES( @@ -49,7 +49,7 @@ async def setup_cem(resource_manager_details, rm_handshake): ########################## # ResourceManagerDetails # ########################## - await cem.handle_message(resource_manager_details) + await cem.handle_message(resource_manager_details_frbc) response = await cem.get_message() ######################### From 852ea26287d1126ef9d67da49e7e7cebeb8d8b11 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 17:28:56 +0100 Subject: [PATCH 66/79] fix: client returns timezone aware datetimes Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 886db1d2..5823ff7b 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -110,7 +110,7 @@ async def test_system_description(cem_in_frbc_control_type, frbc_system_descript first_call = fm_client.post_measurements.call_args_list[0][1] first_call_expected = { "sensor_id": frbc._thp_efficiency_sensor_id, - "start": datetime(2024, 1, 1), + "start": datetime(2024, 1, 1, tzinfo=timezone(timedelta(seconds=0))), "values": [0.2] * N_SAMPLES, "unit": "%", "duration": "PT24H", @@ -123,7 +123,7 @@ async def test_system_description(cem_in_frbc_control_type, frbc_system_descript second_call_expected = { "sensor_id": frbc._nes_efficiency_sensor_id, - "start": datetime(2024, 1, 1), + "start": datetime(2024, 1, 1, tzinfo=timezone(timedelta(seconds=0))), "values": [0.1] * N_SAMPLES, "unit": "%", "duration": "PT24H", From 78176dc48482776d1041a50ac2a3b39df6179bb4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 17:36:36 +0100 Subject: [PATCH 67/79] fix: use uuid4 Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 5823ff7b..29cff2ae 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -3,6 +3,7 @@ import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock +import uuid import numpy as np import pandas as pd @@ -155,7 +156,7 @@ async def test_fill_level_target_profile(cem_in_frbc_control_type): fill_level_target_profile = { "start_time": "2024-01-01T00:00:00+01:00", "message_type": "FRBC.FillLevelTargetProfile", - "message_id": "a-valid-id", + "message_id": uuid.uuid4(), "elements": [ { "duration": 1e3 * 3600, @@ -212,10 +213,10 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): frbc = cem._control_types_handlers[cem.control_type] actuator_status = { - "active_operation_mode_id": "tarnoc-operation-mode", - "actuator_id": "id-of-the-actuator", + "active_operation_mode_id": uuid.uuid4(), # ID representing Tarnoc operation mode + "actuator_id": uuid.uuid4(), # ID of the actuator "message_type": "FRBC.ActuatorStatus", - "message_id": "a-valid-id", + "message_id": uuid.uuid4(), "operation_mode_factor": 0.0, } @@ -237,7 +238,7 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert second_call["sensor_id"] == frbc._fill_rate_sensor_id # Switch operation mode to Nestore - actuator_status["active_operation_mode_id"] = "nestore-operation-mode" + actuator_status["active_operation_mode_id"] = uuid.uuid4() # ID representing NEStore operation mode await cem.handle_message(actuator_status) tasks = get_pending_tasks() From e774e477d6c0113fef499996d25c71d08becbb58 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 17:50:53 +0100 Subject: [PATCH 68/79] fix: fet correct operation mode ID representing that the initial operation mode is set to that of the Tarnoc Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 29cff2ae..e274c839 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -80,12 +80,12 @@ async def cem_in_frbc_control_type(setup_cem, frbc_system_description): await cem.handle_message(frbc_system_description) await cem.get_message() - return cem, fm_client + return cem, fm_client, frbc_system_description @pytest.mark.asyncio async def test_system_description(cem_in_frbc_control_type, frbc_system_description): - cem, fm_client = await cem_in_frbc_control_type + cem, fm_client, frbc_system_description = await cem_in_frbc_control_type ######## # FRBC # @@ -151,7 +151,7 @@ def get_pending_tasks(): @pytest.mark.asyncio async def test_fill_level_target_profile(cem_in_frbc_control_type): - cem, fm_client = await cem_in_frbc_control_type + cem, fm_client, frbc_system_description = await cem_in_frbc_control_type fill_level_target_profile = { "start_time": "2024-01-01T00:00:00+01:00", @@ -209,11 +209,11 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): corresponds correctly to the Tarnoc fill rate sensor or the Nestor fill rate sensor. """ - cem, fm_client = await cem_in_frbc_control_type + cem, fm_client, frbc_system_description = await cem_in_frbc_control_type frbc = cem._control_types_handlers[cem.control_type] actuator_status = { - "active_operation_mode_id": uuid.uuid4(), # ID representing Tarnoc operation mode + "active_operation_mode_id": frbc_system_description.actuators[0].operation_modes[0].id, # ID representing Tarnoc operation mode "actuator_id": uuid.uuid4(), # ID of the actuator "message_type": "FRBC.ActuatorStatus", "message_id": uuid.uuid4(), @@ -275,7 +275,7 @@ async def test_trigger_schedule(cem_in_frbc_control_type): S2 2 FM: converging system description to flex config FM 2 S2: schedules to instructions """ - cem, fm_client = await cem_in_frbc_control_type + cem, fm_client, frbc_system_description = await cem_in_frbc_control_type # frbc = cem._control_types_handlers[cem.control_type] tasks = get_pending_tasks() From ff242d8e390d084f74c1c1b24d33753bdff1fa26 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 17:53:34 +0100 Subject: [PATCH 69/79] style: black Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index e274c839..a803a588 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -213,7 +213,9 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): frbc = cem._control_types_handlers[cem.control_type] actuator_status = { - "active_operation_mode_id": frbc_system_description.actuators[0].operation_modes[0].id, # ID representing Tarnoc operation mode + "active_operation_mode_id": frbc_system_description.actuators[0] + .operation_modes[0] + .id, # ID representing Tarnoc operation mode "actuator_id": uuid.uuid4(), # ID of the actuator "message_type": "FRBC.ActuatorStatus", "message_id": uuid.uuid4(), @@ -238,7 +240,9 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert second_call["sensor_id"] == frbc._fill_rate_sensor_id # Switch operation mode to Nestore - actuator_status["active_operation_mode_id"] = uuid.uuid4() # ID representing NEStore operation mode + actuator_status["active_operation_mode_id"] = ( + uuid.uuid4() + ) # ID representing NEStore operation mode await cem.handle_message(actuator_status) tasks = get_pending_tasks() From 38b730d51f8155e6c9a480e1f0ea93186c514aac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 17:55:10 +0100 Subject: [PATCH 70/79] style: isort Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index a803a588..ecc1528f 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -1,9 +1,9 @@ from __future__ import annotations import asyncio +import uuid from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock -import uuid import numpy as np import pandas as pd From 475d09403da01ab6bed95e3848ceaf0772c79d9a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:31:14 +0100 Subject: [PATCH 71/79] refactor: use get_unique_id() Signed-off-by: F.N. Claessen --- tests/conftest.py | 25 ++++++++++++------------- tests/test_frbc_tunes.py | 10 +++++----- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 46e2ff8e..8d49a3e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,5 @@ from __future__ import annotations -import uuid from datetime import datetime, timedelta, timezone import pytest @@ -54,7 +53,7 @@ def frbc_system_description(): ) thp_operation_mode = FRBCOperationMode( - id=str(uuid.uuid4()), + id=get_unique_id(), elements=[thp_operation_mode_element], abnormal_condition_only=False, ) @@ -72,13 +71,13 @@ def frbc_system_description(): ) nes_operation_mode = FRBCOperationMode( - id=str(uuid.uuid4()), + id=get_unique_id(), elements=[nes_operation_mode_element], abnormal_condition_only=False, ) actuator = FRBCActuatorDescription( - id=str(uuid.uuid4()), + id=get_unique_id(), supported_commodities=[Commodity.ELECTRICITY], operation_modes=[thp_operation_mode, nes_operation_mode], transitions=[], @@ -122,7 +121,7 @@ def ppbc_power_profile_definition(): ) power_sequence1 = PPBCPowerSequence( - id=uuid.uuid4(), + id=get_unique_id(), elements=[element1, element2], is_interruptible=False, max_pause_before=Duration(0), @@ -130,7 +129,7 @@ def ppbc_power_profile_definition(): ) power_sequence2 = PPBCPowerSequence( - id=uuid.uuid4(), + id=get_unique_id(), elements=[element2, element1], is_interruptible=True, max_pause_before=Duration(0), @@ -138,7 +137,7 @@ def ppbc_power_profile_definition(): ) power_sequence3 = PPBCPowerSequence( - id=uuid.uuid4(), + id=get_unique_id(), elements=[element2], is_interruptible=False, max_pause_before=Duration(10000), @@ -146,7 +145,7 @@ def ppbc_power_profile_definition(): ) power_sequence4 = PPBCPowerSequence( - id=uuid.uuid4(), + id=get_unique_id(), elements=[element1], is_interruptible=True, max_pause_before=Duration(10000), @@ -154,7 +153,7 @@ def ppbc_power_profile_definition(): ) power_sequence_container1 = PPBCPowerSequenceContainer( - id=uuid.uuid4(), + id=get_unique_id(), power_sequences=[ power_sequence1, power_sequence2, @@ -162,22 +161,22 @@ def ppbc_power_profile_definition(): ) power_sequence_container2 = PPBCPowerSequenceContainer( - id=uuid.uuid4(), + id=get_unique_id(), power_sequences=[ power_sequence3, ], ) power_sequence_container3 = PPBCPowerSequenceContainer( - id=uuid.uuid4(), + id=get_unique_id(), power_sequences=[ power_sequence4, ], ) power_profile_definition = PPBCPowerProfileDefinition( - message_id=uuid.uuid4(), - id=uuid.uuid4(), + message_id=get_unique_id(), + id=get_unique_id(), start_time=datetime.now(timezone.utc), end_time=datetime.now(timezone.utc) + timedelta(hours=4), power_sequences_containers=[ diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index ecc1528f..d5a8f6be 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import uuid from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock @@ -15,6 +14,7 @@ from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( FillRateBasedControlTUNES, ) +from flexmeasures_client.s2.utils import get_unique_id @pytest.fixture(scope="function") @@ -156,7 +156,7 @@ async def test_fill_level_target_profile(cem_in_frbc_control_type): fill_level_target_profile = { "start_time": "2024-01-01T00:00:00+01:00", "message_type": "FRBC.FillLevelTargetProfile", - "message_id": uuid.uuid4(), + "message_id": get_unique_id(), "elements": [ { "duration": 1e3 * 3600, @@ -216,9 +216,9 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): "active_operation_mode_id": frbc_system_description.actuators[0] .operation_modes[0] .id, # ID representing Tarnoc operation mode - "actuator_id": uuid.uuid4(), # ID of the actuator + "actuator_id": get_unique_id(), # ID of the actuator "message_type": "FRBC.ActuatorStatus", - "message_id": uuid.uuid4(), + "message_id": get_unique_id(), "operation_mode_factor": 0.0, } @@ -241,7 +241,7 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): # Switch operation mode to Nestore actuator_status["active_operation_mode_id"] = ( - uuid.uuid4() + get_unique_id() ) # ID representing NEStore operation mode await cem.handle_message(actuator_status) From 44f884b1a3720d69fc4bbcab5d4e83f3a6825edc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:31:51 +0100 Subject: [PATCH 72/79] fix: test should rely on exact IDs that were meant, not new IDs Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index d5a8f6be..8a0935b3 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -216,7 +216,7 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): "active_operation_mode_id": frbc_system_description.actuators[0] .operation_modes[0] .id, # ID representing Tarnoc operation mode - "actuator_id": get_unique_id(), # ID of the actuator + "actuator_id": frbc_system_description.actuators[0].id, # ID of the actuator "message_type": "FRBC.ActuatorStatus", "message_id": get_unique_id(), "operation_mode_factor": 0.0, @@ -241,7 +241,7 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): # Switch operation mode to Nestore actuator_status["active_operation_mode_id"] = ( - get_unique_id() + frbc_system_description.actuators[0].operation_modes[1].id ) # ID representing NEStore operation mode await cem.handle_message(actuator_status) From 36b3bdc1ab01a09009f2408dc966e31e5787cbbf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:33:13 +0100 Subject: [PATCH 73/79] refactor: prefer timezone.utc (shorter, clearer) Signed-off-by: F.N. Claessen --- tests/test_frbc_tunes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_frbc_tunes.py b/tests/test_frbc_tunes.py index 8a0935b3..a2223d3a 100644 --- a/tests/test_frbc_tunes.py +++ b/tests/test_frbc_tunes.py @@ -111,7 +111,7 @@ async def test_system_description(cem_in_frbc_control_type, frbc_system_descript first_call = fm_client.post_measurements.call_args_list[0][1] first_call_expected = { "sensor_id": frbc._thp_efficiency_sensor_id, - "start": datetime(2024, 1, 1, tzinfo=timezone(timedelta(seconds=0))), + "start": datetime(2024, 1, 1, tzinfo=timezone.utc), "values": [0.2] * N_SAMPLES, "unit": "%", "duration": "PT24H", @@ -124,7 +124,7 @@ async def test_system_description(cem_in_frbc_control_type, frbc_system_descript second_call_expected = { "sensor_id": frbc._nes_efficiency_sensor_id, - "start": datetime(2024, 1, 1, tzinfo=timezone(timedelta(seconds=0))), + "start": datetime(2024, 1, 1, tzinfo=timezone.utc), "values": [0.1] * N_SAMPLES, "unit": "%", "duration": "PT24H", From 0b2ad4726c97d9889f50d0b8fa362bb3dc299f74 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:35:24 +0100 Subject: [PATCH 74/79] docs: fix typos Signed-off-by: F.N. Claessen --- tests/test_cem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cem.py b/tests/test_cem.py index d7e7264e..36283428 100644 --- a/tests/test_cem.py +++ b/tests/test_cem.py @@ -111,7 +111,7 @@ async def test_activate_control_type_frbc( # Activate control type # ######################### - # CEM sends a request to change te control type + # CEM sends a request to change the control type await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) message = await cem.get_message() @@ -278,7 +278,7 @@ async def test_activate_control_type_ppbc( # Activate control type # ######################### - # CEM sends a request to change te control type + # CEM sends a request to change the control type await cem.activate_control_type(ControlType.POWER_PROFILE_BASED_CONTROL) message = await cem.get_message() From e38cbb1f837deeba7a3b4732bff6807e16926a0f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:55:34 +0100 Subject: [PATCH 75/79] docs: add todo Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py index 2e98863c..ef94c534 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py @@ -38,6 +38,7 @@ def __init__( def now(self): return self._timezone.localize(datetime.now()) + # todo: let's make this more like FRBCSimple.trigger_schedule async def send_schedule_instruction(self, instruction: PPBCScheduleInstruction): await self._fm_client.post_schedule( self._power_sensor_id, From 0843602e5326ebbc61ebbad3c0911a30cf0cf510 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:59:21 +0100 Subject: [PATCH 76/79] docs: add todo Signed-off-by: F.N. Claessen --- .../s2/control_types/PPBC/ppbc_simple.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py index ef94c534..670bd332 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py @@ -32,13 +32,15 @@ def __init__( self._timezone = pytz.timezone(timezone) # delay the start of the schedule from the time `valid_from` - # of the PPBC.SystemDescritption. + # of the PPBC.SystemDescription. self._valid_from_shift = valid_from_shift def now(self): return self._timezone.localize(datetime.now()) - # todo: let's make this more like FRBCSimple.trigger_schedule + # todo: let's make this more like FRBCSimple.trigger_schedule: + # a) call self._fm_client.trigger_and_get_schedule + # b) put instructions to sending queue async def send_schedule_instruction(self, instruction: PPBCScheduleInstruction): await self._fm_client.post_schedule( self._power_sensor_id, From 90a4f7260f2e8a9fbcda116a73d9ddae0082eeae Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 22:02:05 +0100 Subject: [PATCH 77/79] fix: call to FM scheduler in FRBCSimple.trigger_schedule Signed-off-by: F.N. Claessen --- .../s2/control_types/FRBC/frbc_simple.py | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 0b023b73..ae8db5a0 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -91,26 +91,23 @@ async def trigger_schedule(self, system_description_id: str): return # call schedule - schedule_id = await self._fm_client.trigger_storage_schedule( + schedule = await self._fm_client.trigger_and_get_schedule( start=system_description.valid_from + self._valid_from_shift, # TODO: localize datetime sensor_id=self._power_sensor_id, - production_price_sensor=self._price_sensor_id, - consumption_price_sensor=self._price_sensor_id, - soc_unit="MWh", - soc_at_start=soc_at_start, # TODO: use forecast of the SOC instead + flex_context=dict( + production_price_sensor=self._price_sensor_id, + consumption_price_sensor=self._price_sensor_id, + ), + flex_model=dict( + soc_unit="MWh", + soc_at_start=soc_at_start, # TODO: use forecast of the SOC instead + ), duration=self._schedule_duration, # next 12 hours # TODO: add SOC MAX AND SOC MIN FROM fill_level_range, # this needs chages on the client ) - # wait for the schedule to finish - schedule = await self._fm_client.get_schedule( - sensor_id=self._power_sensor_id, - schedule_id=schedule_id, - duration=self._schedule_duration, - ) - # translate FlexMeasures schedule into instructions. SOC -> Power -> PowerFactor instructions = fm_schedule_to_instructions( schedule, system_description, soc_at_start From eea98078a08b756fdeb29faf3c74f27cc60c6f30 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 15 Jan 2025 21:59:38 +0100 Subject: [PATCH 78/79] docs: fix typo Signed-off-by: F.N. Claessen --- src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index ae8db5a0..896c2816 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -105,7 +105,7 @@ async def trigger_schedule(self, system_description_id: str): ), duration=self._schedule_duration, # next 12 hours # TODO: add SOC MAX AND SOC MIN FROM fill_level_range, - # this needs chages on the client + # this needs changes on the client ) # translate FlexMeasures schedule into instructions. SOC -> Power -> PowerFactor From 33be489b7814e89e0c91354c4c6b42dfde927890 Mon Sep 17 00:00:00 2001 From: Vlad Iftime Date: Thu, 16 Jan 2025 09:58:24 +0100 Subject: [PATCH 79/79] Restored the wrapper and wrapper_test for s2 (used in simulations). Started implemenation of trigger_schedule for PPBC simple --- .../s2/control_types/PPBC/ppbc_simple.py | 20 +++++++++++ src/flexmeasures_client/s2/wrapper.py | 13 ++++++++ tests/test_s2_models.py | 33 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 src/flexmeasures_client/s2/wrapper.py create mode 100644 tests/test_s2_models.py diff --git a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py index 670bd332..42caae85 100644 --- a/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/PPBC/ppbc_simple.py @@ -53,3 +53,23 @@ async def send_schedule_instruction(self, instruction: PPBCScheduleInstruction): price_unit="EUR/MWh", valid_from=self.now() + self._valid_from_shift, ) + + async def trigger_schedule(self, definition_id: str): + definition: PPBCScheduleInstruction = self._power_profile_definition_history[ + definition_id + ] + + if len(self._power_profile_status_history) == 0: + print("Can't trigger schedule without knowing the status of the profile...") + return + + # Call schedule + # schedule = await self._fm_client.trigger_and_get_schedule( + # start=definition.start_time + self._valid_from_shift, + # sensor_id=self._power_sensor_id, + # flex_context=dict( + # production_price_sensor=self._price_sensor_id, + # consumption_price_sensor=self._price_sensor_id, + # ), + + # ) diff --git a/src/flexmeasures_client/s2/wrapper.py b/src/flexmeasures_client/s2/wrapper.py new file mode 100644 index 00000000..09003bba --- /dev/null +++ b/src/flexmeasures_client/s2/wrapper.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from pydantic import BaseModel, Field +from s2python.message import S2Message + + +class MetaData(BaseModel): + dt: datetime + + +class S2Wrapper(BaseModel): + message: S2Message = Field(discriminator="message_type") + metadata: MetaData diff --git a/tests/test_s2_models.py b/tests/test_s2_models.py new file mode 100644 index 00000000..905094a6 --- /dev/null +++ b/tests/test_s2_models.py @@ -0,0 +1,33 @@ +from flexmeasures_client.s2.utils import get_unique_id +from flexmeasures_client.s2.wrapper import S2Wrapper +from datetime import datetime +import pytz + + +def test_simple_model(): + wrapped_message = { + "message": { + "message_id": get_unique_id(), + "resource_id": get_unique_id(), + "roles": [{"role": "ENERGY_STORAGE", "commodity": "ELECTRICITY"}], + "instruction_processing_delay": 1.0, + "available_control_types": ["FILL_RATE_BASED_CONTROL", "NO_SELECTION"], + "provides_forecast": True, + "provides_power_measurement_types": ["ELECTRIC.POWER.3_PHASE_SYMMETRIC"], + "message_type": "ResourceManagerDetails", + }, + "metadata": {"dt": "2023-01-01T00:00:00+00:00"}, + } + + S2Wrapper.validate(wrapped_message) + + wrapped_message_2 = { + "message": { + "message_id": get_unique_id(), + "message_type": "Handshake", + "role": "CEM", + }, + "metadata": {"dt": "2024-01-01T00:00:00+00:00"}, + } + + S2Wrapper.validate(wrapped_message_2)