From d03ab1b706bd412ffb07e5061058867047e7d069 Mon Sep 17 00:00:00 2001 From: wasikj Date: Fri, 17 Apr 2026 14:04:08 +0200 Subject: [PATCH 01/28] Add new request model. Rename response models. --- backend/ibex/endpoints/data.py | 12 +++++------- backend/ibex/endpoints/data_entry.py | 2 +- backend/ibex/endpoints/ids_info.py | 2 +- backend/ibex/endpoints/info.py | 2 +- .../endpoints/schemas/request_data_schemas.py | 15 +++++++++++++++ ..._schemas.py => response_data_entry_schemas.py} | 0 .../{data_schemas.py => response_data_schemas.py} | 0 ...fo_schemas.py => response_ids_info_schemas.py} | 0 .../{info_schemas.py => response_info_schemas.py} | 0 9 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 backend/ibex/endpoints/schemas/request_data_schemas.py rename backend/ibex/endpoints/schemas/{data_entry_schemas.py => response_data_entry_schemas.py} (100%) rename backend/ibex/endpoints/schemas/{data_schemas.py => response_data_schemas.py} (100%) rename backend/ibex/endpoints/schemas/{ids_info_schemas.py => response_ids_info_schemas.py} (100%) rename backend/ibex/endpoints/schemas/{info_schemas.py => response_info_schemas.py} (100%) diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 9fbc7d7f..73596432 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -1,13 +1,14 @@ """Endpoints extracting data from data source""" import orjson -from typing import List, Any, Optional +from typing import List, Any, Annotated from fastapi import APIRouter, Query # type: ignore from fastapi.responses import ORJSONResponse # type: ignore from ibex.core import ibex_service -from ibex.endpoints.schemas.data_schemas import FieldValueResponse, PlotDataResponse +from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel +from ibex.endpoints.schemas.response_data_schemas import FieldValueResponse, PlotDataResponse router = APIRouter() @@ -75,10 +76,7 @@ def field_value( ) @ibex_service.measure_execution_time def plot_data( - uri: str, - interpolate_over: Optional[List[str]] = Query(None), - downsampling_method: str | None = Query(None), - downsampled_size: int = 1000, + plot_data_query: Annotated[PlotDataRequestModel, Query()] ) -> Any: """ IBEX endpoint. Prepares and returns full information about data node and it's coordinates. @@ -121,5 +119,5 @@ def plot_data( :return: JSON response """ return CustomORJSONResponse( - ibex_service.get_plot_data(uri.strip(), interpolate_over, downsampling_method, downsampled_size) + ibex_service.get_plot_data(plot_data_query.uri.strip(), plot_data_query.interpolate_over, plot_data_query.downsampling_method, plot_data_query.downsampled_size) ) diff --git a/backend/ibex/endpoints/data_entry.py b/backend/ibex/endpoints/data_entry.py index ff09bd56..bd42e342 100644 --- a/backend/ibex/endpoints/data_entry.py +++ b/backend/ibex/endpoints/data_entry.py @@ -5,7 +5,7 @@ from fastapi import APIRouter # type: ignore from ibex.core import ibex_service -from ibex.endpoints.schemas.data_entry_schemas import ( +from ibex.endpoints.schemas.response_data_entry_schemas import ( UriFromPathResponse, ExistsResponse, ListIdsesResponse, diff --git a/backend/ibex/endpoints/ids_info.py b/backend/ibex/endpoints/ids_info.py index 594399ae..f8c0ad05 100644 --- a/backend/ibex/endpoints/ids_info.py +++ b/backend/ibex/endpoints/ids_info.py @@ -3,7 +3,7 @@ from fastapi import APIRouter # type: ignore from ibex.core import ibex_service -from ibex.endpoints.schemas.ids_info_schemas import NodeInfoResponse, FindPathsResponse, ArraySummaryResponse +from ibex.endpoints.schemas.response_ids_info_schemas import NodeInfoResponse, FindPathsResponse, ArraySummaryResponse router = APIRouter() diff --git a/backend/ibex/endpoints/info.py b/backend/ibex/endpoints/info.py index 5229aa75..e2721109 100644 --- a/backend/ibex/endpoints/info.py +++ b/backend/ibex/endpoints/info.py @@ -5,7 +5,7 @@ from ibex.core import ibex_service from ibex.core.utils import DownsamplingMethods from ibex import __version__ -from ibex.endpoints.schemas.info_schemas import VersionResponse, DownsamplingMethodsResponse +from ibex.endpoints.schemas.response_info_schemas import VersionResponse, DownsamplingMethodsResponse router = APIRouter() diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py new file mode 100644 index 00000000..3ab1a60c --- /dev/null +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, Field +from typing import Optional, List + +# ========== PLOT DATA ========== + +class PlotDataRequestModel(BaseModel): + """...""" + + uri: str = Field(description="IMAS URI") + interpolate_over: Optional[List[str]] = Field(default=None, description="List of IMAS URIs to be used in data interpolation") + downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") + downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") + apply_smoothing: bool | None = Field(default=False, description="Whenever to apply data smoothing to response data") + smoothing_sigma: float | None = Field(default=None, description="Parameter used in Gaussian smoothing algorithm. See https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter.html.") + diff --git a/backend/ibex/endpoints/schemas/data_entry_schemas.py b/backend/ibex/endpoints/schemas/response_data_entry_schemas.py similarity index 100% rename from backend/ibex/endpoints/schemas/data_entry_schemas.py rename to backend/ibex/endpoints/schemas/response_data_entry_schemas.py diff --git a/backend/ibex/endpoints/schemas/data_schemas.py b/backend/ibex/endpoints/schemas/response_data_schemas.py similarity index 100% rename from backend/ibex/endpoints/schemas/data_schemas.py rename to backend/ibex/endpoints/schemas/response_data_schemas.py diff --git a/backend/ibex/endpoints/schemas/ids_info_schemas.py b/backend/ibex/endpoints/schemas/response_ids_info_schemas.py similarity index 100% rename from backend/ibex/endpoints/schemas/ids_info_schemas.py rename to backend/ibex/endpoints/schemas/response_ids_info_schemas.py diff --git a/backend/ibex/endpoints/schemas/info_schemas.py b/backend/ibex/endpoints/schemas/response_info_schemas.py similarity index 100% rename from backend/ibex/endpoints/schemas/info_schemas.py rename to backend/ibex/endpoints/schemas/response_info_schemas.py From 5a1db85ac7c39f808dec564136c395a8617b3f00 Mon Sep 17 00:00:00 2001 From: wasikj Date: Fri, 24 Apr 2026 12:15:16 +0200 Subject: [PATCH 02/28] Data smoothing (WIP) --- backend/ibex/core/ibex_service.py | 9 +++- .../ibex/data_source/imas_python_source.py | 10 ++++ .../data_source/imas_python_source_utils.py | 8 ++++ backend/ibex/endpoints/data.py | 21 +++++---- .../endpoints/schemas/request_data_schemas.py | 11 +++-- .../_templates/custom-module-template.rst | 1 + .../custom-pydantic-model-template.rst | 47 +++++++++++++++++++ .../backend_development/api-hidden.rst | 5 +- .../backend_development.rst | 1 + .../backend_development/data_manipulation.rst | 25 ++++++++++ 10 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 docs/source/_templates/custom-pydantic-model-template.rst create mode 100644 docs/source/developers_manual/backend_development/data_manipulation.rst diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index 4d99c5fc..99878804 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -116,7 +116,12 @@ def get_multiple_node_data(uri: str) -> dict: def get_plot_data( - uri: str, interpolate_over: List[str] | None, downsampling_method: str | None, downsampled_size: int + uri: str, + interpolate_over: List[str] | None, + apply_smoothing: bool | None, + smoothing_sigma: float | None, + downsampling_method: str | None, + downsampled_size: int, ) -> dict: uri_obj = IMAS_URI(uri) return data_source.get_plot_data( @@ -125,6 +130,8 @@ def get_plot_data( node_path=uri_obj.node_path, occurrence=uri_obj.occurrence, interpolate_over=interpolate_over, + apply_smoothing=apply_smoothing, + smoothing_sigma=smoothing_sigma, downsampling_method=downsampling_method, downsampled_size=downsampled_size, ) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index dd87c01c..b57609c9 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -45,6 +45,7 @@ flatten, expand, calculate_coordinate_shapes, + data_smoothing, ) @@ -598,6 +599,8 @@ def get_plot_data( node_path: str, occurrence: int = 0, interpolate_over: List[str] | None = None, + apply_smoothing: bool | None = False, + smoothing_sigma: float | None = 1, downsampling_method: str | None = None, downsampled_size: int = 1000, ): @@ -787,6 +790,13 @@ def get_plot_data( # FE expects data's first dimension to be connected with second dimension, thus this transformation data_to_be_returned = transform_2D_data(data_to_be_returned) + # ============= BEGIN data smoothing ============= + if apply_smoothing or True: + if smoothing_sigma is None: + smoothing_sigma = 1 + data_to_be_returned = data_smoothing(data_to_be_returned, smoothing_sigma) + # ============= END data smoothing ============= + # ============= BEGIN resample data onto new time vector ============= def convert_to_lists(data): diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 31556a31..46ec8d36 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -3,6 +3,14 @@ import numpy as np from imas.ids_primitive import IDSNumericArray from scipy.interpolate import RegularGridInterpolator +from scipy.ndimage import gaussian_filter + + +def data_smoothing(data: list, sigma: float = 1): + if isinstance(data, list): + return [data_smoothing(x) for x in data] + if isinstance(data, (np.ndarray, IDSNumericArray)): + return gaussian_filter(data, sigma=sigma) def union_arrays(data: list): diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 73596432..df530957 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -75,9 +75,7 @@ def field_value( description="Returns single (or tensorized) data node value with detailed parameters used to plot the data", ) @ibex_service.measure_execution_time -def plot_data( - plot_data_query: Annotated[PlotDataRequestModel, Query()] -) -> Any: +def plot_data(plot_data_query: Annotated[PlotDataRequestModel, Query()]) -> CustomORJSONResponse: """ IBEX endpoint. Prepares and returns full information about data node and it's coordinates. @@ -110,14 +108,21 @@ def plot_data( | "value": | } | } + :param plot_data_query: See :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` + :type plot_data_query: :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` - :param uri: IMAS URI with the path to leaf node - :param interpolate_over: list of IMAS URIs used in interpolation. E.g. imas:hdf5?path=/home/ITER/wasikj/Desktop/work/IBEX/testdb2#equilibrium/time_slice[:]/profiles_2d[:]/psi - :param downsampling_method: one of the downsampling metods returend by :func:`~ibex.endpoints.info.downsampling_methods` endpoint, or None - :param downsampled_size: target size of downsampled data :rtype: dict (automatically converted to JSON by FastAPI) :return: JSON response + """ + return CustomORJSONResponse( - ibex_service.get_plot_data(plot_data_query.uri.strip(), plot_data_query.interpolate_over, plot_data_query.downsampling_method, plot_data_query.downsampled_size) + ibex_service.get_plot_data( + uri=plot_data_query.uri.strip(), + interpolate_over=plot_data_query.interpolate_over, + apply_smoothing=plot_data_query.apply_smoothing, + smoothing_sigma=plot_data_query.smoothing_sigma, + downsampling_method=plot_data_query.downsampling_method, + downsampled_size=plot_data_query.downsampled_size, + ) ) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 3ab1a60c..e5f690e1 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -3,13 +3,18 @@ # ========== PLOT DATA ========== + class PlotDataRequestModel(BaseModel): """...""" uri: str = Field(description="IMAS URI") - interpolate_over: Optional[List[str]] = Field(default=None, description="List of IMAS URIs to be used in data interpolation") + interpolate_over: Optional[List[str]] = Field( + default=None, description="List of IMAS URIs to be used in data interpolation" + ) downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") apply_smoothing: bool | None = Field(default=False, description="Whenever to apply data smoothing to response data") - smoothing_sigma: float | None = Field(default=None, description="Parameter used in Gaussian smoothing algorithm. See https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter.html.") - + smoothing_sigma: float | None = Field( + default=None, + description="Parameter used in Gaussian smoothing algorithm. See https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter.html.", + ) diff --git a/docs/source/_templates/custom-module-template.rst b/docs/source/_templates/custom-module-template.rst index bd69dd82..e1157fa6 100644 --- a/docs/source/_templates/custom-module-template.rst +++ b/docs/source/_templates/custom-module-template.rst @@ -58,6 +58,7 @@ .. autosummary:: :toctree: :template: custom-module-template.rst + :recursive: {% for item in modules | reject("equalto", "test") %} {{ item }} {%- endfor %} diff --git a/docs/source/_templates/custom-pydantic-model-template.rst b/docs/source/_templates/custom-pydantic-model-template.rst new file mode 100644 index 00000000..fe699057 --- /dev/null +++ b/docs/source/_templates/custom-pydantic-model-template.rst @@ -0,0 +1,47 @@ +{{ fullname | escape | underline}} + +{% if objtype in ['module', 'package'] %} +.. automodule:: {{ fullname }} + + {% if modules %} + .. rubric:: Modules + + .. autosummary:: + :toctree: + :template: custom-pydantic-model-template.rst + :recursive: + {% for item in modules %} + {{ item }} + {%- endfor %} + {% endif %} + + {% set public_members = [] %} + {% for item in members %} + {% if not item.startswith('_') %} + {% set _ = public_members.append(item) %} + {% endif %} + {% endfor %} + + {% if public_members %} + .. rubric:: Models + + .. autosummary:: + :toctree: + :template: custom-pydantic-model-template.rst + {% for item in public_members %} + {{ fullname }}.{{ item }} + {%- endfor %} + {% endif %} + +{% elif objtype == 'pydantic_model' %} +.. currentmodule:: {{ module }} + +.. autopydantic_model:: {{ objname }} + :members: + :undoc-members: + :model-summary-list-order: bysource + :model-show-validator-members: False + :model-show-validator-summary: False + :model-show-config-summary: False + :model-show-json: False +{% endif %} \ No newline at end of file diff --git a/docs/source/developers_manual/backend_development/api-hidden.rst b/docs/source/developers_manual/backend_development/api-hidden.rst index d3d1a5a2..83101542 100644 --- a/docs/source/developers_manual/backend_development/api-hidden.rst +++ b/docs/source/developers_manual/backend_development/api-hidden.rst @@ -11,4 +11,7 @@ API autosummary :recursive: :template: custom-module-template.rst - ibex \ No newline at end of file + ibex + ibex.endpoints.schemas + + diff --git a/docs/source/developers_manual/backend_development/backend_development.rst b/docs/source/developers_manual/backend_development/backend_development.rst index 6dc22a94..ee5cd177 100644 --- a/docs/source/developers_manual/backend_development/backend_development.rst +++ b/docs/source/developers_manual/backend_development/backend_development.rst @@ -6,6 +6,7 @@ Backend development backend_development_introduction data_interpolation + data_manipulation adding_new_data_source adding_new_downsampling_method benchmarking diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst new file mode 100644 index 00000000..286ea242 --- /dev/null +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -0,0 +1,25 @@ +.. _`Data manipulation`: + +====================== +Data manipulation +====================== + +Introduction +------------- + +The IBEX backend provides a range of data manipulation techniques that directly affect the shape and appearance of the resulting plots. + +Smoothing/Denoising +-------------------- + +IBEX leverages a Gaussian filtering mechanism to smooth irregular data series while reducing the impact of noise. +This approach enhances signal clarity without significantly distorting underlying trends. + +.. note:: + The algorithm is implemented using the Gaussian filter available in ``scipy.ndimage`` package: + https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter.html + +This feature extends the ``data.plot_data`` endpoint with two additional parameters: + +apply_smoothing: Boolean - enables or disables the application of Gaussian smoothing. +smoothing_sigma: Float - specifies the standard deviation of the Gaussian kernel; defaults to 1. \ No newline at end of file From a52b06adbbba63cc0568b17f8c83d0bf9b4d4285 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 28 Apr 2026 13:59:53 +0200 Subject: [PATCH 03/28] Update docs --- .../data_source/imas_python_source_utils.py | 2 +- backend/ibex/endpoints/data.py | 1 + .../_templates/custom-class-template.rst | 4 +- .../_templates/custom-module-template.rst | 47 ++++++++++++++----- .../backend_development/api-hidden.rst | 2 +- 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 46ec8d36..bafa5dfe 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -69,7 +69,7 @@ def expand(data: list, grid_shape: list): :param data: 1D input array of shape (N,) :param grid_shape: target grid shape (e.g. [4, 3, 5]) - :return: broadcasted array of shape (*grid_shape, N) + :return: broadcasted array of shape ``(*grid_shape, N)`` :raises ValueError: if input data is not 1-dimensional """ diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index df530957..835e1a04 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -108,6 +108,7 @@ def plot_data(plot_data_query: Annotated[PlotDataRequestModel, Query()]) -> Cust | "value": | } | } + :param plot_data_query: See :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` :type plot_data_query: :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` diff --git a/docs/source/_templates/custom-class-template.rst b/docs/source/_templates/custom-class-template.rst index b29757c5..0ad8c217 100644 --- a/docs/source/_templates/custom-class-template.rst +++ b/docs/source/_templates/custom-class-template.rst @@ -8,14 +8,14 @@ :inherited-members: {% block methods %} - .. automethod:: __init__ - {% if methods %} .. rubric:: {{ _('Methods') }} .. autosummary:: {% for item in methods %} + {% if item != '__init__' %} ~{{ name }}.{{ item }} + {% endif %} {%- endfor %} {% endif %} {% endblock %} diff --git a/docs/source/_templates/custom-module-template.rst b/docs/source/_templates/custom-module-template.rst index e1157fa6..89a9dbdc 100644 --- a/docs/source/_templates/custom-module-template.rst +++ b/docs/source/_templates/custom-module-template.rst @@ -51,16 +51,39 @@ {% endif %} {% endblock %} -{% block modules %} -{% if modules %} -.. rubric:: Modules + {% block modules %} + {% if modules %} + .. rubric:: Modules -.. autosummary:: - :toctree: - :template: custom-module-template.rst - :recursive: -{% for item in modules | reject("equalto", "test") %} - {{ item }} -{%- endfor %} -{% endif %} -{% endblock %} + {% set schema_modules = [] %} + {% set regular_modules = [] %} + {% for item in modules | reject("equalto", "test") %} + {% if item == "ibex.endpoints.schemas" %} + {% set _ = schema_modules.append(item) %} + {% else %} + {% set _ = regular_modules.append(item) %} + {% endif %} + {% endfor %} + + {% if regular_modules %} + .. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: + {% for item in regular_modules %} + {{ item }} + {%- endfor %} + {% endif %} + + {% if schema_modules %} + .. autosummary:: + :toctree: + :template: custom-pydantic-model-template.rst + :recursive: + {% for item in schema_modules %} + {{ item }} + {%- endfor %} + {% endif %} + + {% endif %} + {% endblock %} diff --git a/docs/source/developers_manual/backend_development/api-hidden.rst b/docs/source/developers_manual/backend_development/api-hidden.rst index 83101542..18e5fcdf 100644 --- a/docs/source/developers_manual/backend_development/api-hidden.rst +++ b/docs/source/developers_manual/backend_development/api-hidden.rst @@ -12,6 +12,6 @@ API autosummary :template: custom-module-template.rst ibex - ibex.endpoints.schemas + From b1c3327ece69e562d5c1114a63c534f0dbafecc0 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 4 May 2026 14:00:02 +0200 Subject: [PATCH 04/28] Add interpolation without data creation. Update docstrings. --- backend/ibex/core/ibex_service.py | 7 +- .../ibex/data_source/imas_python_source.py | 21 ++++- .../data_source/imas_python_source_utils.py | 46 +++++++++- backend/ibex/endpoints/data.py | 10 ++- backend/ibex/endpoints/info.py | 77 ++++++++++++++++- .../ibex/endpoints/schemas/info_schemas.py | 47 ++++++++++ backend/tests/test_data_interpolation.py | 86 +++++++++++++++++++ 7 files changed, 285 insertions(+), 9 deletions(-) diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index 4d99c5fc..4f440b3b 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -116,7 +116,11 @@ def get_multiple_node_data(uri: str) -> dict: def get_plot_data( - uri: str, interpolate_over: List[str] | None, downsampling_method: str | None, downsampled_size: int + uri: str, + interpolate_over: List[str] | None, + interpolation_method: str | None, + downsampling_method: str | None, + downsampled_size: int, ) -> dict: uri_obj = IMAS_URI(uri) return data_source.get_plot_data( @@ -125,6 +129,7 @@ def get_plot_data( node_path=uri_obj.node_path, occurrence=uri_obj.occurrence, interpolate_over=interpolate_over, + interpolation_method=interpolation_method, downsampling_method=downsampling_method, downsampled_size=downsampled_size, ) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index dd87c01c..3eb9c08a 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -40,7 +40,8 @@ from ibex.core.utils import IMAS_URI from ibex.data_source.imas_python_source_utils import ( convert_ids_data_into_numpy_array, - resample_data, + resample_data_with_interpolation, + resample_data_without_interpolation, pad_to_rectangular, flatten, expand, @@ -598,6 +599,7 @@ def get_plot_data( node_path: str, occurrence: int = 0, interpolate_over: List[str] | None = None, + interpolation_method: str | None = None, downsampling_method: str | None = None, downsampled_size: int = 1000, ): @@ -609,6 +611,7 @@ def get_plot_data( :param node_path: path to ids node e.g. ids_properties/version_put :param occurrence: ids occurrence number :param interpolate_over: list of uris used in interpolation + :param interpolation_method: method to be used in data interpolation; one from scipy.interpolate.RegularGridInterpolator or 'exact_value' :param downsampling_method: one of the downsampling metods returend by :func:`~ibex.endpoints.info.downsampling_methods` endpoint, or None :param downsampled_size: target size of downsampled data :return: Dictionary containing data values, metadata and coordinates. @@ -841,9 +844,19 @@ def convert_to_lists(data): # === make data vector rectangular === data_to_be_returned = pad_to_rectangular(data_to_be_returned) - data_to_be_returned = resample_data( - tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values) - ) + + # === run interpolation === + if interpolation_method == "exact_value": + data_to_be_returned = resample_data_without_interpolation( + tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values) + ) + else: + data_to_be_returned = resample_data_with_interpolation( + tuple(original_coord_values), + data_to_be_returned, + tuple(common_coords_values), + interpolation_method=interpolation_method, + ) new_coordinate_shapes = calculate_coordinate_shapes( list(np.asarray(data_to_be_returned).shape), diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 31556a31..199157c9 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -126,7 +126,9 @@ def pad_to_rectangular(lst): return arr -def resample_data(original_coords: list, data: list, target_coords: list): +def resample_data_with_interpolation( + original_coords: list, data: list, target_coords: list, interpolation_method: str | None = None +): """ Resamples data onto new set of coordinates. :param original_coords: List of original data coordinates. @@ -134,7 +136,9 @@ def resample_data(original_coords: list, data: list, target_coords: list): :param target_coords: List of target coordinates. :return: Resampled data array. """ - interpolator = RegularGridInterpolator(original_coords, data, bounds_error=False) + if not interpolation_method: + interpolation_method = "linear" + interpolator = RegularGridInterpolator(original_coords, data, bounds_error=False, method=interpolation_method) # build mesh grid (manipulate coordinates to be list of coordinates e.g. [[x1,y1,z1,h1...], [x2,y2,z2,h3...]]) mesh = np.meshgrid(*target_coords, indexing="ij") @@ -148,6 +152,44 @@ def resample_data(original_coords: list, data: list, target_coords: list): return result +def resample_data_without_interpolation(original_coords, data, target_coords): + """ + Fast exact resampling using dictionaries. + Best for large grids / many dimensions. + """ + + # Create output array filled with NaN + output_shape = [] + for target in target_coords: + output_shape.append(len(target)) + + result = np.full(output_shape, np.nan, dtype=float) + + # Build target indices for each axis + target_indices = [] + + for orig, target in zip(original_coords, target_coords): + # Build dictionary: coordinate -> target index + lookup = {} + for i, value in enumerate(target): + lookup[value] = i + + axis_indices = [] + + for value in orig: + axis_indices.append(lookup[value]) + + target_indices.append(axis_indices) + + # Create mesh + mesh = np.meshgrid(*target_indices, indexing="ij") + + # Copy data + result[tuple(mesh)] = data + + return result + + def convert_ids_data_into_numpy_array(data: list): if isinstance(data, list): diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 9fbc7d7f..3b559461 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -77,6 +77,7 @@ def field_value( def plot_data( uri: str, interpolate_over: Optional[List[str]] = Query(None), + interpolation_method: Optional[List[str]] = Query(None), downsampling_method: str | None = Query(None), downsampled_size: int = 1000, ) -> Any: @@ -115,11 +116,18 @@ def plot_data( :param uri: IMAS URI with the path to leaf node :param interpolate_over: list of IMAS URIs used in interpolation. E.g. imas:hdf5?path=/home/ITER/wasikj/Desktop/work/IBEX/testdb2#equilibrium/time_slice[:]/profiles_2d[:]/psi + :param interpolation_method: method of interpolation; one of the possible parameters provided from /info/data_manipulation_methods :param downsampling_method: one of the downsampling metods returend by :func:`~ibex.endpoints.info.downsampling_methods` endpoint, or None :param downsampled_size: target size of downsampled data :rtype: dict (automatically converted to JSON by FastAPI) :return: JSON response """ return CustomORJSONResponse( - ibex_service.get_plot_data(uri.strip(), interpolate_over, downsampling_method, downsampled_size) + ibex_service.get_plot_data( + uri=uri.strip(), + interpolate_over=interpolate_over, + interpolation_method=interpolation_method, + downsampling_method=downsampling_method, + downsampled_size=downsampled_size, + ) ) diff --git a/backend/ibex/endpoints/info.py b/backend/ibex/endpoints/info.py index 5229aa75..f4e4a892 100644 --- a/backend/ibex/endpoints/info.py +++ b/backend/ibex/endpoints/info.py @@ -5,7 +5,11 @@ from ibex.core import ibex_service from ibex.core.utils import DownsamplingMethods from ibex import __version__ -from ibex.endpoints.schemas.info_schemas import VersionResponse, DownsamplingMethodsResponse +from ibex.endpoints.schemas.info_schemas import ( + VersionResponse, + DownsamplingMethodsResponse, + DataManipulationMethodsResponse, +) router = APIRouter() @@ -70,3 +74,74 @@ def downsampling_methods() -> dict: methods = [{"name": val.value["name"], "description": val.value["description"]} for val in DownsamplingMethods] res = {"downsampling_methods": methods} return res + + +@router.get( + "/info/data_manipulation_methods", + status_code=200, + response_model=DataManipulationMethodsResponse, + responses={ + 200: {"description": "Data manipulation methods returned successfully"}, + }, + description="Returns list of available data manipulation methods provided by the server", +) +@ibex_service.measure_execution_time +def data_manipulation_methods() -> dict: + """ + IBEX endpoint. Returns list of available data manipulation methods to be passed to /data/plot_data endpoint as query argument. + + :rtype: dict (automatically converted to JSON by FastAPI) + :return: JSON response + + """ + res = { + "data_manipulation_methods": [ + { + "name": "Data interpolation", + "description": "Operation performed in order to represent dataset over different set of coordinates", + "method_parameters": [ + { + "human_readable_name": "Interpolate over", + "name": "interpolate_over", + "description": "List of URIs to gather coordinates from, for interpolation", + }, + { + "human_readable_name": "Data interpolation method", + "name": "interpolation_method", + "description": "Method used during data interpolation. All possible for scipy.interpolate.RegularGridInterpolator 'method' parameter or 'exact'", + "possible_values": [ + { + "value": "exact", + "description": "values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", + }, + { + "value": "linear", + "description": "see scipy.interpolate.RegularGridInterpolator documentation", + }, + { + "value": "nearest", + "description": "see scipy.interpolate.RegularGridInterpolator documentation", + }, + { + "value": "slinear", + "description": "see scipy.interpolate.RegularGridInterpolator documentation", + }, + { + "value": "cubic", + "description": "see scipy.interpolate.RegularGridInterpolator documentation", + }, + { + "value": "quintic", + "description": "see scipy.interpolate.RegularGridInterpolator documentation", + }, + { + "value": "pchip", + "description": "see scipy.interpolate.RegularGridInterpolator documentation", + }, + ], + }, + ], + } + ] + } + return res diff --git a/backend/ibex/endpoints/schemas/info_schemas.py b/backend/ibex/endpoints/schemas/info_schemas.py index 17b0c9e2..d03f3d0e 100644 --- a/backend/ibex/endpoints/schemas/info_schemas.py +++ b/backend/ibex/endpoints/schemas/info_schemas.py @@ -20,3 +20,50 @@ class DownsamplingMethodsResponse(BaseModel): """Response for /info/downsampling_methods endpoint""" downsampling_methods: list[DownsamplingMethodModel] = Field(description="Available downsampling methods") + + +# ========== DATA MANIPULATION METHODS ========== + + +class DataManipulationMethodParameterPossibleValuesModel(BaseModel): + """Intermediate model for /info/data_manipulation_methods endpoint""" + + value: str = Field(description="Possible value of parameter", examples=["linear", "nearest"]) + description: str = Field( + description="Value description", + examples=[ + "New point value will be interpolated using linear algorithm", + "New point value will be interpolated using nearest value", + ], + ) + + +class DataManipulationMethodParametersModel(BaseModel): + """Intermediate model for /info/data_manipulation_methods endpoint""" + + human_readable_name: str = Field( + description="Human readable name of the parameter to be displayed in FE", examples=["Sigma", "Deviation level"] + ) + name: str = Field(description="URL parameter name", examples=["sigma", "deviation_level"]) + description: str = Field(description="Parameter description", examples=["Standard deviation for Gaussian kernel."]) + possible_values: list[DataManipulationMethodParameterPossibleValuesModel] | None = Field( + default=None, description="Possible values for manipulation parameter" + ) + + +class DataManipulationMethodModel(BaseModel): + """Intermediate model for /info/data_manipulation_methods endpoint""" + + name: str = Field(description="Method name", examples=["interpolation", "smoothing"]) + description: str = Field(description="Method description", examples=["Gaussian smoothing"]) + method_parameters: list[DataManipulationMethodParametersModel] = Field( + description="Parameters used in data manipulation method" + ) + + +class DataManipulationMethodsResponse(BaseModel): + """Response for /info/data_manipulation_methods endpoint""" + + data_manipulation_methods: list[DataManipulationMethodModel] = Field( + description="Available data manipulation methods" + ) diff --git a/backend/tests/test_data_interpolation.py b/backend/tests/test_data_interpolation.py index ac986f14..5ada6442 100644 --- a/backend/tests/test_data_interpolation.py +++ b/backend/tests/test_data_interpolation.py @@ -1,5 +1,9 @@ import numpy as np import pytest +from ibex.data_source.imas_python_source_utils import ( + resample_data_with_interpolation, + resample_data_without_interpolation, +) def test_simple_interpolation(interpolation_entry_path_directory): @@ -19,3 +23,85 @@ def test_simple_interpolation(interpolation_entry_path_directory): coord_shapes = [list(np.asarray(c["value"]).shape) for c in coords] assert coord_shapes == [[4, 4, 12], [4, 4, 3], [4, 4], [4]] + + +def test_resample_without_interpolation(interpolation_entry_path_directory): + + db_names = [ + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", + f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", + ] + uri_fragment = "#equilibrium/time_slice[:]/profiles_2d[:]/psi" + parameters = {"uri": f"{db_names[0]}/{uri_fragment}", "interpolate_over": [f"{db_names[1]}/{uri_fragment}"]} + response = pytest.test_client.get("/data/plot_data", params=parameters) + + assert response.status_code == 200 + + json_data = response.json()["data"] + coords = json_data["coordinates"] + + coord_shapes = [list(np.asarray(c["value"]).shape) for c in coords] + assert coord_shapes == [[4, 4, 12], [4, 4, 3], [4, 4], [4]] + + +@pytest.mark.skip() +def test_resample_with_interpolation_function(): + # ================== 1D ================== + data_1d = [1.0, 2.0, 3.0] + coords_1d = [[0.0, 1.0, 2.0]] + target_coords_1d = [[0.0, 0.5, 1.0, 1.5, 2.0, 2.5]] + returned_data = resample_data_with_interpolation( + original_coords=coords_1d, data=data_1d, target_coords=target_coords_1d + ) + + expected_returned_data = [1.0, 1.5, 2.0, 2.5, 3.0, np.nan] + assert np.allclose(returned_data, expected_returned_data, equal_nan=True) + + # ================== 2D ================== + data_2d = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] + coords_2d = [[0.0, 1.0, 2.0], [0.0, 1.0, 2.0]] + target_coords_2d = [[0.0, 0.5, 1.0, 1.5, 2.0, 2.5], [0.0, 0.5, 1.0, 1.5, 2.0, 2.5]] + returned_data = resample_data_with_interpolation( + original_coords=coords_2d, data=data_2d, target_coords=target_coords_2d + ) + + expected_returned_data = [ + [1.0, 1.5, 2.0, 2.5, 3.0, np.nan], + [2.5, 3.0, 3.5, 4.0, 4.5, np.nan], + [4.0, 4.5, 5.0, 5.5, 6.0, np.nan], + [5.5, 6.0, 6.5, 7.0, 7.5, np.nan], + [7.0, 7.5, 8.0, 8.5, 9.0, np.nan], + [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], + ] + assert np.allclose(returned_data, expected_returned_data, equal_nan=True) + + +def test_resample_without_interpolation_function(): + + data_1d = [1.0, 2.0, 3.0] + coords_1d = [[0.0, 1.0, 2.0]] + target_coords_1d = [[0.0, 0.5, 1.0, 1.5, 2.0, 2.5]] + returned_data = resample_data_without_interpolation( + original_coords=coords_1d, data=data_1d, target_coords=target_coords_1d + ) + + expected_returned_data = [1.0, np.nan, 2.0, np.nan, 3.0, np.nan] + assert np.allclose(returned_data, expected_returned_data, equal_nan=True) + + # ================== 2D ================== + data_2d = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]] + coords_2d = [[0.0, 1.0, 2.0], [0.0, 1.0, 2.0]] + target_coords_2d = [[0.0, 0.5, 1.0, 1.5, 2.0, 2.5], [0.0, 0.5, 1.0, 1.5, 2.0, 2.5]] + returned_data = resample_data_without_interpolation( + original_coords=coords_2d, data=data_2d, target_coords=target_coords_2d + ) + + expected_returned_data = [ + [1.0, np.nan, 2.0, np.nan, 3.0, np.nan], + [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], + [4.0, np.nan, 5.0, np.nan, 6.0, np.nan], + [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], + [7.0, np.nan, 8.0, np.nan, 9.0, np.nan], + [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], + ] + assert np.allclose(returned_data, expected_returned_data, equal_nan=True) From 279c759ca60c8c2aef69c1cf5e0f7874be503945 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 5 May 2026 08:56:43 +0200 Subject: [PATCH 05/28] Fix interpolation bugs. Update tests. --- .../ibex/data_source/imas_python_source.py | 2 +- .../data_source/imas_python_source_utils.py | 7 +++++- backend/ibex/endpoints/data.py | 2 +- backend/ibex/endpoints/info.py | 2 +- backend/tests/conftest.py | 4 ++-- backend/tests/test_data_interpolation.py | 24 ++++++++++++------- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 3eb9c08a..13590349 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -846,7 +846,7 @@ def convert_to_lists(data): data_to_be_returned = pad_to_rectangular(data_to_be_returned) # === run interpolation === - if interpolation_method == "exact_value": + if interpolation_method == "exact_value" or not interpolation_method: data_to_be_returned = resample_data_without_interpolation( tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values) ) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 199157c9..e43b29f3 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -3,6 +3,7 @@ import numpy as np from imas.ids_primitive import IDSNumericArray from scipy.interpolate import RegularGridInterpolator +from ibex.data_source.exception import InvalidParametersException def union_arrays(data: list): @@ -138,7 +139,11 @@ def resample_data_with_interpolation( """ if not interpolation_method: interpolation_method = "linear" - interpolator = RegularGridInterpolator(original_coords, data, bounds_error=False, method=interpolation_method) + try: + interpolator = RegularGridInterpolator(original_coords, data, bounds_error=False, method=interpolation_method) + except ValueError as e: + message = f"Invalid parameter passed to interpolator: {e}" + raise InvalidParametersException(message) from None # build mesh grid (manipulate coordinates to be list of coordinates e.g. [[x1,y1,z1,h1...], [x2,y2,z2,h3...]]) mesh = np.meshgrid(*target_coords, indexing="ij") diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 3b559461..6cec59d9 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -77,7 +77,7 @@ def field_value( def plot_data( uri: str, interpolate_over: Optional[List[str]] = Query(None), - interpolation_method: Optional[List[str]] = Query(None), + interpolation_method: Optional[str] = Query(None), downsampling_method: str | None = Query(None), downsampled_size: int = 1000, ) -> Any: diff --git a/backend/ibex/endpoints/info.py b/backend/ibex/endpoints/info.py index f4e4a892..e6ce21a3 100644 --- a/backend/ibex/endpoints/info.py +++ b/backend/ibex/endpoints/info.py @@ -111,7 +111,7 @@ def data_manipulation_methods() -> dict: "description": "Method used during data interpolation. All possible for scipy.interpolate.RegularGridInterpolator 'method' parameter or 'exact'", "possible_values": [ { - "value": "exact", + "value": "exact_value", "description": "values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", }, { diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index b7046f64..1bedf148 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -16,7 +16,7 @@ def interpolation_entry_path_directory(tmp_path_factory): eq = entry.factory.equilibrium() eq.ids_properties.homogeneous_time = 1 - eq.time = np.asarray([1, 2, 3, 4]) + eq.time = np.asarray([1, 2, 3, 4], dtype=float) eq.time_slice.resize(4) for ts in eq.time_slice: ts.profiles_2d.resize(2) @@ -30,7 +30,7 @@ def interpolation_entry_path_directory(tmp_path_factory): eq = entry.factory.equilibrium() eq.ids_properties.homogeneous_time = 1 - eq.time = np.asarray([1, 2, 3]) + eq.time = np.asarray([1, 2, 3], dtype=float) eq.time_slice.resize(3) for ts in eq.time_slice: ts.profiles_2d.resize(4) diff --git a/backend/tests/test_data_interpolation.py b/backend/tests/test_data_interpolation.py index 5ada6442..3095ee38 100644 --- a/backend/tests/test_data_interpolation.py +++ b/backend/tests/test_data_interpolation.py @@ -6,23 +6,30 @@ ) -def test_simple_interpolation(interpolation_entry_path_directory): - +def test_interpolation_workflow(interpolation_entry_path_directory): + """ + This function tests only returned data shape. It doesn't check values. + """ db_names = [ f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_1", f"imas:hdf5?path={interpolation_entry_path_directory}/interpolation_db_2", ] uri_fragment = "#equilibrium/time_slice[:]/profiles_2d[:]/psi" parameters = {"uri": f"{db_names[0]}/{uri_fragment}", "interpolate_over": [f"{db_names[1]}/{uri_fragment}"]} - response = pytest.test_client.get("/data/plot_data", params=parameters) - assert response.status_code == 200 + for method in ["exact_value", "linear", "slinear", "nearest"]: + parameters["interpolation_method"] = method + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 - json_data = response.json()["data"] - coords = json_data["coordinates"] + json_data = response.json()["data"] + coords = json_data["coordinates"] + coord_shapes = [list(np.asarray(c["value"]).shape) for c in coords] + assert coord_shapes == [[4, 4, 12], [4, 4, 3], [4, 4], [4]] - coord_shapes = [list(np.asarray(c["value"]).shape) for c in coords] - assert coord_shapes == [[4, 4, 12], [4, 4, 3], [4, 4], [4]] + parameters["interpolation_method"] = "non-existing-method" + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 466 def test_resample_without_interpolation(interpolation_entry_path_directory): @@ -44,7 +51,6 @@ def test_resample_without_interpolation(interpolation_entry_path_directory): assert coord_shapes == [[4, 4, 12], [4, 4, 3], [4, 4], [4]] -@pytest.mark.skip() def test_resample_with_interpolation_function(): # ================== 1D ================== data_1d = [1.0, 2.0, 3.0] From d1a410d788fbfb1a70d968bace99ee6da3fb6f5b Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 5 May 2026 11:51:09 +0200 Subject: [PATCH 06/28] Update interpolation docs. --- .../backend_development/data_interpolation.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/source/developers_manual/backend_development/data_interpolation.rst b/docs/source/developers_manual/backend_development/data_interpolation.rst index 4b8ab757..dbfb60ef 100644 --- a/docs/source/developers_manual/backend_development/data_interpolation.rst +++ b/docs/source/developers_manual/backend_development/data_interpolation.rst @@ -19,6 +19,21 @@ The IDS name and node path must be identical for all URIs participating in the i Failure to meet this requirement will prevent interpolation from being performed. +Configuration +-------------- + +IBEX supports configurable interpolation behavior via the ``interpolation_method`` parameter of the ``/data/plot_data`` endpoint. + +By default, the ``exact_value`` method is used. In this mode, the interpolated dataset retains values only at the original data points, while the coordinate grid may be extended. +No new values are computed between existing points. + +Other supported interpolation methods include ``linear``, ``nearest``, ``slinear``, ``cubic``, ``quintic``, and ``pchip``. +These methods generate interpolated values across the full coordinate grid and follow the behavior described in the SciPy documentation for ``RegularGridInterpolator``. + +You can retrieve the full list of available interpolation methods by querying the ``/info/data_manipulation_methods`` endpoint. + + + Implementation --------------- From 8dfe76946feff674d757dfc167bca589ba6c040a Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 12 May 2026 07:28:04 +0200 Subject: [PATCH 07/28] Merge data-smoothing-denoising into data-smoothing --- backend/ibex/endpoints/data.py | 2 -- backend/ibex/endpoints/info.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 5e1c4d22..967470d0 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -122,8 +122,6 @@ def plot_data(plot_data_query: Annotated[PlotDataRequestModel, Query()]) -> Cust uri=plot_data_query.uri.strip(), interpolate_over=plot_data_query.interpolate_over, interpolation_method=plot_data_query.interpolation_method, - apply_smoothing=plot_data_query.apply_smoothing, - smoothing_sigma=plot_data_query.smoothing_sigma, downsampling_method=plot_data_query.downsampling_method, downsampled_size=plot_data_query.downsampled_size, ) diff --git a/backend/ibex/endpoints/info.py b/backend/ibex/endpoints/info.py index e6ce21a3..237f51f6 100644 --- a/backend/ibex/endpoints/info.py +++ b/backend/ibex/endpoints/info.py @@ -5,7 +5,7 @@ from ibex.core import ibex_service from ibex.core.utils import DownsamplingMethods from ibex import __version__ -from ibex.endpoints.schemas.info_schemas import ( +from ibex.endpoints.schemas.response_info_schemas import ( VersionResponse, DownsamplingMethodsResponse, DataManipulationMethodsResponse, From d0959d0ccff66cdf5f07a398db4cb81afee6440c Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 12 May 2026 13:49:03 +0200 Subject: [PATCH 08/28] Add models for data manipulation --- .../ibex/core/data_manipulation_methods.py | 173 ++++++++++++++++++ backend/ibex/endpoints/info.py | 54 +----- .../endpoints/schemas/request_data_schemas.py | 97 +++++++++- .../schemas/response_info_schemas.py | 8 +- 4 files changed, 268 insertions(+), 64 deletions(-) create mode 100644 backend/ibex/core/data_manipulation_methods.py diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py new file mode 100644 index 00000000..f832f197 --- /dev/null +++ b/backend/ibex/core/data_manipulation_methods.py @@ -0,0 +1,173 @@ +from dataclasses import dataclass +from typing import Optional +from enum import Enum +from pydantic import BaseModel, Field + + +class InterpolationMethod(str, Enum): + EXACT_VALUE = "exact_value" + LINEAR = "linear" + NEAREST = "nearest" + + +class SmoothingMethod(str, Enum): + GAUSSIAN_FILTER = "gaussian_filter" + SAVITZKY_GOLAY_FILTER = "savitzky–golay_filter" + + +class ConnectedParameter(BaseModel): + """ + Parameters for specific data manipulation methods. E.g. sigma -> for Gaussian smoothing + """ + + name: str + human_readable_name: str + description: str + possible_values: Optional[list[str]] = None + + +class PossibleValue(BaseModel): + """ + Possible value for data manipulation method parameter. E.g. Gaussian smoothing + """ + + value: str + description: str + parameters: Optional[list[ConnectedParameter]] = None + + +class DataManipulationParameter(BaseModel): + """ + E.g. interpolate_over, type of filtering, binary operation + """ + + human_readable_name: str + name: str + description: str + possible_values: Optional[list[PossibleValue]] = None + + +class DataManipulationOperation(BaseModel): + """ + Type of data manipulation method. E.g. data-smoothing, data-interpolation, binary-operation + """ + + name: str + description: str + method_parameters: list[DataManipulationParameter] + + +class DataManipulationMethodsResponse(BaseModel): + data_manipulation_methods: list[DataManipulationOperation] + + +available_methods = DataManipulationMethodsResponse(data_manipulation_methods=[]) + +# ====================== DATA INTERPOLATION ====================== + +data_interpolation_description = DataManipulationOperation( + name="Data interpolation", + description="Operation performed in order to represent dataset over different set of coordinates", + method_parameters=[], +) + +# ====================== DATA INTERPOLATION PARAMETERS ====================== + +data_interpolation_interpolate_over_parameter = DataManipulationParameter( + human_readable_name="Interpolate over", + name="interpolate_over", + description="List of URIs to gather coordinates from, for interpolation", +) + +data_interpolation_method_parameter = DataManipulationParameter( + human_readable_name="Interpolation method", + name="interpolation_method", + description="List of URIs to gather coordinates from, for interpolation", + possible_values=[ + PossibleValue( + value=InterpolationMethod.EXACT_VALUE, + description="values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", + ), + PossibleValue( + value=InterpolationMethod.LINEAR, + description="see scipy.interpolate.RegularGridInterpolator documentation", + ), + PossibleValue( + value=InterpolationMethod.NEAREST, + description="see scipy.interpolate.RegularGridInterpolator documentation", + ), + ], +) + +data_interpolation_description.method_parameters.append(data_interpolation_interpolate_over_parameter) +data_interpolation_description.method_parameters.append(data_interpolation_method_parameter) +available_methods.data_manipulation_methods.append(data_interpolation_description) + +# ====================== DATA SMOOTHING ====================== + +data_smoothing_description = DataManipulationOperation( + name="Data smoothing/denoising", + description="Operation performed in order to eliminate noise from data", + method_parameters=[], +) + +# ====================== DATA SMOOTHING PARAMETERS ====================== + +data_smoothing_method_parameter = DataManipulationParameter( + human_readable_name="Smoothing method", + name="smoothing_method", + description="Method to be used in data smoothing process", + possible_values=[ + PossibleValue( + value=SmoothingMethod.GAUSSIAN_FILTER, + description="values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", + parameters=[ + ConnectedParameter( + name="gaussian_smoothing_sigma", + human_readable_name="Sigma", + description="Standard deviation for Gaussian kernel.", + ) + ], + ), + PossibleValue( + value=SmoothingMethod.SAVITZKY_GOLAY_FILTER, + description="see scipy.interpolate.RegularGridInterpolator documentation", + parameters=[ + ConnectedParameter( + name="savgol_smoothing_window_length", + human_readable_name="Window length", + description="The length of the filter window (i.e., the number of coefficients). If mode is ‘interp’, window_length must be less than or equal to the size of x.", + ), + ConnectedParameter( + name="savgol_smoothing_polyorder", + human_readable_name="Polyorder", + description="The order of the polynomial used to fit the samples. polyorder must be less than window_length.", + ), + ConnectedParameter( + name="savgol_smoothing_deriv", + human_readable_name="Deriv", + description="The order of the derivative to compute. This must be a nonnegative integer. The default is 0, which means to filter the data without differentiating.", + ), + ConnectedParameter( + name="savgol_smoothing_delta", + human_readable_name="Window delta", + description="The spacing of the samples to which the filter will be applied. This is only used if deriv > 0. Default is 1.0.", + ), + ConnectedParameter( + name="savgol_smoothing_mode", + human_readable_name="Mode", + description="This determines the type of extension to use for the padded signal to which the filter is applied.", + possible_values=["mirror", "constant", "nearest", "wrap", "interp"], + ), + ConnectedParameter( + name="savgol_smoothing_cval", + human_readable_name="C-Val", + description="Value to fill past the edges of the input if mode is ‘constant’. Default is 0.0.", + ), + ], + ), + ], +) + +data_smoothing_description.method_parameters.append(data_smoothing_method_parameter) +available_methods.data_manipulation_methods.append(data_smoothing_description) diff --git a/backend/ibex/endpoints/info.py b/backend/ibex/endpoints/info.py index 237f51f6..214b918c 100644 --- a/backend/ibex/endpoints/info.py +++ b/backend/ibex/endpoints/info.py @@ -10,6 +10,7 @@ DownsamplingMethodsResponse, DataManipulationMethodsResponse, ) +from ibex.core.data_manipulation_methods import available_methods router = APIRouter() @@ -94,54 +95,5 @@ def data_manipulation_methods() -> dict: :return: JSON response """ - res = { - "data_manipulation_methods": [ - { - "name": "Data interpolation", - "description": "Operation performed in order to represent dataset over different set of coordinates", - "method_parameters": [ - { - "human_readable_name": "Interpolate over", - "name": "interpolate_over", - "description": "List of URIs to gather coordinates from, for interpolation", - }, - { - "human_readable_name": "Data interpolation method", - "name": "interpolation_method", - "description": "Method used during data interpolation. All possible for scipy.interpolate.RegularGridInterpolator 'method' parameter or 'exact'", - "possible_values": [ - { - "value": "exact_value", - "description": "values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", - }, - { - "value": "linear", - "description": "see scipy.interpolate.RegularGridInterpolator documentation", - }, - { - "value": "nearest", - "description": "see scipy.interpolate.RegularGridInterpolator documentation", - }, - { - "value": "slinear", - "description": "see scipy.interpolate.RegularGridInterpolator documentation", - }, - { - "value": "cubic", - "description": "see scipy.interpolate.RegularGridInterpolator documentation", - }, - { - "value": "quintic", - "description": "see scipy.interpolate.RegularGridInterpolator documentation", - }, - { - "value": "pchip", - "description": "see scipy.interpolate.RegularGridInterpolator documentation", - }, - ], - }, - ], - } - ] - } - return res + + return available_methods diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 3f0c2087..2287bbea 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -1,21 +1,104 @@ from pydantic import BaseModel, Field from typing import Optional, List +from ibex.core.data_manipulation_methods import available_methods +from enum import Enum + + +def _get_parameter_possible_values(parameter_name: str) -> list[str]: + """ + Helper function. Return allowed values declared for a top-level data manipulation parameter. + """ + for method in available_methods.data_manipulation_methods: + for method_parameter in method.method_parameters: + if method_parameter.name == parameter_name: + return [possible_value.value for possible_value in (method_parameter.possible_values or [])] + return [] + + +def _get_connected_parameter_possible_values(parameter_name: str) -> list[str]: + """ + Helper function. Returns allowed values for a connected parameter referenced by name. + :param parameter_name: + :return: + """ + for method in available_methods.data_manipulation_methods: + for method_parameter in method.method_parameters: + if not method_parameter.possible_values: + continue + + for possible_value in method_parameter.possible_values: + if not possible_value.parameters: + continue + + for connected_parameter in possible_value.parameters: + if connected_parameter.name == parameter_name: + return connected_parameter.possible_values or [] + + raise ValueError(f"Possible values for '{parameter_name}' not found") + + +# Create ENUM from savgol smoothing mode (for validation) +SavgolSmoothingMode = Enum( + "SavgolSmoothingMode", + {value.upper(): value for value in _get_connected_parameter_possible_values("savgol_smoothing_mode")}, + type=str, +) + +SmoothingAlgorithms = Enum( + "SmoothingAlgorithm", + {value.upper(): value for value in _get_parameter_possible_values("smoothing_method")}, + type=str, +) + # ========== PLOT DATA ========== -class PlotDataRequestModel(BaseModel): +class SavgolSmoothingParameters(BaseModel): + savgol_smoothing_window_length: int | None = Field( + default=None, + description="The length of the filter window (i.e., the number of coefficients). If mode is 'interp', window_length must be less than or equal to the size of x.", + ) + savgol_smoothing_polyorder: int | None = Field( + default=None, + description="The order of the polynomial used to fit the samples. polyorder must be less than window_length.", + ) + savgol_smoothing_deriv: int | None = Field( + default=None, + description="The order of the derivative to compute. This must be a nonnegative integer. The default is 0, which means to filter the data without differentiating.", + ) + savgol_smoothing_delta: float | None = Field( + default=None, + description="The spacing of the samples to which the filter will be applied. This is only used if deriv > 0. Default is 1.0.", + ) + savgol_smoothing_mode: SavgolSmoothingMode = Field( + default=SavgolSmoothingMode.INTERP, + description="Must be 'mirror', 'constant', 'nearest', 'wrap' or 'interp' (default).", + ) + savgol_smoothing_cval: float | None = Field( + default=None, + description="Value to fill past the edges of the input if mode is 'constant'. Default is 0.0.", + ) + + +class GaussianSmoothingParameters(BaseModel): + gaussian_smoothing_sigma: float | None = Field( + default=None, + description="Standard deviation for Gaussian kernel.", + ) + + +class PlotDataBasicParameters(BaseModel): """...""" uri: str = Field(description="IMAS URI") interpolate_over: Optional[List[str]] = Field( default=None, description="List of IMAS URIs to be used in data interpolation" ) - interpolation_method: Optional[str] = Field(default=None, description="Interpolation method to be used"), + interpolation_method: Optional[str] = (Field(default=None, description="Interpolation method to be used"),) downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") - apply_smoothing: bool | None = Field(default=False, description="Whenever to apply data smoothing to response data") - smoothing_sigma: float | None = Field( - default=None, - description="Parameter used in Gaussian smoothing algorithm. See https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.gaussian_filter.html.", - ) + smoothing_method: SmoothingAlgorithms | None = Field(default=None, description="Smoothing method to be used") + + +class PlotDataRequestModel(PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters): ... diff --git a/backend/ibex/endpoints/schemas/response_info_schemas.py b/backend/ibex/endpoints/schemas/response_info_schemas.py index d03f3d0e..5e79e614 100644 --- a/backend/ibex/endpoints/schemas/response_info_schemas.py +++ b/backend/ibex/endpoints/schemas/response_info_schemas.py @@ -1,4 +1,5 @@ from pydantic import BaseModel, Field +from ibex.core import data_manipulation_methods # ========== VERSION ========== @@ -61,9 +62,4 @@ class DataManipulationMethodModel(BaseModel): ) -class DataManipulationMethodsResponse(BaseModel): - """Response for /info/data_manipulation_methods endpoint""" - - data_manipulation_methods: list[DataManipulationMethodModel] = Field( - description="Available data manipulation methods" - ) +DataManipulationMethodsResponse = data_manipulation_methods.DataManipulationMethodsResponse From 9f0a1b87c6537f71ba368c046d2e449234e7fde0 Mon Sep 17 00:00:00 2001 From: wasikj Date: Fri, 15 May 2026 09:49:06 +0200 Subject: [PATCH 09/28] Add missing interface def. Delete old, unused argument. --- .../ibex/data_source/data_source_interface.py | 31 +++++++++++++++++-- .../ibex/data_source/imas_python_source.py | 6 ++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/backend/ibex/data_source/data_source_interface.py b/backend/ibex/data_source/data_source_interface.py index 0189a539..36eb0052 100644 --- a/backend/ibex/data_source/data_source_interface.py +++ b/backend/ibex/data_source/data_source_interface.py @@ -56,7 +56,7 @@ def get_node_info( ... @abstractmethod - def get_data(self, uri: str, ids: str, node_path: str, occurrence: int = 0, range: List[int] | None = None) -> dict: + def get_data(self, uri: str, ids: str, node_path: str, occurrence: int = 0, downsampling_method: str | None = None, downsampled_size: int = 1000,) -> dict: """ Returns data extracted from IDS, converted into dictionary @@ -64,7 +64,8 @@ def get_data(self, uri: str, ids: str, node_path: str, occurrence: int = 0, rang :param ids: name of ids e.g. core_profiles :param node_path: path to ids node e.g. ids_properties/version_put :param occurrence: ids occurrence number - :param range: + :param downsampling_method: method to be used during downsampling process + :param downsampled_size: target size for downsampling :return: dictionary {'value':}, where represents data extracted from IDS node """ ... @@ -113,3 +114,29 @@ def list_db_entries( :return: dictionary {'entries': [, , ...]} """ ... + + def get_plot_data( + self, + uri: str, + ids: str, + node_path: str, + occurrence: int = 0, + interpolate_over: List[str] | None = None, + interpolation_method: str | None = None, + downsampling_method: str | None = None, + downsampled_size: int = 1000, + ) -> dict: + """ + Returns all data used to plot selected quantity. Result contains data values, metadata and coordinates. + + :param uri: imas URI + :param ids: name of ids e.g. core_profiles + :param node_path: path to ids node e.g. ids_properties/version_put + :param occurrence: ids occurrence number + :param interpolate_over: list of uris used in interpolation + :param interpolation_method: method to be used in data interpolation; one from scipy.interpolate.RegularGridInterpolator or 'exact_value' + :param downsampling_method: one of the downsampling metods returend by :func:`~ibex.endpoints.info.downsampling_methods` endpoint, or None + :param downsampled_size: target size of downsampled data + :return: Dictionary containing data values, metadata and coordinates. + """ + ... diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 13590349..c16921ba 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -393,7 +393,6 @@ def get_data( occurrence: int = 0, downsampling_method: str | None = None, downsampled_size: int = 1000, - range: List[int] | None = None, ) -> dict: """ Returns data extracted from IDS, converted into dictionary @@ -402,7 +401,8 @@ def get_data( :param ids: name of ids e.g. core_profiles :param node_path: path to ids node e.g. ids_properties/version_put :param occurrence: ids occurrence number - :param range: + :param downsampling_method: method to be used during downsampling process + :param downsampled_size: target size for downsampling :return: dictionary {'value':}, where represents data extracted from IDS node """ @@ -602,7 +602,7 @@ def get_plot_data( interpolation_method: str | None = None, downsampling_method: str | None = None, downsampled_size: int = 1000, - ): + ) -> dict: """ Returns all data used to plot selected quantity. Result contains data values, metadata and coordinates. From ad14563fb44acca1b7bb0c5cf07351d3c95ed558 Mon Sep 17 00:00:00 2001 From: wasikj Date: Fri, 15 May 2026 10:13:59 +0200 Subject: [PATCH 10/28] Commit of shame - run linter --- backend/ibex/data_source/data_source_interface.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/ibex/data_source/data_source_interface.py b/backend/ibex/data_source/data_source_interface.py index 36eb0052..e457258d 100644 --- a/backend/ibex/data_source/data_source_interface.py +++ b/backend/ibex/data_source/data_source_interface.py @@ -56,7 +56,15 @@ def get_node_info( ... @abstractmethod - def get_data(self, uri: str, ids: str, node_path: str, occurrence: int = 0, downsampling_method: str | None = None, downsampled_size: int = 1000,) -> dict: + def get_data( + self, + uri: str, + ids: str, + node_path: str, + occurrence: int = 0, + downsampling_method: str | None = None, + downsampled_size: int = 1000, + ) -> dict: """ Returns data extracted from IDS, converted into dictionary From 782d1d8599ebf54356a37890fedbfae6bf76cbcf Mon Sep 17 00:00:00 2001 From: wasikj Date: Fri, 15 May 2026 10:24:24 +0200 Subject: [PATCH 11/28] Delete missed argument --- backend/ibex/core/ibex_service.py | 3 +-- backend/ibex/endpoints/data.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index 4f440b3b..63c287de 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -69,7 +69,7 @@ def get_node_info(uri: str, recursive: bool = False, show_error_bars: bool = Fal ) -def get_data(uri: str, downsampling_method: str | None, downsampled_size: int, range: List[int]) -> dict: +def get_data(uri: str, downsampling_method: str | None, downsampled_size: int) -> dict: uri_obj = IMAS_URI(uri) return data_source.get_data( uri=uri_obj.uri_entry_identifiers, @@ -78,7 +78,6 @@ def get_data(uri: str, downsampling_method: str | None, downsampled_size: int, r occurrence=uri_obj.occurrence, downsampling_method=downsampling_method, downsampled_size=downsampled_size, - range=range, ) diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 6cec59d9..031f9830 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -41,7 +41,6 @@ def field_value( uri: str, downsampling_method: str | None = Query(None), downsampled_size: int = 1000, - range: List[int] = Query(None), ) -> Any: """ IBEX endpoint. Returns value extracted from pulsefile's leaf node. @@ -58,7 +57,7 @@ def field_value( :return: JSON response """ - return CustomORJSONResponse(ibex_service.get_data(uri.strip(), downsampling_method, downsampled_size, range)) + return CustomORJSONResponse(ibex_service.get_data(uri.strip(), downsampling_method, downsampled_size)) @router.get( From ed502cb4fcaf10bc2de3a560eb75de075b2a7177 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 18 May 2026 11:55:08 +0200 Subject: [PATCH 12/28] Pass query object down to data source layer --- .../ibex/core/data_manipulation_methods.py | 22 +++---- backend/ibex/core/ibex_service.py | 21 +------ .../ibex/data_source/data_source_interface.py | 23 ++------ .../ibex/data_source/imas_python_source.py | 58 ++++++++----------- backend/ibex/endpoints/data.py | 10 +--- .../endpoints/schemas/request_data_schemas.py | 6 +- 6 files changed, 46 insertions(+), 94 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index f832f197..8baa38d7 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -15,7 +15,7 @@ class SmoothingMethod(str, Enum): SAVITZKY_GOLAY_FILTER = "savitzky–golay_filter" -class ConnectedParameter(BaseModel): +class AdditionalParameter(BaseModel): """ Parameters for specific data manipulation methods. E.g. sigma -> for Gaussian smoothing """ @@ -33,7 +33,7 @@ class PossibleValue(BaseModel): value: str description: str - parameters: Optional[list[ConnectedParameter]] = None + additional_parameters: Optional[list[AdditionalParameter]] = None class DataManipulationParameter(BaseModel): @@ -121,8 +121,8 @@ class DataManipulationMethodsResponse(BaseModel): PossibleValue( value=SmoothingMethod.GAUSSIAN_FILTER, description="values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", - parameters=[ - ConnectedParameter( + additional_parameters=[ + AdditionalParameter( name="gaussian_smoothing_sigma", human_readable_name="Sigma", description="Standard deviation for Gaussian kernel.", @@ -132,34 +132,34 @@ class DataManipulationMethodsResponse(BaseModel): PossibleValue( value=SmoothingMethod.SAVITZKY_GOLAY_FILTER, description="see scipy.interpolate.RegularGridInterpolator documentation", - parameters=[ - ConnectedParameter( + additional_parameters=[ + AdditionalParameter( name="savgol_smoothing_window_length", human_readable_name="Window length", description="The length of the filter window (i.e., the number of coefficients). If mode is ‘interp’, window_length must be less than or equal to the size of x.", ), - ConnectedParameter( + AdditionalParameter( name="savgol_smoothing_polyorder", human_readable_name="Polyorder", description="The order of the polynomial used to fit the samples. polyorder must be less than window_length.", ), - ConnectedParameter( + AdditionalParameter( name="savgol_smoothing_deriv", human_readable_name="Deriv", description="The order of the derivative to compute. This must be a nonnegative integer. The default is 0, which means to filter the data without differentiating.", ), - ConnectedParameter( + AdditionalParameter( name="savgol_smoothing_delta", human_readable_name="Window delta", description="The spacing of the samples to which the filter will be applied. This is only used if deriv > 0. Default is 1.0.", ), - ConnectedParameter( + AdditionalParameter( name="savgol_smoothing_mode", human_readable_name="Mode", description="This determines the type of extension to use for the padded signal to which the filter is applied.", possible_values=["mirror", "constant", "nearest", "wrap", "interp"], ), - ConnectedParameter( + AdditionalParameter( name="savgol_smoothing_cval", human_readable_name="C-Val", description="Value to fill past the edges of the input if mode is ‘constant’. Default is 0.0.", diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index 63c287de..bb2191ba 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -8,6 +8,7 @@ from ibex.data_source.imas_python_source import IMASPythonSource from ibex.data_source.exception import CannotGenerateUriException from ibex.core.utils import IMAS_URI +from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel # helper decorator used during development @@ -114,21 +115,5 @@ def get_multiple_node_data(uri: str) -> dict: ) -def get_plot_data( - uri: str, - interpolate_over: List[str] | None, - interpolation_method: str | None, - downsampling_method: str | None, - downsampled_size: int, -) -> dict: - uri_obj = IMAS_URI(uri) - return data_source.get_plot_data( - uri=uri_obj.uri_entry_identifiers, - ids=uri_obj.ids_name, - node_path=uri_obj.node_path, - occurrence=uri_obj.occurrence, - interpolate_over=interpolate_over, - interpolation_method=interpolation_method, - downsampling_method=downsampling_method, - downsampled_size=downsampled_size, - ) +def get_plot_data(plot_data_query: PlotDataRequestModel) -> dict: + return data_source.get_plot_data(plot_data_query) diff --git a/backend/ibex/data_source/data_source_interface.py b/backend/ibex/data_source/data_source_interface.py index e457258d..b54a3081 100644 --- a/backend/ibex/data_source/data_source_interface.py +++ b/backend/ibex/data_source/data_source_interface.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import Sequence, Optional, List +from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel class DataSourceInterface(ABC): @@ -123,28 +124,12 @@ def list_db_entries( """ ... - def get_plot_data( - self, - uri: str, - ids: str, - node_path: str, - occurrence: int = 0, - interpolate_over: List[str] | None = None, - interpolation_method: str | None = None, - downsampling_method: str | None = None, - downsampled_size: int = 1000, - ) -> dict: + def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: """ Returns all data used to plot selected quantity. Result contains data values, metadata and coordinates. - :param uri: imas URI - :param ids: name of ids e.g. core_profiles - :param node_path: path to ids node e.g. ids_properties/version_put - :param occurrence: ids occurrence number - :param interpolate_over: list of uris used in interpolation - :param interpolation_method: method to be used in data interpolation; one from scipy.interpolate.RegularGridInterpolator or 'exact_value' - :param downsampling_method: one of the downsampling metods returend by :func:`~ibex.endpoints.info.downsampling_methods` endpoint, or None - :param downsampled_size: target size of downsampled data + :param plot_data_query: See :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` + :type plot_data_query: :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` :return: Dictionary containing data values, metadata and coordinates. """ ... diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index c16921ba..514d0a3c 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -5,6 +5,7 @@ import imas # type: ignore import numpy as np # type: ignore import re # type: ignore +from copy import copy # type: ignore from idstools.database import DBMaster # type: ignore from imas.ids_metadata import IDSMetadata # type: ignore from imas.ids_primitive import ( @@ -47,6 +48,7 @@ expand, calculate_coordinate_shapes, ) +from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel class IMASPythonSource(DataSourceInterface): @@ -592,31 +594,21 @@ def _check_data_is_leaf_node(self, data) -> None: elif isinstance(data, IDSStructure): raise NotALeafNodeException("Cannot serialize non-leaf node") - def get_plot_data( - self, - uri: str, - ids: str, - node_path: str, - occurrence: int = 0, - interpolate_over: List[str] | None = None, - interpolation_method: str | None = None, - downsampling_method: str | None = None, - downsampled_size: int = 1000, - ) -> dict: + def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: """ Returns all data used to plot selected quantity. Result contains data values, metadata and coordinates. - :param uri: imas URI - :param ids: name of ids e.g. core_profiles - :param node_path: path to ids node e.g. ids_properties/version_put - :param occurrence: ids occurrence number - :param interpolate_over: list of uris used in interpolation - :param interpolation_method: method to be used in data interpolation; one from scipy.interpolate.RegularGridInterpolator or 'exact_value' - :param downsampling_method: one of the downsampling metods returend by :func:`~ibex.endpoints.info.downsampling_methods` endpoint, or None - :param downsampled_size: target size of downsampled data + :param plot_data_query: See :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` + :type plot_data_query: :class:`ibex.endpoints.schemas.request_data_schemas.PlotDataRequestModel` :return: Dictionary containing data values, metadata and coordinates. """ + uri_obj = IMAS_URI(plot_data_query.uri) + uri = uri_obj.uri_entry_identifiers + ids = uri_obj.ids_name + node_path = uri_obj.node_path + occurrence = uri_obj.occurrence + with self._open_entry(uri) as entry: ids_obj = self._get_ids_from_entry(entry, ids, occurrence) @@ -800,7 +792,7 @@ def convert_to_lists(data): else: return data - if interpolate_over: + if plot_data_query.interpolate_over: # =================== GATHER ALL COORDINATES =================== original_coord_values = [] new_common_coords = coordinates_to_be_returned @@ -809,7 +801,7 @@ def convert_to_lists(data): original_coord_values.append(sorted(set(flatten(c["value"])))) original_coord_values.reverse() - for _uri in interpolate_over: + for _uri in plot_data_query.interpolate_over: _uri_obj = IMAS_URI(_uri) if _uri_obj.ids_name != ids or _uri_obj.node_path != node_path: @@ -817,14 +809,10 @@ def convert_to_lists(data): "IDS name and node path should be the same for source and target URI when interpolating data" ) - interpolate_to_coordinates = self.get_plot_data( - uri=_uri_obj.uri_entry_identifiers, - ids=_uri_obj.ids_name, - node_path=_uri_obj.node_path, - occurrence=_uri_obj.occurrence, - downsampling_method=downsampling_method, - downsampled_size=downsampled_size, - )["data"]["coordinates"] + new_plot_data_query = copy(plot_data_query) + new_plot_data_query.uri = _uri + new_plot_data_query.interpolate_over = None + interpolate_to_coordinates = self.get_plot_data(new_plot_data_query)["data"]["coordinates"] if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): message = "Interpolation error. Source and target nodes have different number of coordinates." @@ -846,7 +834,7 @@ def convert_to_lists(data): data_to_be_returned = pad_to_rectangular(data_to_be_returned) # === run interpolation === - if interpolation_method == "exact_value" or not interpolation_method: + if plot_data_query.interpolation_method == "exact_value" or not plot_data_query.interpolation_method: data_to_be_returned = resample_data_without_interpolation( tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values) ) @@ -855,7 +843,7 @@ def convert_to_lists(data): tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values), - interpolation_method=interpolation_method, + interpolation_method=plot_data_query.interpolation_method, ) new_coordinate_shapes = calculate_coordinate_shapes( @@ -881,15 +869,17 @@ def convert_to_lists(data): # If coordinate targets node -> downsample coordinate as well coordinates_to_be_returned[0]["value"], data_to_be_returned = downsample_data( data_to_be_returned, - target_size=downsampled_size, - method=downsampling_method, + target_size=plot_data_query.downsampled_size, + method=plot_data_query.downsampling_method, x=coordinates_to_be_returned[0]["value"], single_x_axis=(coordinates_to_be_returned[0]["path"] == f"#{ids}/time"), ) else: _, data_to_be_returned = downsample_data( - data_to_be_returned, target_size=downsampled_size, method=downsampling_method + data_to_be_returned, + target_size=plot_data_query.downsampled_size, + method=plot_data_query.downsampling_method, ) # serialize coordinates and update shapes (they could be changed by downsampling) for c in coordinates_to_be_returned: diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 6161b14c..85f31c34 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -116,12 +116,4 @@ def plot_data(plot_data_query: Annotated[PlotDataRequestModel, Query()]) -> Cust """ - return CustomORJSONResponse( - ibex_service.get_plot_data( - uri=plot_data_query.uri.strip(), - interpolate_over=plot_data_query.interpolate_over, - interpolation_method=plot_data_query.interpolation_method, - downsampling_method=plot_data_query.downsampling_method, - downsampled_size=plot_data_query.downsampled_size, - ) - ) + return CustomORJSONResponse(ibex_service.get_plot_data(plot_data_query)) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 2287bbea..6d19e5da 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -27,10 +27,10 @@ def _get_connected_parameter_possible_values(parameter_name: str) -> list[str]: continue for possible_value in method_parameter.possible_values: - if not possible_value.parameters: + if not possible_value.additional_parameters: continue - for connected_parameter in possible_value.parameters: + for connected_parameter in possible_value.additional_parameters: if connected_parameter.name == parameter_name: return connected_parameter.possible_values or [] @@ -95,7 +95,7 @@ class PlotDataBasicParameters(BaseModel): interpolate_over: Optional[List[str]] = Field( default=None, description="List of IMAS URIs to be used in data interpolation" ) - interpolation_method: Optional[str] = (Field(default=None, description="Interpolation method to be used"),) + interpolation_method: str | None = Field(default=None, description="Interpolation method to be used") downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") smoothing_method: SmoothingAlgorithms | None = Field(default=None, description="Smoothing method to be used") From 4c3099606e9f19eccbfe4e8e068177d5ce7ae716 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 19 May 2026 11:38:23 +0200 Subject: [PATCH 13/28] Add data smoothing --- .../ibex/core/data_manipulation_methods.py | 3 +- backend/ibex/core/ibex_service.py | 2 +- .../ibex/data_source/data_source_interface.py | 2 +- .../ibex/data_source/imas_python_source.py | 29 ++++++++- .../data_source/imas_python_source_utils.py | 59 +++++++++++++++++++ backend/ibex/endpoints/data.py | 2 +- .../endpoints/schemas/request_data_schemas.py | 32 ++++++---- backend/tests/test_data_endpoints.py | 56 ++++++++++++++++++ backend/tests/test_data_manipulation.py | 49 +++++++++++++++ 9 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 backend/tests/test_data_manipulation.py diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 8baa38d7..c10d0f1c 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -1,7 +1,6 @@ -from dataclasses import dataclass from typing import Optional from enum import Enum -from pydantic import BaseModel, Field +from pydantic import BaseModel class InterpolationMethod(str, Enum): diff --git a/backend/ibex/core/ibex_service.py b/backend/ibex/core/ibex_service.py index bb2191ba..b1354e39 100644 --- a/backend/ibex/core/ibex_service.py +++ b/backend/ibex/core/ibex_service.py @@ -3,7 +3,7 @@ import time from pathlib import Path from functools import wraps # for measure_execution_time() -from typing import Any, Callable, Optional, Sequence, List +from typing import Any, Callable, Optional, Sequence from ibex.data_source.imas_python_source import IMASPythonSource from ibex.data_source.exception import CannotGenerateUriException diff --git a/backend/ibex/data_source/data_source_interface.py b/backend/ibex/data_source/data_source_interface.py index b54a3081..7c8f6219 100644 --- a/backend/ibex/data_source/data_source_interface.py +++ b/backend/ibex/data_source/data_source_interface.py @@ -1,7 +1,7 @@ """Interface for all data sources""" from abc import ABC, abstractmethod -from typing import Sequence, Optional, List +from typing import Sequence, Optional from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 514d0a3c..c7167e0c 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -47,7 +47,10 @@ flatten, expand, calculate_coordinate_shapes, + apply_savgol_filter, + apply_gaussian_filter, ) +from ibex.core.data_manipulation_methods import SmoothingMethod, InterpolationMethod from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel @@ -782,6 +785,27 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # FE expects data's first dimension to be connected with second dimension, thus this transformation data_to_be_returned = transform_2D_data(data_to_be_returned) + # ============= BEGIN data smoothing ============ + + if plot_data_query.smoothing_method is not None: + if plot_data_query.smoothing_method == SmoothingMethod.SAVITZKY_GOLAY_FILTER: + data_to_be_returned = apply_savgol_filter( + data_to_be_returned, + window_length=plot_data_query.savgol_smoothing_window_length, + polyorder=plot_data_query.savgol_smoothing_polyorder, + deriv=plot_data_query.savgol_smoothing_deriv, + delta=plot_data_query.savgol_smoothing_delta, + mode=plot_data_query.savgol_smoothing_mode, + cval=plot_data_query.savgol_smoothing_cval, + ) + + elif plot_data_query.smoothing_method == SmoothingMethod.GAUSSIAN_FILTER: + data_to_be_returned = apply_gaussian_filter( + data_to_be_returned, sigma=plot_data_query.gaussian_smoothing_sigma + ) + + # ============= END data smoothing ============= + # ============= BEGIN resample data onto new time vector ============= def convert_to_lists(data): @@ -834,7 +858,10 @@ def convert_to_lists(data): data_to_be_returned = pad_to_rectangular(data_to_be_returned) # === run interpolation === - if plot_data_query.interpolation_method == "exact_value" or not plot_data_query.interpolation_method: + if ( + plot_data_query.interpolation_method == InterpolationMethod.EXACT_VALUE + or not plot_data_query.interpolation_method + ): data_to_be_returned = resample_data_without_interpolation( tuple(original_coord_values), data_to_be_returned, tuple(common_coords_values) ) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 45d799ff..795e88a0 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -3,9 +3,68 @@ import numpy as np from imas.ids_primitive import IDSNumericArray from scipy.interpolate import RegularGridInterpolator +from scipy.ndimage import gaussian_filter +from scipy.signal import savgol_filter from ibex.data_source.exception import InvalidParametersException +def apply_savgol_filter( + data: list | np.ndarray, + window_length: int | None, + polyorder: int | None, + deriv: int | None, + delta: float | None, + mode: str | None, + cval: float | None, +): + """ + Apply Savitzky-Golay filer to data + :param data: The input array. + :param window_length: The length of the filter window (i.e., the number of coefficients). If mode is ‘interp’, window_length must be less than or equal to the size of x. + :param polyorder: The order of the polynomial used to fit the samples. polyorder must be less than window_length. + :param deriv: The order of the derivative to compute. This must be a nonnegative integer. The default is 0, which means to filter the data without differentiating. + :param delta: The spacing of the samples to which the filter will be applied. This is only used if deriv > 0. Default is 1.0. + :param mode: Must be ‘mirror’, ‘constant’, ‘nearest’, ‘wrap’ or ‘interp’. + :param cval: Value to fill past the edges of the input if mode is ‘constant’. Default is 0.0. + + :return: Data with filter applied + """ + + params = { + "window_length": window_length, + "polyorder": polyorder, + "deriv": deriv, + "delta": delta, + "mode": mode, + "cval": cval, + } + non_empty_params = {k: v for k, v in params.items() if v is not None} + + if isinstance(data, list): + return [apply_savgol_filter(x, **params) for x in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + return savgol_filter(data, **non_empty_params) + else: + msg = "Smoothing can be executed only on numeric arrays, not single values or strings." + raise InvalidParametersException(msg) + + +def apply_gaussian_filter(data: list | np.ndarray, sigma): + """ + Apply Gaussian filer to data + :param data: The input array. + :param sigma: Standard deviation for Gaussian kernel. The standard deviations of the Gaussian filter are given for each axis as a sequence, or as a single number, in which case it is equal for all axes. + :return: Data with filter applied + """ + if isinstance(data, list): + return [apply_gaussian_filter(x, sigma) for x in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + return gaussian_filter(data, sigma=sigma) + else: + msg = "Smoothing can be executed only on numeric arrays, not single values or strings." + raise InvalidParametersException(msg) + + def union_arrays(data: list): return reduce(np.union1d, data) diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index 85f31c34..cbd19dc4 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -1,7 +1,7 @@ """Endpoints extracting data from data source""" import orjson -from typing import List, Any, Annotated +from typing import Any, Annotated from fastapi import APIRouter, Query # type: ignore from fastapi.responses import ORJSONResponse # type: ignore diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 6d19e5da..f4b93643 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from typing import Optional, List -from ibex.core.data_manipulation_methods import available_methods +from ibex.core.data_manipulation_methods import available_methods, SmoothingMethod from enum import Enum @@ -44,13 +44,6 @@ def _get_connected_parameter_possible_values(parameter_name: str) -> list[str]: type=str, ) -SmoothingAlgorithms = Enum( - "SmoothingAlgorithm", - {value.upper(): value for value in _get_parameter_possible_values("smoothing_method")}, - type=str, -) - - # ========== PLOT DATA ========== @@ -98,7 +91,24 @@ class PlotDataBasicParameters(BaseModel): interpolation_method: str | None = Field(default=None, description="Interpolation method to be used") downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") - smoothing_method: SmoothingAlgorithms | None = Field(default=None, description="Smoothing method to be used") + smoothing_method: SmoothingMethod | None = Field(default=None, description="Smoothing method to be used") + + +class PlotDataRequestModel(PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters): + @model_validator(mode="after") + def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": + if self.smoothing_method == SmoothingMethod.GAUSSIAN_FILTER and self.gaussian_smoothing_sigma is None: + raise ValueError("gaussian_smoothing_sigma is required when smoothing_method is 'gaussian_filter'") + + if self.smoothing_method == SmoothingMethod.SAVITZKY_GOLAY_FILTER: + if self.savgol_smoothing_window_length is None: + raise ValueError( + "savgol_smoothing_window_length is required when smoothing_method is 'savitzky_golay_filter'" + ) + if self.savgol_smoothing_polyorder is None: + raise ValueError( + "savgol_smoothing_polyorder is required when smoothing_method is 'savitzky_golay_filter'" + ) -class PlotDataRequestModel(PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters): ... + return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 5ddd8f51..ecd08af1 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -69,6 +69,33 @@ def test_plot_data(entry_path): assert time_coordinate["description"] == "Generic time" +def test_plot_data_with_gaussian_smoothing(entry_path): + parameters = { + "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", + "smoothing_method": "gaussian_filter", + "gaussian_smoothing_sigma": 1, + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx([1.42, 2.06, 3.0, 3.93, 4.57], 0.1) + + +def test_plot_data_with_savgol_smoothing(entry_path): + parameters = { + "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", + "smoothing_method": "savitzky–golay_filter", + "savgol_smoothing_window_length": 5, + "savgol_smoothing_polyorder": 2, + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx([0.99, 2.0, 3.0, 4.0, 5.0], 0.1) + + def test_plot_data_2d(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/profiles_2d[:]/ion[:]/temperature", @@ -122,3 +149,32 @@ def test_plot_data_1_N_coord(entry_path): assert numeric_coordinate["ndim"] == 1 assert numeric_coordinate["path"] == "" assert numeric_coordinate["description"] == "1...N" + + +def test_plot_data_requires_gaussian_sigma(entry_path): + parameters = { + "uri": f"imas:hdf5?path={entry_path}#core_profiles/profiles_1d[:]/time", + "smoothing_method": "gaussian_filter", + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + + assert response.status_code == 422 + assert "gaussian_smoothing_sigma is required" in response.text + + +def test_plot_data_requires_savgol_window_length_and_polyorder(entry_path): + base_parameters = { + "uri": f"imas:hdf5?path={entry_path}#core_profiles/profiles_1d[:]/time", + "smoothing_method": "savitzky–golay_filter", + } + + response = pytest.test_client.get("/data/plot_data", params=base_parameters) + assert response.status_code == 422 + assert "savgol_smoothing_window_length is required" in response.text + + response = pytest.test_client.get( + "/data/plot_data", + params={**base_parameters, "savgol_smoothing_window_length": 5}, + ) + assert response.status_code == 422 + assert "savgol_smoothing_polyorder is required" in response.text diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py new file mode 100644 index 00000000..c987bc9f --- /dev/null +++ b/backend/tests/test_data_manipulation.py @@ -0,0 +1,49 @@ +import numpy as np +import pytest +from ibex.data_source.imas_python_source_utils import ( + apply_gaussian_filter, + apply_savgol_filter, +) + + +def test_apply_gaussian_smoothing(): + data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1]) + + expected_sigma_1 = [11.343, 13.002, 15.479, 18.228, 21.18, 24.309, 27.414, 30.417, 33.211, 35.019] + expected_sigma_2 = [13.286, 14.311, 16.176, 18.615, 21.385, 24.291, 27.152, 29.743, 31.761, 32.881] + + assert expected_sigma_1 == pytest.approx(apply_gaussian_filter(data, sigma=1), 0.1) + assert expected_sigma_2 == pytest.approx(apply_gaussian_filter(data, sigma=2), 0.1) + + +def test_apply_savitzky_golay_smoothing(): + data = np.array([10.25, 12.8, 15.4, 18.15, 21.0, 24.35, 27.6, 30.2, 33.75, 36.1]) + + expected_window_5_poly_2 = [10.257, 12.781, 15.413, 18.111, 21.086, 24.346, 27.416, 30.521, 33.426, 36.209] + expected_window_7_poly_3_deriv_1 = [2.533, 4.694, 5.717, 5.69, 6.303, 6.242, 6.338, 6.442, 5.184, 2.735] + + assert expected_window_5_poly_2 == pytest.approx( + apply_savgol_filter( + data, + window_length=5, + polyorder=2, + deriv=0, + delta=1.0, + mode="interp", + cval=0.0, + ), + 0.1, + ) + + assert expected_window_7_poly_3_deriv_1 == pytest.approx( + apply_savgol_filter( + data, + window_length=7, + polyorder=3, + deriv=1, + delta=0.5, + mode="nearest", + cval=0.0, + ), + 0.1, + ) From 53958af891f778926c5c63fe8a992a9ed63270d4 Mon Sep 17 00:00:00 2001 From: wasikj Date: Wed, 20 May 2026 14:03:48 +0200 Subject: [PATCH 14/28] Update data manipulation docs --- .../ibex/core/data_manipulation_methods.py | 4 +- .../adding_new_data_manipulation_method.rst | 51 +++++++++++++ .../backend_development.rst | 1 + .../backend_development/data_manipulation.rst | 76 ++++++++++++++++++- 4 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index c10d0f1c..ed755663 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -119,7 +119,7 @@ class DataManipulationMethodsResponse(BaseModel): possible_values=[ PossibleValue( value=SmoothingMethod.GAUSSIAN_FILTER, - description="values are present only on data points where they were originally. Rest of the data grid is filled with NaNs", + description="see scipy.ndimage.gaussian_filter documentation", additional_parameters=[ AdditionalParameter( name="gaussian_smoothing_sigma", @@ -130,7 +130,7 @@ class DataManipulationMethodsResponse(BaseModel): ), PossibleValue( value=SmoothingMethod.SAVITZKY_GOLAY_FILTER, - description="see scipy.interpolate.RegularGridInterpolator documentation", + description="see scipy.signal.savgol_filter documentation", additional_parameters=[ AdditionalParameter( name="savgol_smoothing_window_length", diff --git a/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst b/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst new file mode 100644 index 00000000..775914ff --- /dev/null +++ b/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst @@ -0,0 +1,51 @@ +.. _`Adding new data manipulation method`: + + +Adding a new data manipulation method +----------------------- + +Adding a new data manipulation operation requires updates in three places: the request model, the operation description registry, and the backend execution path. + +Request model +~~~~~~~~~~~~~~ + +Expose the new operation through the request schema in ``backend/ibex/endpoints/schemas/request_data_schemas.py``. + +In practice this usually means: + +* adding a new top-level selector field to ``PlotDataBasicParameters`` if the operation introduces a new method family +* adding a dedicated parameter model when the operation needs extra configuration fields +* extending ``PlotDataRequestModel`` so the new parameters are accepted by ``/data/plot_data/`` +* adding validation in a ``model_validator`` when some parameters are required only for specific operation modes + +This is the layer that defines which query parameters are accepted and how they are validated before the request reaches the data source. + +Operation description registry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Register the operation in ``backend/ibex/core/data_manipulation_methods.py``. + +This file provides the metadata returned by ``/info/data_manipulation_methods/``, so every new operation should be described there using: + +* ``DataManipulationOperation`` for the operation itself +* ``DataManipulationParameter`` for top-level request parameters +* ``PossibleValue`` for supported modes or variants +* ``AdditionalParameter`` for parameters that are only relevant to a specific mode + +This description should match the request schema exactly. +If a parameter is accepted by ``PlotDataRequestModel``, it should also be reflected here so that the API can describe it consistently. + +Backend execution +~~~~~~~~~~~~~~~~~~ + +Implement the actual operation in ``backend/ibex/data_source/imas_python_source.py``. + +This is where the backend transforms the numerical data returned from the IDS source. + +When adding a new operation: + +* read the parameters from ``plot_data_query`` +* apply the transformation to ``data_to_be_returned`` + +If the logic becomes substantial or reusable, the numerical transformation itself should be extracted into a helper function and then called from the data source flow. + diff --git a/docs/source/developers_manual/backend_development/backend_development.rst b/docs/source/developers_manual/backend_development/backend_development.rst index ee5cd177..1a06293d 100644 --- a/docs/source/developers_manual/backend_development/backend_development.rst +++ b/docs/source/developers_manual/backend_development/backend_development.rst @@ -7,6 +7,7 @@ Backend development backend_development_introduction data_interpolation data_manipulation + adding_new_data_manipulation_method adding_new_data_source adding_new_downsampling_method benchmarking diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index 69409be6..53c172c2 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -8,8 +8,78 @@ Introduction ------------- The IBEX backend provides a range of data manipulation techniques that directly affect the shape and appearance of the resulting plots. +These operations are applied as part of the ``/data/plot_data/`` request flow and allow the backend to transform datasets before they are returned to the frontend. -Smoothing/Denoising --------------------- +Data smoothing +--------------- -Test text \ No newline at end of file +IBEX supports smoothing and denoising of returned datasets. +This functionality is intended for cases where the raw signal contains high-frequency noise and a filtered representation is preferred for visualization or analysis. + +Configuration +~~~~~~~~~~~~~~ + +Data smoothing is configured through the ``smoothing_method`` parameter of the ``/data/plot_data/`` endpoint. + +At the moment, the backend supports the following smoothing methods: + +* ``gaussian_filter`` +* ``savitzky_golay_filter`` + +The full list of available methods and their parameters can be retrieved from the ``/info/data_manipulation_methods/`` endpoint. + +Gaussian smoothing +~~~~~~~~~~~~~~~~~~~ + +The ``gaussian_filter`` method applies a Gaussian kernel to the returned data. +It requires the ``gaussian_smoothing_sigma`` parameter, which defines the standard deviation of the Gaussian kernel. + + +Savitzky-Golay smoothing +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``savitzky_golay_filter`` method applies a Savitzky-Golay filter to the returned data. +This approach smooths the data while preserving local shape better than a simple Gaussian filter in many cases. + +The following parameters are supported: + +* ``savgol_smoothing_window_length``: required +* ``savgol_smoothing_polyorder``: required +* ``savgol_smoothing_deriv``: optional +* ``savgol_smoothing_delta``: optional +* ``savgol_smoothing_mode``: optional, one of ``mirror``, ``constant``, ``nearest``, ``wrap``, ``interp`` +* ``savgol_smoothing_cval``: optional + + +Implementation +~~~~~~~~~~~~~~~ + +Data smoothing is applied in the backend after the raw IDS data has been converted to a NumPy array and after the internal 2D data transformation step, when applicable. + +The current implementation uses SciPy-based smoothing routines: + +* Gaussian smoothing is applied with a Gaussian filter implementation. +* Savitzky-Golay smoothing is applied with a Savitzky-Golay filter implementation. + +Because smoothing modifies the returned numerical values, it should be treated as a visualization-oriented transformation and not as a lossless representation of the original dataset. + +Example usage +~~~~~~~~~~~~~~ + +The following examples demonstrate how smoothing can be enabled for testing purposes. + +Gaussian smoothing: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&smoothing_method=gaussian_filter&gaussian_smoothing_sigma=1.0' \ + -H 'accept: application/json' + +Savitzky-Golay smoothing: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&smoothing_method=savitzky_golay_filter&savgol_smoothing_window_length=5&savgol_smoothing_polyorder=2' \ + -H 'accept: application/json' From b9702f6c55f2590a7159c220646f2e4ca46e4e29 Mon Sep 17 00:00:00 2001 From: wasikj Date: Wed, 20 May 2026 14:14:41 +0200 Subject: [PATCH 15/28] Delete unused models --- .../schemas/response_info_schemas.py | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/backend/ibex/endpoints/schemas/response_info_schemas.py b/backend/ibex/endpoints/schemas/response_info_schemas.py index 5e79e614..c6bd4e02 100644 --- a/backend/ibex/endpoints/schemas/response_info_schemas.py +++ b/backend/ibex/endpoints/schemas/response_info_schemas.py @@ -25,41 +25,4 @@ class DownsamplingMethodsResponse(BaseModel): # ========== DATA MANIPULATION METHODS ========== - -class DataManipulationMethodParameterPossibleValuesModel(BaseModel): - """Intermediate model for /info/data_manipulation_methods endpoint""" - - value: str = Field(description="Possible value of parameter", examples=["linear", "nearest"]) - description: str = Field( - description="Value description", - examples=[ - "New point value will be interpolated using linear algorithm", - "New point value will be interpolated using nearest value", - ], - ) - - -class DataManipulationMethodParametersModel(BaseModel): - """Intermediate model for /info/data_manipulation_methods endpoint""" - - human_readable_name: str = Field( - description="Human readable name of the parameter to be displayed in FE", examples=["Sigma", "Deviation level"] - ) - name: str = Field(description="URL parameter name", examples=["sigma", "deviation_level"]) - description: str = Field(description="Parameter description", examples=["Standard deviation for Gaussian kernel."]) - possible_values: list[DataManipulationMethodParameterPossibleValuesModel] | None = Field( - default=None, description="Possible values for manipulation parameter" - ) - - -class DataManipulationMethodModel(BaseModel): - """Intermediate model for /info/data_manipulation_methods endpoint""" - - name: str = Field(description="Method name", examples=["interpolation", "smoothing"]) - description: str = Field(description="Method description", examples=["Gaussian smoothing"]) - method_parameters: list[DataManipulationMethodParametersModel] = Field( - description="Parameters used in data manipulation method" - ) - - DataManipulationMethodsResponse = data_manipulation_methods.DataManipulationMethodsResponse From 7ac0dad2301ae4e75f8d581ee5c7a19f4d7ec941 Mon Sep 17 00:00:00 2001 From: jwasikpsnc Date: Mon, 25 May 2026 12:27:46 +0200 Subject: [PATCH 16/28] Add more underlines in adding_new_data_manipulation_method.rst --- .../backend_development/adding_new_data_manipulation_method.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst b/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst index 775914ff..287aefde 100644 --- a/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst +++ b/docs/source/developers_manual/backend_development/adding_new_data_manipulation_method.rst @@ -2,7 +2,7 @@ Adding a new data manipulation method ------------------------ +-------------------------------------- Adding a new data manipulation operation requires updates in three places: the request model, the operation description registry, and the backend execution path. From 23683816622d050ebae4f5f24cd2443638e7db8e Mon Sep 17 00:00:00 2001 From: Olivier Hoenen Date: Fri, 29 May 2026 17:52:36 +0200 Subject: [PATCH 17/28] char encoding issue --- backend/ibex/core/data_manipulation_methods.py | 2 +- backend/tests/test_data_endpoints.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index ed755663..b35e330e 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -11,7 +11,7 @@ class InterpolationMethod(str, Enum): class SmoothingMethod(str, Enum): GAUSSIAN_FILTER = "gaussian_filter" - SAVITZKY_GOLAY_FILTER = "savitzky–golay_filter" + SAVITZKY_GOLAY_FILTER = "savitzky-golay_filter" class AdditionalParameter(BaseModel): diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index ecd08af1..106e8a60 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -85,7 +85,7 @@ def test_plot_data_with_gaussian_smoothing(entry_path): def test_plot_data_with_savgol_smoothing(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", - "smoothing_method": "savitzky–golay_filter", + "smoothing_method": "savitzky-golay_filter", "savgol_smoothing_window_length": 5, "savgol_smoothing_polyorder": 2, } @@ -165,7 +165,7 @@ def test_plot_data_requires_gaussian_sigma(entry_path): def test_plot_data_requires_savgol_window_length_and_polyorder(entry_path): base_parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/profiles_1d[:]/time", - "smoothing_method": "savitzky–golay_filter", + "smoothing_method": "savitzky-golay_filter", } response = pytest.test_client.get("/data/plot_data", params=base_parameters) From 03148ddd89ceed79c6aba35eac48062a069a9db4 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 1 Jun 2026 07:15:23 +0200 Subject: [PATCH 18/28] Set smoothing_method to None when interpolating --- backend/ibex/data_source/imas_python_source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index c7167e0c..da885a89 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -836,6 +836,7 @@ def convert_to_lists(data): new_plot_data_query = copy(plot_data_query) new_plot_data_query.uri = _uri new_plot_data_query.interpolate_over = None + new_plot_data_query.smoothing_method = None interpolate_to_coordinates = self.get_plot_data(new_plot_data_query)["data"]["coordinates"] if len(interpolate_to_coordinates) != len(coordinates_to_be_returned): From cc73e6a5d39a07d3197f09fce3cad17dc925e8a6 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 2 Jun 2026 07:35:59 +0200 Subject: [PATCH 19/28] Allow only time-based smoothing --- backend/ibex/data_source/imas_python_source.py | 7 +++++++ backend/ibex/endpoints/data.py | 1 + backend/tests/conftest.py | 3 +++ backend/tests/test_data_endpoints.py | 14 ++++++++++++-- .../backend_development/data_manipulation.rst | 2 ++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index da885a89..fa85793e 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -788,6 +788,13 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= BEGIN data smoothing ============ if plot_data_query.smoothing_method is not None: + if first_value.metadata.ndim != 1: + raise InvalidParametersException("Data smoothing is only supported for 1D data") + if not coordinates_to_be_returned or coordinates_to_be_returned[0]["name"] != "time": + raise InvalidParametersException( + "Data smoothing is only supported when the first coordinate is time" + ) + if plot_data_query.smoothing_method == SmoothingMethod.SAVITZKY_GOLAY_FILTER: data_to_be_returned = apply_savgol_filter( data_to_be_returned, diff --git a/backend/ibex/endpoints/data.py b/backend/ibex/endpoints/data.py index cbd19dc4..05930d35 100644 --- a/backend/ibex/endpoints/data.py +++ b/backend/ibex/endpoints/data.py @@ -70,6 +70,7 @@ def field_value( 200: {"description": "Plot data returned successfully"}, 404: {"description": "Data node not found"}, 464: {"description": "Given data node is empty"}, + 466: {"description": "Invalid parameters for requested data manipulation"}, }, description="Returns single (or tensorized) data node value with detailed parameters used to plot the data", ) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 1bedf148..bdfcaeab 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -82,6 +82,9 @@ def entry_path(tmp_path_factory): profiles_2d.grid.dim2 = np.array([0, 1, 2]) i += 10 + # ===== for data smoothing (must be time-based) ===== + core_profiles.global_quantities.ip = np.array([1.0, 2.0, 3.0, 4.0, 5.0]) + entry.put(core_profiles) entry.close() diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 106e8a60..030fd616 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -71,7 +71,7 @@ def test_plot_data(entry_path): def test_plot_data_with_gaussian_smoothing(entry_path): parameters = { - "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", + "uri": f"imas:hdf5?path={entry_path}#core_profiles/global_quantities/ip", "smoothing_method": "gaussian_filter", "gaussian_smoothing_sigma": 1, } @@ -84,7 +84,7 @@ def test_plot_data_with_gaussian_smoothing(entry_path): def test_plot_data_with_savgol_smoothing(entry_path): parameters = { - "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", + "uri": f"imas:hdf5?path={entry_path}#core_profiles/global_quantities/ip", "smoothing_method": "savitzky-golay_filter", "savgol_smoothing_window_length": 5, "savgol_smoothing_polyorder": 2, @@ -96,6 +96,16 @@ def test_plot_data_with_savgol_smoothing(entry_path): assert response_body["data"]["value"] == pytest.approx([0.99, 2.0, 3.0, 4.0, 5.0], 0.1) +def test_plot_data_smoothing_with_wrong_target_node(entry_path): + parameters = { + "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", # targetet quantity must be time-based + "smoothing_method": "gaussian_filter", + "gaussian_smoothing_sigma": 1, + } + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 466 + + def test_plot_data_2d(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/profiles_2d[:]/ion[:]/temperature", diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index 53c172c2..dc08be09 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -20,6 +20,8 @@ Configuration ~~~~~~~~~~~~~~ Data smoothing is configured through the ``smoothing_method`` parameter of the ``/data/plot_data/`` endpoint. +It is only accepted for nodes whose first coordinate is ``time``. +If a different first coordinate is used, the backend rejects the request with an invalid-parameters error. At the moment, the backend supports the following smoothing methods: From 59008f7babfc98a23eb535848c4abb361ecaa157 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 2 Jun 2026 13:55:51 +0200 Subject: [PATCH 20/28] Add simple data operations --- .../ibex/core/data_manipulation_methods.py | 46 ++++++++++++++++ .../ibex/data_source/imas_python_source.py | 7 +++ .../data_source/imas_python_source_utils.py | 38 +++++++++++++ .../endpoints/schemas/request_data_schemas.py | 37 ++++++++++++- backend/tests/test_data_endpoints.py | 29 ++++++++++ backend/tests/test_data_manipulation.py | 43 +++++++++++++++ docs/source/conf.py | 4 +- .../backend_development/data_manipulation.rst | 53 +++++++++++++++++++ 8 files changed, 254 insertions(+), 3 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index b35e330e..65d42b19 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -170,3 +170,49 @@ class DataManipulationMethodsResponse(BaseModel): data_smoothing_description.method_parameters.append(data_smoothing_method_parameter) available_methods.data_manipulation_methods.append(data_smoothing_description) + +# ====================== SIMPLE DATA OPERATIONS ====================== + +simple_data_operations_description = DataManipulationOperation( + name="Simple Data Operations", + description="Sequence of scalar operations applied to the dataset in this order: addition, multiplication, division, exponentiation, and root.", + method_parameters=[], +) + +data_addition_scalar_parameter = DataManipulationParameter( + human_readable_name="Addition", + name="addition_addend", + description="Scalar value added to every data point. Executed first.", +) + +data_multiplication_scalar_parameter = DataManipulationParameter( + human_readable_name="Multiplication", + name="multiplication_factor", + description="Scalar value used to multiply every data point. Executed after addition.", +) + +data_division_scalar_parameter = DataManipulationParameter( + human_readable_name="Division", + name="division_divisor", + description="Scalar value used as the divisor for every data point. Executed after multiplication.", +) + +data_exponentiation_exponent_parameter = DataManipulationParameter( + human_readable_name="Exponentiation", + name="exponentiation_exponent", + description="Scalar exponent used to raise the input data to a power. Executed after division.", +) + +data_root_degree_parameter = DataManipulationParameter( + human_readable_name="Root", + name="root_degree", + description="Scalar degree used to compute the nth root of the input data. Executed last.", +) + +simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) +simple_data_operations_description.method_parameters.append(data_root_degree_parameter) + +available_methods.data_manipulation_methods.append(simple_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index fa85793e..2696e856 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -49,6 +49,7 @@ calculate_coordinate_shapes, apply_savgol_filter, apply_gaussian_filter, + apply_simple_operations, ) from ibex.core.data_manipulation_methods import SmoothingMethod, InterpolationMethod from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel @@ -785,6 +786,12 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # FE expects data's first dimension to be connected with second dimension, thus this transformation data_to_be_returned = transform_2D_data(data_to_be_returned) + # ============= BEGIN simple operations ============ + + data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) + + # ============= END simple operations ============= + # ============= BEGIN data smoothing ============ if plot_data_query.smoothing_method is not None: diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 795e88a0..12ce1f5f 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -1,4 +1,5 @@ from functools import reduce +from typing import Any import numpy as np from imas.ids_primitive import IDSNumericArray @@ -65,6 +66,43 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): raise InvalidParametersException(msg) +def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): + """ + Apply simple scalar operations to data. + Operations are applied in the following order: + addition, multiplication, division, exponentiation, root. + """ + + if isinstance(data, list): + return [apply_simple_operations(x, plot_data_query) for x in data] + elif isinstance(data, (np.ndarray, IDSNumericArray)): + result = data + + if plot_data_query.addition_addend is not None: + result = result + plot_data_query.addition_addend + + if plot_data_query.multiplication_factor is not None: + result = result * plot_data_query.multiplication_factor + + if plot_data_query.division_divisor is not None: + if plot_data_query.division_divisor == 0: + raise InvalidParametersException("division_divisor cannot be 0") + result = result / plot_data_query.division_divisor + + if plot_data_query.exponentiation_exponent is not None: + result = np.power(result, plot_data_query.exponentiation_exponent) + + if plot_data_query.root_degree is not None: + if plot_data_query.root_degree == 0: + raise InvalidParametersException("root_degree cannot be 0") + result = np.power(result, 1 / plot_data_query.root_degree) + + return result + else: + msg = "Simple operations can be executed only on numeric arrays, not single values or strings." + raise InvalidParametersException(msg) + + def union_arrays(data: list): return reduce(np.union1d, data) diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index f4b93643..43991f8b 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -81,6 +81,29 @@ class GaussianSmoothingParameters(BaseModel): ) +class SimpleOperationsParameters(BaseModel): + addition_addend: float | None = Field( + default=None, + description="Scalar value added to every data point.", + ) + multiplication_factor: float | None = Field( + default=None, + description="Scalar value used to multiply every data point.", + ) + division_divisor: float | None = Field( + default=None, + description="Scalar value used as the divisor for every data point.", + ) + exponentiation_exponent: float | None = Field( + default=None, + description="Scalar exponent used to raise the input data to a power.", + ) + root_degree: float | None = Field( + default=None, + description="Scalar degree used to compute the nth root of the input data.", + ) + + class PlotDataBasicParameters(BaseModel): """...""" @@ -94,7 +117,12 @@ class PlotDataBasicParameters(BaseModel): smoothing_method: SmoothingMethod | None = Field(default=None, description="Smoothing method to be used") -class PlotDataRequestModel(PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters): +class PlotDataRequestModel( + PlotDataBasicParameters, + SavgolSmoothingParameters, + GaussianSmoothingParameters, + SimpleOperationsParameters, +): @model_validator(mode="after") def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": if self.smoothing_method == SmoothingMethod.GAUSSIAN_FILTER and self.gaussian_smoothing_sigma is None: @@ -112,3 +140,10 @@ def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": ) return self + + @model_validator(mode="after") + def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": + if self.division_divisor == 0: + raise ValueError("division_divisor cannot be 0") + + return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 030fd616..463cc66f 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -96,6 +96,35 @@ def test_plot_data_with_savgol_smoothing(entry_path): assert response_body["data"]["value"] == pytest.approx([0.99, 2.0, 3.0, 4.0, 5.0], 0.1) +def test_plot_data_with_simple_operations(entry_path): + cases = [ + ( + {"addition_addend": 2, "multiplication_factor": 3}, + [9.0, 12.0, 15.0, 18.0, 21.0], + ), + ( + {"division_divisor": 2}, + [0.5, 1.0, 1.5, 2.0, 2.5], + ), + ( + {"exponentiation_exponent": 2}, + [1.0, 4.0, 9.0, 16.0, 25.0], + ), + ( + {"root_degree": 2}, + [1.0, 1.41421356237, 1.73205080757, 2.0, 2.2360679775], + ), + ] + + for params, expected in cases: + parameters = {"uri": f"imas:hdf5?path={entry_path}#core_profiles/time", **params} + response = pytest.test_client.get("/data/plot_data", params=parameters) + assert response.status_code == 200 + + response_body = response.json() + assert response_body["data"]["value"] == pytest.approx(expected) + + def test_plot_data_smoothing_with_wrong_target_node(entry_path): parameters = { "uri": f"imas:hdf5?path={entry_path}#core_profiles/time", # targetet quantity must be time-based diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index c987bc9f..0eda2d69 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,9 +1,13 @@ import numpy as np import pytest +from types import SimpleNamespace +from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, + apply_simple_operations, ) +from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel def test_apply_gaussian_smoothing(): @@ -47,3 +51,42 @@ def test_apply_savitzky_golay_smoothing(): ), 0.1, ) + + +@pytest.mark.parametrize( + ("request_kwargs", "data", "expected"), + [ + ({"addition_addend": 2}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), + ({"multiplication_factor": 3}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), + ({"division_divisor": 2}, np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), + ({"exponentiation_exponent": 2}, np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), + ({"root_degree": 2}, np.array([1.0, 4.0, 9.0]), np.array([1.0, 2.0, 3.0])), + ], +) +def test_apply_simple_operations(request_kwargs, data, expected): + request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", **request_kwargs) + + assert np.asarray(expected) == pytest.approx(apply_simple_operations(data, request)) + + +def test_apply_simple_operations_recurses_over_lists(): + request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", addition_addend=1) + data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] + + result = apply_simple_operations(data, request) + + assert np.asarray(result[0]) == pytest.approx([2.0, 3.0]) + assert np.asarray(result[1]) == pytest.approx([4.0, 5.0]) + + +def test_apply_simple_operations_rejects_division_by_zero(): + request = SimpleNamespace( + division_divisor=0, + addition_addend=None, + multiplication_factor=None, + exponentiation_exponent=None, + root_degree=None, + ) + + with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + apply_simple_operations(np.array([1.0, 2.0]), request) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3860d5a5..ed8bbfe2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -294,8 +294,8 @@ # Configuration of sphinx.ext.mathjax # https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.mathjax -autodoc_pydantic_model_show_json = True -autodoc_pydantic_model_show_config_summary = True +autodoc_pydantic_model_show_json = False +autodoc_pydantic_model_show_config_summary = False def escape_underscores(string): diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index dc08be09..08385bad 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -9,6 +9,14 @@ Introduction The IBEX backend provides a range of data manipulation techniques that directly affect the shape and appearance of the resulting plots. These operations are applied as part of the ``/data/plot_data/`` request flow and allow the backend to transform datasets before they are returned to the frontend. +The backend applies the manipulation stages in this order: + +1. simple data operations +2. data smoothing +3. data interpolation +4. downsampling + +This means later stages operate on the output of earlier ones when the corresponding request parameters are enabled. Data smoothing --------------- @@ -85,3 +93,48 @@ Savitzky-Golay smoothing: curl -X 'GET' \ '/data/plot_data?uri=&smoothing_method=savitzky_golay_filter&savgol_smoothing_window_length=5&savgol_smoothing_polyorder=2' \ -H 'accept: application/json' + + +Simple scalar operations +------------------------ + +IBEX also supports a sequence of scalar operations that can be applied to the returned dataset: + +* addition +* multiplication +* division +* exponentiation +* root + +These operations are executed in that order. In practice, the backend applies them sequentially to the numerical data before any smoothing or resampling step. + +The corresponding request parameters are: + +* ``addition_addend`` +* ``multiplication_factor`` +* ``division_divisor`` +* ``exponentiation_exponent`` +* ``root_degree`` + +Division by zero is rejected by the backend. + +Example usage +~~~~~~~~~~~~~~ + +The following examples demonstrate how simple data operations can be enabled for testing purposes. + +Single operation: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&addition_addend=2' \ + -H 'accept: application/json' + +Two operations: + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&addition_addend=2&multiplication_factor=3' \ + -H 'accept: application/json' From c86a6f23fc3e51cef4722249f7593ab5f38c6e15 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 10:23:09 +0200 Subject: [PATCH 21/28] Add subtraction. Check parameters before operation. --- backend/ibex/core/data_manipulation_methods.py | 11 +++++++++-- backend/ibex/data_source/imas_python_source.py | 12 +++++++++++- backend/ibex/data_source/imas_python_source_utils.py | 5 ++++- .../ibex/endpoints/schemas/request_data_schemas.py | 4 ++++ backend/tests/test_data_manipulation.py | 2 ++ .../backend_development/data_manipulation.rst | 2 ++ 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 65d42b19..fc633eb4 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -175,7 +175,7 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description = DataManipulationOperation( name="Simple Data Operations", - description="Sequence of scalar operations applied to the dataset in this order: addition, multiplication, division, exponentiation, and root.", + description="Sequence of scalar operations applied to the dataset in this order: addition, subtraction, multiplication, division, exponentiation, and root.", method_parameters=[], ) @@ -185,10 +185,16 @@ class DataManipulationMethodsResponse(BaseModel): description="Scalar value added to every data point. Executed first.", ) +data_subtraction_subtrahend_parameter = DataManipulationParameter( + human_readable_name="Subtraction", + name="subtraction_subtrahend", + description="Scalar value subtracted from every data point. Executed after addition.", +) + data_multiplication_scalar_parameter = DataManipulationParameter( human_readable_name="Multiplication", name="multiplication_factor", - description="Scalar value used to multiply every data point. Executed after addition.", + description="Scalar value used to multiply every data point. Executed after subtraction.", ) data_division_scalar_parameter = DataManipulationParameter( @@ -210,6 +216,7 @@ class DataManipulationMethodsResponse(BaseModel): ) simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_subtraction_subtrahend_parameter) simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 2696e856..1f185c13 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -788,7 +788,17 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= BEGIN simple operations ============ - data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) + if any( + [ + plot_data_query.addition_addend is not None, + plot_data_query.subtraction_subtrahend is not None, + plot_data_query.multiplication_factor is not None, + plot_data_query.division_divisor is not None, + plot_data_query.exponentiation_exponent is not None, + plot_data_query.root_degree is not None, + ] + ): + data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) # ============= END simple operations ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 12ce1f5f..800bb2c4 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -70,7 +70,7 @@ def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): """ Apply simple scalar operations to data. Operations are applied in the following order: - addition, multiplication, division, exponentiation, root. + addition, subtraction, multiplication, division, exponentiation, root. """ if isinstance(data, list): @@ -81,6 +81,9 @@ def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): if plot_data_query.addition_addend is not None: result = result + plot_data_query.addition_addend + if plot_data_query.subtraction_subtrahend is not None: + result = result - plot_data_query.subtraction_subtrahend + if plot_data_query.multiplication_factor is not None: result = result * plot_data_query.multiplication_factor diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 43991f8b..23c6386f 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -86,6 +86,10 @@ class SimpleOperationsParameters(BaseModel): default=None, description="Scalar value added to every data point.", ) + subtraction_subtrahend: float | None = Field( + default=None, + description="Scalar value subtracted from every data point.", + ) multiplication_factor: float | None = Field( default=None, description="Scalar value used to multiply every data point.", diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 0eda2d69..0dcec80b 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -57,6 +57,7 @@ def test_apply_savitzky_golay_smoothing(): ("request_kwargs", "data", "expected"), [ ({"addition_addend": 2}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), + ({"subtraction_subtrahend": 1}, np.array([3.0, 4.0, 5.0]), np.array([2.0, 3.0, 4.0])), ({"multiplication_factor": 3}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), ({"division_divisor": 2}, np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), ({"exponentiation_exponent": 2}, np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), @@ -83,6 +84,7 @@ def test_apply_simple_operations_rejects_division_by_zero(): request = SimpleNamespace( division_divisor=0, addition_addend=None, + subtraction_subtrahend=None, multiplication_factor=None, exponentiation_exponent=None, root_degree=None, diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index 08385bad..e0e02599 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -101,6 +101,7 @@ Simple scalar operations IBEX also supports a sequence of scalar operations that can be applied to the returned dataset: * addition +* subtraction * multiplication * division * exponentiation @@ -111,6 +112,7 @@ These operations are executed in that order. In practice, the backend applies th The corresponding request parameters are: * ``addition_addend`` +* ``subtraction_subtrahend`` * ``multiplication_factor`` * ``division_divisor`` * ``exponentiation_exponent`` From e67f0f1a9581f0a1fe88cc4fbc59e783aa44b370 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 12:34:08 +0200 Subject: [PATCH 22/28] Add operation priorities --- .../ibex/core/data_manipulation_methods.py | 82 +++++++++++++++++-- .../data_source/imas_python_source_utils.py | 77 ++++++++++++----- .../endpoints/schemas/request_data_schemas.py | 36 ++++++++ backend/tests/test_data_endpoints.py | 5 ++ backend/tests/test_data_manipulation.py | 25 ++++-- 5 files changed, 187 insertions(+), 38 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index fc633eb4..730befc5 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -43,6 +43,8 @@ class DataManipulationParameter(BaseModel): human_readable_name: str name: str description: str + type: str + default: Optional[str] = None possible_values: Optional[list[PossibleValue]] = None @@ -76,12 +78,15 @@ class DataManipulationMethodsResponse(BaseModel): human_readable_name="Interpolate over", name="interpolate_over", description="List of URIs to gather coordinates from, for interpolation", + type="list[string]", ) data_interpolation_method_parameter = DataManipulationParameter( human_readable_name="Interpolation method", name="interpolation_method", description="List of URIs to gather coordinates from, for interpolation", + type="string", + default=InterpolationMethod.EXACT_VALUE, possible_values=[ PossibleValue( value=InterpolationMethod.EXACT_VALUE, @@ -116,6 +121,7 @@ class DataManipulationMethodsResponse(BaseModel): human_readable_name="Smoothing method", name="smoothing_method", description="Method to be used in data smoothing process", + type="string", possible_values=[ PossibleValue( value=SmoothingMethod.GAUSSIAN_FILTER, @@ -175,51 +181,113 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description = DataManipulationOperation( name="Simple Data Operations", - description="Sequence of scalar operations applied to the dataset in this order: addition, subtraction, multiplication, division, exponentiation, and root.", + description="Sequence of scalar operations applied to the dataset. " + "Execution order is determined by the *_priority parameters. " + "Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6.", method_parameters=[], ) data_addition_scalar_parameter = DataManipulationParameter( human_readable_name="Addition", name="addition_addend", - description="Scalar value added to every data point. Executed first.", + description="Scalar value added to every data point.", + type="number", ) data_subtraction_subtrahend_parameter = DataManipulationParameter( human_readable_name="Subtraction", name="subtraction_subtrahend", - description="Scalar value subtracted from every data point. Executed after addition.", + description="Scalar value subtracted from every data point.", + type="number", ) data_multiplication_scalar_parameter = DataManipulationParameter( human_readable_name="Multiplication", name="multiplication_factor", - description="Scalar value used to multiply every data point. Executed after subtraction.", + description="Scalar value used to multiply every data point.", + type="number", ) data_division_scalar_parameter = DataManipulationParameter( human_readable_name="Division", name="division_divisor", - description="Scalar value used as the divisor for every data point. Executed after multiplication.", + description="Scalar value used as the divisor for every data point.", + type="number", ) data_exponentiation_exponent_parameter = DataManipulationParameter( human_readable_name="Exponentiation", name="exponentiation_exponent", - description="Scalar exponent used to raise the input data to a power. Executed after division.", + description="Scalar exponent used to raise the input data to a power.", + type="number", ) data_root_degree_parameter = DataManipulationParameter( human_readable_name="Root", name="root_degree", - description="Scalar degree used to compute the nth root of the input data. Executed last.", + description="Scalar degree used to compute the nth root of the input data.", + type="number", +) + +data_addition_priority_parameter = DataManipulationParameter( + human_readable_name="Addition priority", + name="addition_priority", + description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", + type="int", + default="1", +) + +data_subtraction_priority_parameter = DataManipulationParameter( + human_readable_name="Subtraction priority", + name="subtraction_priority", + description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", + type="int", + default="2", +) + +data_multiplication_priority_parameter = DataManipulationParameter( + human_readable_name="Multiplication priority", + name="multiplication_priority", + description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", + type="int", + default="3", +) + +data_division_priority_parameter = DataManipulationParameter( + human_readable_name="Division priority", + name="division_priority", + description="Execution order priority for division. Lower value = earlier execution. Default: 4.", + type="int", + default="4", +) + +data_exponentiation_priority_parameter = DataManipulationParameter( + human_readable_name="Exponentiation priority", + name="exponentiation_priority", + description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", + type="int", + default="5", +) + +data_root_priority_parameter = DataManipulationParameter( + human_readable_name="Root priority", + name="root_priority", + description="Execution order priority for root. Lower value = earlier execution. Default: 6.", + type="int", + default="6", ) simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_addition_priority_parameter) simple_data_operations_description.method_parameters.append(data_subtraction_subtrahend_parameter) +simple_data_operations_description.method_parameters.append(data_subtraction_priority_parameter) simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_multiplication_priority_parameter) simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) +simple_data_operations_description.method_parameters.append(data_division_priority_parameter) simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) +simple_data_operations_description.method_parameters.append(data_exponentiation_priority_parameter) simple_data_operations_description.method_parameters.append(data_root_degree_parameter) +simple_data_operations_description.method_parameters.append(data_root_priority_parameter) available_methods.data_manipulation_methods.append(simple_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 800bb2c4..3be95215 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -66,39 +66,72 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): raise InvalidParametersException(msg) +def _safe_division(data, divisor): + if divisor == 0: + raise InvalidParametersException("division_divisor cannot be 0") + return data / divisor + + def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): """ Apply simple scalar operations to data. - Operations are applied in the following order: - addition, subtraction, multiplication, division, exponentiation, root. + Execution order is determined by the *_priority parameters from the request. + Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6. """ + _SIMPLE_OPERATION_DEFS = [ + { + "value_field": "addition_addend", + "priority_field": "addition_priority", + "default_priority": 1, + "func": lambda r, v: r + v, + }, + { + "value_field": "subtraction_subtrahend", + "priority_field": "subtraction_priority", + "default_priority": 2, + "func": lambda r, v: r - v, + }, + { + "value_field": "multiplication_factor", + "priority_field": "multiplication_priority", + "default_priority": 3, + "func": lambda r, v: r * v, + }, + { + "value_field": "division_divisor", + "priority_field": "division_priority", + "default_priority": 4, + "func": lambda r, v: _safe_division(r, v), + }, + { + "value_field": "exponentiation_exponent", + "priority_field": "exponentiation_priority", + "default_priority": 5, + "func": lambda r, v: np.power(r, v), + }, + { + "value_field": "root_degree", + "priority_field": "root_priority", + "default_priority": 6, + "func": lambda r, v: np.power(r, 1 / v), + }, + ] + if isinstance(data, list): return [apply_simple_operations(x, plot_data_query) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): result = data - if plot_data_query.addition_addend is not None: - result = result + plot_data_query.addition_addend - - if plot_data_query.subtraction_subtrahend is not None: - result = result - plot_data_query.subtraction_subtrahend - - if plot_data_query.multiplication_factor is not None: - result = result * plot_data_query.multiplication_factor - - if plot_data_query.division_divisor is not None: - if plot_data_query.division_divisor == 0: - raise InvalidParametersException("division_divisor cannot be 0") - result = result / plot_data_query.division_divisor - - if plot_data_query.exponentiation_exponent is not None: - result = np.power(result, plot_data_query.exponentiation_exponent) + operations = [] + for op in _SIMPLE_OPERATION_DEFS: + value = getattr(plot_data_query, op["value_field"]) + if value is not None: + priority = getattr(plot_data_query, op["priority_field"]) or op["default_priority"] + operations.append({"priority": priority, "func": op["func"], "value": value}) - if plot_data_query.root_degree is not None: - if plot_data_query.root_degree == 0: - raise InvalidParametersException("root_degree cannot be 0") - result = np.power(result, 1 / plot_data_query.root_degree) + for op in sorted(operations, key=lambda x: x["priority"]): + result = op["func"](result, op["value"]) return result else: diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 23c6386f..84ee0e63 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -86,26 +86,50 @@ class SimpleOperationsParameters(BaseModel): default=None, description="Scalar value added to every data point.", ) + addition_priority: int | None = Field( + default=None, + description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", + ) subtraction_subtrahend: float | None = Field( default=None, description="Scalar value subtracted from every data point.", ) + subtraction_priority: int | None = Field( + default=None, + description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", + ) multiplication_factor: float | None = Field( default=None, description="Scalar value used to multiply every data point.", ) + multiplication_priority: int | None = Field( + default=None, + description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", + ) division_divisor: float | None = Field( default=None, description="Scalar value used as the divisor for every data point.", ) + division_priority: int | None = Field( + default=None, + description="Execution order priority for division. Lower value = earlier execution. Default: 4.", + ) exponentiation_exponent: float | None = Field( default=None, description="Scalar exponent used to raise the input data to a power.", ) + exponentiation_priority: int | None = Field( + default=None, + description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", + ) root_degree: float | None = Field( default=None, description="Scalar degree used to compute the nth root of the input data.", ) + root_priority: int | None = Field( + default=None, + description="Execution order priority for root. Lower value = earlier execution. Default: 6.", + ) class PlotDataBasicParameters(BaseModel): @@ -150,4 +174,16 @@ def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": if self.division_divisor == 0: raise ValueError("division_divisor cannot be 0") + priorities = [ + self.addition_priority, + self.subtraction_priority, + self.multiplication_priority, + self.division_priority, + self.exponentiation_priority, + self.root_priority, + ] + defined_priorities = [p for p in priorities if p is not None] + if len(defined_priorities) != len(set(defined_priorities)): + raise ValueError("operation priorities must be unique") + return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index 463cc66f..e390f92a 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -97,11 +97,16 @@ def test_plot_data_with_savgol_smoothing(entry_path): def test_plot_data_with_simple_operations(entry_path): + # core_profiles.time = [1,2,3,4,5] (float) cases = [ ( {"addition_addend": 2, "multiplication_factor": 3}, [9.0, 12.0, 15.0, 18.0, 21.0], ), + ( + {"addition_addend": 2, "multiplication_factor": 3, "addition_priority": 6, "multiplication_priority": 1}, + [5.0, 8.0, 11.0, 14.0, 17.0], + ), ( {"division_divisor": 2}, [0.5, 1.0, 1.5, 2.0, 2.5], diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 0dcec80b..dbe94c18 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from types import SimpleNamespace from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, @@ -81,14 +80,22 @@ def test_apply_simple_operations_recurses_over_lists(): def test_apply_simple_operations_rejects_division_by_zero(): - request = SimpleNamespace( - division_divisor=0, - addition_addend=None, - subtraction_subtrahend=None, - multiplication_factor=None, - exponentiation_exponent=None, - root_degree=None, - ) + request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): apply_simple_operations(np.array([1.0, 2.0]), request) + + +def test_apply_simple_operations_uses_priority_order(): + request = PlotDataRequestModel( + uri="imas:hdf5?path=/dummy#dummy", + addition_addend=1, + multiplication_factor=2, + addition_priority=2, + multiplication_priority=1, + ) + data = np.array([5.0]) + # default order: add then multiply -> (5+1)*2 = 12 + # priority order: multiply then add -> (5*2)+1 = 11 + result = apply_simple_operations(data, request) + assert result == pytest.approx([11.0]) From fc9c0b33095c7638f48966b2b1600cf70eb24df4 Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 12:49:38 +0200 Subject: [PATCH 23/28] Fix broken test --- backend/tests/test_data_manipulation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index dbe94c18..163dc649 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -7,6 +7,7 @@ apply_simple_operations, ) from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel +from pydantic_core._pydantic_core import ValidationError def test_apply_gaussian_smoothing(): @@ -80,10 +81,8 @@ def test_apply_simple_operations_recurses_over_lists(): def test_apply_simple_operations_rejects_division_by_zero(): - request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) - - with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): - apply_simple_operations(np.array([1.0, 2.0]), request) + with pytest.raises(ValidationError, match="division_divisor cannot be 0"): + PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) def test_apply_simple_operations_uses_priority_order(): From 1c7e665ac9828491049e5b4dd811f467e2f477bf Mon Sep 17 00:00:00 2001 From: wasikj Date: Mon, 15 Jun 2026 12:59:02 +0200 Subject: [PATCH 24/28] Apply linter --- backend/tests/test_data_manipulation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 163dc649..5ee91c1c 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,6 +1,5 @@ import numpy as np import pytest -from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, From a105b666d5f0cabad3924eaffcc52bb9e777d9d9 Mon Sep 17 00:00:00 2001 From: wasikj Date: Tue, 16 Jun 2026 13:51:30 +0200 Subject: [PATCH 25/28] Basic operations order (WIP) --- .../ibex/core/data_manipulation_methods.py | 138 +++++------------- .../ibex/data_source/imas_python_source.py | 13 +- .../data_source/imas_python_source_utils.py | 78 +++------- .../endpoints/schemas/request_data_schemas.py | 75 +--------- backend/tests/test_data_endpoints.py | 10 +- backend/tests/test_data_manipulation.py | 49 +++---- 6 files changed, 85 insertions(+), 278 deletions(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index 730befc5..d768259f 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -46,6 +46,8 @@ class DataManipulationParameter(BaseModel): type: str default: Optional[str] = None possible_values: Optional[list[PossibleValue]] = None + group_label: Optional[str] = None + fields: Optional[list["DataManipulationParameter"]] = None class DataManipulationOperation(BaseModel): @@ -62,6 +64,7 @@ class DataManipulationMethodsResponse(BaseModel): data_manipulation_methods: list[DataManipulationOperation] +DataManipulationParameter.model_rebuild() available_methods = DataManipulationMethodsResponse(data_manipulation_methods=[]) # ====================== DATA INTERPOLATION ====================== @@ -181,113 +184,40 @@ class DataManipulationMethodsResponse(BaseModel): simple_data_operations_description = DataManipulationOperation( name="Simple Data Operations", - description="Sequence of scalar operations applied to the dataset. " - "Execution order is determined by the *_priority parameters. " - "Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6.", + description="Ordered list of scalar operations applied to the dataset. " + "Execution order is determined by the order of parameters in the request.", method_parameters=[], ) -data_addition_scalar_parameter = DataManipulationParameter( - human_readable_name="Addition", - name="addition_addend", - description="Scalar value added to every data point.", - type="number", -) - -data_subtraction_subtrahend_parameter = DataManipulationParameter( - human_readable_name="Subtraction", - name="subtraction_subtrahend", - description="Scalar value subtracted from every data point.", - type="number", -) - -data_multiplication_scalar_parameter = DataManipulationParameter( - human_readable_name="Multiplication", - name="multiplication_factor", - description="Scalar value used to multiply every data point.", - type="number", -) - -data_division_scalar_parameter = DataManipulationParameter( - human_readable_name="Division", - name="division_divisor", - description="Scalar value used as the divisor for every data point.", - type="number", -) - -data_exponentiation_exponent_parameter = DataManipulationParameter( - human_readable_name="Exponentiation", - name="exponentiation_exponent", - description="Scalar exponent used to raise the input data to a power.", - type="number", -) - -data_root_degree_parameter = DataManipulationParameter( - human_readable_name="Root", - name="root_degree", - description="Scalar degree used to compute the nth root of the input data.", - type="number", -) - -data_addition_priority_parameter = DataManipulationParameter( - human_readable_name="Addition priority", - name="addition_priority", - description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", - type="int", - default="1", -) - -data_subtraction_priority_parameter = DataManipulationParameter( - human_readable_name="Subtraction priority", - name="subtraction_priority", - description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", - type="int", - default="2", -) - -data_multiplication_priority_parameter = DataManipulationParameter( - human_readable_name="Multiplication priority", - name="multiplication_priority", - description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", - type="int", - default="3", -) - -data_division_priority_parameter = DataManipulationParameter( - human_readable_name="Division priority", - name="division_priority", - description="Execution order priority for division. Lower value = earlier execution. Default: 4.", - type="int", - default="4", -) - -data_exponentiation_priority_parameter = DataManipulationParameter( - human_readable_name="Exponentiation priority", - name="exponentiation_priority", - description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", - type="int", - default="5", -) - -data_root_priority_parameter = DataManipulationParameter( - human_readable_name="Root priority", - name="root_priority", - description="Execution order priority for root. Lower value = earlier execution. Default: 6.", - type="int", - default="6", +data_operations_parameter = DataManipulationParameter( + human_readable_name="Operations", + name="operations", + description="Ordered list of scalar operations applied to every data point.", + type="list[object]", + group_label="Operation", + fields=[ + DataManipulationParameter( + human_readable_name="Type", + name="operation_type", + description="Type of operation", + type="string", + possible_values=[ + PossibleValue(value="add", description="Addition"), + PossibleValue(value="sub", description="Subtraction"), + PossibleValue(value="mul", description="Multiplication"), + PossibleValue(value="div", description="Division"), + PossibleValue(value="pow", description="Exponentiation"), + PossibleValue(value="root", description="Nth root"), + ], + ), + DataManipulationParameter( + human_readable_name="Value", + name="operation_value", + description="Scalar value for the operation", + type="number", + ), + ], ) -simple_data_operations_description.method_parameters.append(data_addition_scalar_parameter) -simple_data_operations_description.method_parameters.append(data_addition_priority_parameter) -simple_data_operations_description.method_parameters.append(data_subtraction_subtrahend_parameter) -simple_data_operations_description.method_parameters.append(data_subtraction_priority_parameter) -simple_data_operations_description.method_parameters.append(data_multiplication_scalar_parameter) -simple_data_operations_description.method_parameters.append(data_multiplication_priority_parameter) -simple_data_operations_description.method_parameters.append(data_division_scalar_parameter) -simple_data_operations_description.method_parameters.append(data_division_priority_parameter) -simple_data_operations_description.method_parameters.append(data_exponentiation_exponent_parameter) -simple_data_operations_description.method_parameters.append(data_exponentiation_priority_parameter) -simple_data_operations_description.method_parameters.append(data_root_degree_parameter) -simple_data_operations_description.method_parameters.append(data_root_priority_parameter) - +simple_data_operations_description.method_parameters.append(data_operations_parameter) available_methods.data_manipulation_methods.append(simple_data_operations_description) diff --git a/backend/ibex/data_source/imas_python_source.py b/backend/ibex/data_source/imas_python_source.py index 1f185c13..6a77c6ad 100644 --- a/backend/ibex/data_source/imas_python_source.py +++ b/backend/ibex/data_source/imas_python_source.py @@ -788,17 +788,8 @@ def get_plot_data(self, plot_data_query: PlotDataRequestModel) -> dict: # ============= BEGIN simple operations ============ - if any( - [ - plot_data_query.addition_addend is not None, - plot_data_query.subtraction_subtrahend is not None, - plot_data_query.multiplication_factor is not None, - plot_data_query.division_divisor is not None, - plot_data_query.exponentiation_exponent is not None, - plot_data_query.root_degree is not None, - ] - ): - data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query) + if plot_data_query.operations is not None: + data_to_be_returned = apply_simple_operations(data_to_be_returned, plot_data_query.operations) # ============= END simple operations ============= diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 3be95215..e24b37a0 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -1,5 +1,4 @@ from functools import reduce -from typing import Any import numpy as np from imas.ids_primitive import IDSNumericArray @@ -72,67 +71,32 @@ def _safe_division(data, divisor): return data / divisor -def apply_simple_operations(data: list | np.ndarray, plot_data_query: Any): - """ - Apply simple scalar operations to data. - Execution order is determined by the *_priority parameters from the request. - Defaults: addition=1, subtraction=2, multiplication=3, division=4, exponentiation=5, root=6. - """ +_OP_FUNCS = { + "add": lambda r, v: r + v, + "sub": lambda r, v: r - v, + "mul": lambda r, v: r * v, + "div": lambda r, v: _safe_division(r, v), + "pow": lambda r, v: np.power(r, v), + "root": lambda r, v: np.power(r, 1 / v), +} - _SIMPLE_OPERATION_DEFS = [ - { - "value_field": "addition_addend", - "priority_field": "addition_priority", - "default_priority": 1, - "func": lambda r, v: r + v, - }, - { - "value_field": "subtraction_subtrahend", - "priority_field": "subtraction_priority", - "default_priority": 2, - "func": lambda r, v: r - v, - }, - { - "value_field": "multiplication_factor", - "priority_field": "multiplication_priority", - "default_priority": 3, - "func": lambda r, v: r * v, - }, - { - "value_field": "division_divisor", - "priority_field": "division_priority", - "default_priority": 4, - "func": lambda r, v: _safe_division(r, v), - }, - { - "value_field": "exponentiation_exponent", - "priority_field": "exponentiation_priority", - "default_priority": 5, - "func": lambda r, v: np.power(r, v), - }, - { - "value_field": "root_degree", - "priority_field": "root_priority", - "default_priority": 6, - "func": lambda r, v: np.power(r, 1 / v), - }, - ] +def apply_simple_operations(data: list | np.ndarray, operations: list[str]): + """ + Apply simple scalar operations to data in the order given. + Each operation is a string in the format 'type:value', e.g. 'add:10', 'mul:5'. + """ if isinstance(data, list): - return [apply_simple_operations(x, plot_data_query) for x in data] + return [apply_simple_operations(x, operations) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): result = data - - operations = [] - for op in _SIMPLE_OPERATION_DEFS: - value = getattr(plot_data_query, op["value_field"]) - if value is not None: - priority = getattr(plot_data_query, op["priority_field"]) or op["default_priority"] - operations.append({"priority": priority, "func": op["func"], "value": value}) - - for op in sorted(operations, key=lambda x: x["priority"]): - result = op["func"](result, op["value"]) - + for op_str in operations: + op_type, value_str = op_str.split(":", 1) + value = float(value_str) + func = _OP_FUNCS.get(op_type) + if func is None: + raise InvalidParametersException(f"Unknown operation type: {op_type}") + result = func(result, value) return result else: msg = "Simple operations can be executed only on numeric arrays, not single values or strings." diff --git a/backend/ibex/endpoints/schemas/request_data_schemas.py b/backend/ibex/endpoints/schemas/request_data_schemas.py index 84ee0e63..4c0bc2a6 100644 --- a/backend/ibex/endpoints/schemas/request_data_schemas.py +++ b/backend/ibex/endpoints/schemas/request_data_schemas.py @@ -81,57 +81,6 @@ class GaussianSmoothingParameters(BaseModel): ) -class SimpleOperationsParameters(BaseModel): - addition_addend: float | None = Field( - default=None, - description="Scalar value added to every data point.", - ) - addition_priority: int | None = Field( - default=None, - description="Execution order priority for addition. Lower value = earlier execution. Default: 1.", - ) - subtraction_subtrahend: float | None = Field( - default=None, - description="Scalar value subtracted from every data point.", - ) - subtraction_priority: int | None = Field( - default=None, - description="Execution order priority for subtraction. Lower value = earlier execution. Default: 2.", - ) - multiplication_factor: float | None = Field( - default=None, - description="Scalar value used to multiply every data point.", - ) - multiplication_priority: int | None = Field( - default=None, - description="Execution order priority for multiplication. Lower value = earlier execution. Default: 3.", - ) - division_divisor: float | None = Field( - default=None, - description="Scalar value used as the divisor for every data point.", - ) - division_priority: int | None = Field( - default=None, - description="Execution order priority for division. Lower value = earlier execution. Default: 4.", - ) - exponentiation_exponent: float | None = Field( - default=None, - description="Scalar exponent used to raise the input data to a power.", - ) - exponentiation_priority: int | None = Field( - default=None, - description="Execution order priority for exponentiation. Lower value = earlier execution. Default: 5.", - ) - root_degree: float | None = Field( - default=None, - description="Scalar degree used to compute the nth root of the input data.", - ) - root_priority: int | None = Field( - default=None, - description="Execution order priority for root. Lower value = earlier execution. Default: 6.", - ) - - class PlotDataBasicParameters(BaseModel): """...""" @@ -143,13 +92,16 @@ class PlotDataBasicParameters(BaseModel): downsampling_method: str | None = Field(default=None, description="Downsampling method to be used") downsampled_size: int = Field(default=1000, description="Desired size of the data after downsampling") smoothing_method: SmoothingMethod | None = Field(default=None, description="Smoothing method to be used") + operations: Optional[List[str]] = Field( + default=None, + description="Ordered list of scalar operations in format 'type:value' e.g. 'add:10'", + ) class PlotDataRequestModel( PlotDataBasicParameters, SavgolSmoothingParameters, GaussianSmoothingParameters, - SimpleOperationsParameters, ): @model_validator(mode="after") def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": @@ -168,22 +120,3 @@ def validate_gaussian_smoothing_parameters(self) -> "PlotDataRequestModel": ) return self - - @model_validator(mode="after") - def validate_arithmetic_parameters(self) -> "PlotDataRequestModel": - if self.division_divisor == 0: - raise ValueError("division_divisor cannot be 0") - - priorities = [ - self.addition_priority, - self.subtraction_priority, - self.multiplication_priority, - self.division_priority, - self.exponentiation_priority, - self.root_priority, - ] - defined_priorities = [p for p in priorities if p is not None] - if len(defined_priorities) != len(set(defined_priorities)): - raise ValueError("operation priorities must be unique") - - return self diff --git a/backend/tests/test_data_endpoints.py b/backend/tests/test_data_endpoints.py index e390f92a..791a325e 100644 --- a/backend/tests/test_data_endpoints.py +++ b/backend/tests/test_data_endpoints.py @@ -100,23 +100,23 @@ def test_plot_data_with_simple_operations(entry_path): # core_profiles.time = [1,2,3,4,5] (float) cases = [ ( - {"addition_addend": 2, "multiplication_factor": 3}, + {"operations": ["add:2", "mul:3"]}, [9.0, 12.0, 15.0, 18.0, 21.0], ), ( - {"addition_addend": 2, "multiplication_factor": 3, "addition_priority": 6, "multiplication_priority": 1}, + {"operations": ["mul:3", "add:2"]}, [5.0, 8.0, 11.0, 14.0, 17.0], ), ( - {"division_divisor": 2}, + {"operations": ["div:2"]}, [0.5, 1.0, 1.5, 2.0, 2.5], ), ( - {"exponentiation_exponent": 2}, + {"operations": ["pow:2"]}, [1.0, 4.0, 9.0, 16.0, 25.0], ), ( - {"root_degree": 2}, + {"operations": ["root:2"]}, [1.0, 1.41421356237, 1.73205080757, 2.0, 2.2360679775], ), ] diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 5ee91c1c..d0eeae7f 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -1,12 +1,11 @@ import numpy as np import pytest +from ibex.data_source.exception import InvalidParametersException from ibex.data_source.imas_python_source_utils import ( apply_gaussian_filter, apply_savgol_filter, apply_simple_operations, ) -from ibex.endpoints.schemas.request_data_schemas import PlotDataRequestModel -from pydantic_core._pydantic_core import ValidationError def test_apply_gaussian_smoothing(): @@ -53,47 +52,37 @@ def test_apply_savitzky_golay_smoothing(): @pytest.mark.parametrize( - ("request_kwargs", "data", "expected"), + ("operations", "data", "expected"), [ - ({"addition_addend": 2}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), - ({"subtraction_subtrahend": 1}, np.array([3.0, 4.0, 5.0]), np.array([2.0, 3.0, 4.0])), - ({"multiplication_factor": 3}, np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), - ({"division_divisor": 2}, np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), - ({"exponentiation_exponent": 2}, np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), - ({"root_degree": 2}, np.array([1.0, 4.0, 9.0]), np.array([1.0, 2.0, 3.0])), + (["add:2"], np.array([1.0, 2.0, 3.0]), np.array([3.0, 4.0, 5.0])), + (["sub:1"], np.array([3.0, 4.0, 5.0]), np.array([2.0, 3.0, 4.0])), + (["mul:3"], np.array([1.0, 2.0, 3.0]), np.array([3.0, 6.0, 9.0])), + (["div:2"], np.array([2.0, 4.0, 6.0]), np.array([1.0, 2.0, 3.0])), + (["pow:2"], np.array([2.0, 3.0, 4.0]), np.array([4.0, 9.0, 16.0])), + (["root:2"], np.array([1.0, 4.0, 9.0]), np.array([1.0, 2.0, 3.0])), ], ) -def test_apply_simple_operations(request_kwargs, data, expected): - request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", **request_kwargs) - - assert np.asarray(expected) == pytest.approx(apply_simple_operations(data, request)) +def test_apply_simple_operations(operations, data, expected): + assert np.asarray(expected) == pytest.approx(apply_simple_operations(data, operations)) def test_apply_simple_operations_recurses_over_lists(): - request = PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", addition_addend=1) data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] - - result = apply_simple_operations(data, request) - + result = apply_simple_operations(data, ["add:1"]) assert np.asarray(result[0]) == pytest.approx([2.0, 3.0]) assert np.asarray(result[1]) == pytest.approx([4.0, 5.0]) def test_apply_simple_operations_rejects_division_by_zero(): - with pytest.raises(ValidationError, match="division_divisor cannot be 0"): - PlotDataRequestModel(uri="imas:hdf5?path=/dummy#dummy", division_divisor=0) + with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + apply_simple_operations(np.array([1.0]), ["div:0"]) -def test_apply_simple_operations_uses_priority_order(): - request = PlotDataRequestModel( - uri="imas:hdf5?path=/dummy#dummy", - addition_addend=1, - multiplication_factor=2, - addition_priority=2, - multiplication_priority=1, - ) +def test_apply_simple_operations_uses_order(): data = np.array([5.0]) - # default order: add then multiply -> (5+1)*2 = 12 - # priority order: multiply then add -> (5*2)+1 = 11 - result = apply_simple_operations(data, request) + # mul then add -> (5*2)+1 = 11 + result = apply_simple_operations(data, ["mul:2", "add:1"]) assert result == pytest.approx([11.0]) + # add then mul -> (5+1)*2 = 12 + result = apply_simple_operations(data, ["add:1", "mul:2"]) + assert result == pytest.approx([12.0]) From f5da1cfce42e5c94655f33ea5a953e5568fe0883 Mon Sep 17 00:00:00 2001 From: wasikj Date: Wed, 17 Jun 2026 10:42:12 +0200 Subject: [PATCH 26/28] Fix wrong description --- backend/ibex/core/data_manipulation_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ibex/core/data_manipulation_methods.py b/backend/ibex/core/data_manipulation_methods.py index d768259f..74696422 100644 --- a/backend/ibex/core/data_manipulation_methods.py +++ b/backend/ibex/core/data_manipulation_methods.py @@ -87,7 +87,7 @@ class DataManipulationMethodsResponse(BaseModel): data_interpolation_method_parameter = DataManipulationParameter( human_readable_name="Interpolation method", name="interpolation_method", - description="List of URIs to gather coordinates from, for interpolation", + description="Method used during data interpolation", type="string", default=InterpolationMethod.EXACT_VALUE, possible_values=[ From 6639d458b14c3d8e09274146170f11a28d39e875 Mon Sep 17 00:00:00 2001 From: wasikj Date: Wed, 17 Jun 2026 10:54:56 +0200 Subject: [PATCH 27/28] Update docstring and test. --- .../data_source/imas_python_source_utils.py | 23 +++++++++++-------- backend/tests/test_data_manipulation.py | 6 ++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index e24b37a0..29ab3be2 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -71,21 +71,24 @@ def _safe_division(data, divisor): return data / divisor -_OP_FUNCS = { - "add": lambda r, v: r + v, - "sub": lambda r, v: r - v, - "mul": lambda r, v: r * v, - "div": lambda r, v: _safe_division(r, v), - "pow": lambda r, v: np.power(r, v), - "root": lambda r, v: np.power(r, 1 / v), -} - - def apply_simple_operations(data: list | np.ndarray, operations: list[str]): """ Apply simple scalar operations to data in the order given. Each operation is a string in the format 'type:value', e.g. 'add:10', 'mul:5'. + :param data: Input data + :param operations: List of operations and operands divided by colon (:) + :return: Data after operation """ + + _OP_FUNCS = { + "add": lambda r, v: r + v, + "sub": lambda r, v: r - v, + "mul": lambda r, v: r * v, + "div": lambda r, v: _safe_division(r, v), + "pow": lambda r, v: np.power(r, v), + "root": lambda r, v: np.power(r, 1 / v), + } + if isinstance(data, list): return [apply_simple_operations(x, operations) for x in data] elif isinstance(data, (np.ndarray, IDSNumericArray)): diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index d0eeae7f..2ff43a52 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -68,9 +68,9 @@ def test_apply_simple_operations(operations, data, expected): def test_apply_simple_operations_recurses_over_lists(): data = [np.array([1.0, 2.0]), np.array([3.0, 4.0])] - result = apply_simple_operations(data, ["add:1"]) - assert np.asarray(result[0]) == pytest.approx([2.0, 3.0]) - assert np.asarray(result[1]) == pytest.approx([4.0, 5.0]) + result = apply_simple_operations(data, ["add:1", "mul:2", "add:3"]) + assert np.asarray(result[0]) == pytest.approx([7.0, 9.0]) + assert np.asarray(result[1]) == pytest.approx([11.0, 13.0]) def test_apply_simple_operations_rejects_division_by_zero(): From d79fac57fecf557a2c3264a21ca201af996a58da Mon Sep 17 00:00:00 2001 From: wasikj Date: Thu, 18 Jun 2026 09:18:18 +0200 Subject: [PATCH 28/28] Update docs. Mini-fixes. --- .../data_source/imas_python_source_utils.py | 3 +- backend/tests/test_data_manipulation.py | 2 +- .../backend_development/data_manipulation.rst | 64 +++++++++++++------ 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/backend/ibex/data_source/imas_python_source_utils.py b/backend/ibex/data_source/imas_python_source_utils.py index 29ab3be2..4e22c52c 100644 --- a/backend/ibex/data_source/imas_python_source_utils.py +++ b/backend/ibex/data_source/imas_python_source_utils.py @@ -67,7 +67,7 @@ def apply_gaussian_filter(data: list | np.ndarray, sigma): def _safe_division(data, divisor): if divisor == 0: - raise InvalidParametersException("division_divisor cannot be 0") + raise InvalidParametersException("Division by zero is not allowed") return data / divisor @@ -79,7 +79,6 @@ def apply_simple_operations(data: list | np.ndarray, operations: list[str]): :param operations: List of operations and operands divided by colon (:) :return: Data after operation """ - _OP_FUNCS = { "add": lambda r, v: r + v, "sub": lambda r, v: r - v, diff --git a/backend/tests/test_data_manipulation.py b/backend/tests/test_data_manipulation.py index 2ff43a52..78ca8633 100644 --- a/backend/tests/test_data_manipulation.py +++ b/backend/tests/test_data_manipulation.py @@ -74,7 +74,7 @@ def test_apply_simple_operations_recurses_over_lists(): def test_apply_simple_operations_rejects_division_by_zero(): - with pytest.raises(InvalidParametersException, match="division_divisor cannot be 0"): + with pytest.raises(InvalidParametersException, match="Division by zero is not allowed"): apply_simple_operations(np.array([1.0]), ["div:0"]) diff --git a/docs/source/developers_manual/backend_development/data_manipulation.rst b/docs/source/developers_manual/backend_development/data_manipulation.rst index e0e02599..39263c23 100644 --- a/docs/source/developers_manual/backend_development/data_manipulation.rst +++ b/docs/source/developers_manual/backend_development/data_manipulation.rst @@ -95,48 +95,70 @@ Savitzky-Golay smoothing: -H 'accept: application/json' -Simple scalar operations +Simple data operations ------------------------ -IBEX also supports a sequence of scalar operations that can be applied to the returned dataset: +IBEX supports a sequence of scalar operations that can be applied element-wise to every data point in the returned dataset. +Simple data operations use fixed numeric values. -* addition -* subtraction -* multiplication -* division -* exponentiation -* root +The following operation types are supported: -These operations are executed in that order. In practice, the backend applies them sequentially to the numerical data before any smoothing or resampling step. +* ``add`` — addition +* ``sub`` — subtraction +* ``mul`` — multiplication +* ``div`` — division +* ``pow`` — exponentiation +* ``root`` — Nth root -The corresponding request parameters are: +Operations are applied **in the order they appear** in the request. +The backend executes them sequentially on the numerical data **before** smoothing, interpolation, or downsampling. -* ``addition_addend`` -* ``subtraction_subtrahend`` -* ``multiplication_factor`` -* ``division_divisor`` -* ``exponentiation_exponent`` -* ``root_degree`` +Configuration +~~~~~~~~~~~~~~ + +Simple data operations are configured through the ``operations`` parameter of the ``/data/plot_data/`` endpoint. +It accepts a list of strings in the format ``type:value``, for example ``add:10`` or ``mul:2.5``. + +The full list of available operations can be retrieved from the ``/info/data_manipulation_methods/`` endpoint. + +Division by zero is rejected by the backend with an ``InvalidParametersException`` ("Division by zero is not allowed"). + +Implementation +~~~~~~~~~~~~~~~ -Division by zero is rejected by the backend. +Simple data operations are applied in ``apply_simple_operations()`` in +``backend/ibex/data_source/imas_python_source_utils.py``. + +The function iterates over the list of operation strings in order, splitting each on the colon to extract the operation type and the scalar value. +For each operation the corresponding arithmetic lambda is applied element-wise to the data array. + +The implementation handles both flat arrays and nested lists of arrays (higher-dimensional data) recursively. Example usage ~~~~~~~~~~~~~~ The following examples demonstrate how simple data operations can be enabled for testing purposes. -Single operation: +Single operation (addition by 2): + +.. code-block:: bash + + curl -X 'GET' \ + '/data/plot_data?uri=&operations=add:2' \ + -H 'accept: application/json' + +Two chained operations (add then multiply): .. code-block:: bash curl -X 'GET' \ - '/data/plot_data?uri=&addition_addend=2' \ + '/data/plot_data?uri=&operations=add:2&operations=mul:3' \ -H 'accept: application/json' -Two operations: +Order matters (multiply then add yields different result): .. code-block:: bash curl -X 'GET' \ - '/data/plot_data?uri=&addition_addend=2&multiplication_factor=3' \ + '/data/plot_data?uri=&operations=mul:3&operations=add:2' \ -H 'accept: application/json'