diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 3bbaa6cbb..dd05d4f72 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -18,6 +18,7 @@ New features * Show sensor attributes on sensor page, if not empty [see `PR #2015 `_] * Separate the ``StorageScheduler``'s tie-breaking preference for a full :abbr:`SoC (state of charge)` from its reported energy costs [see `PR #2023 `_] * Improve asset audit log messages for JSON field edits (especially ``sensors_to_show`` and nested flex-config values) [see `PR #2055 `_] +* Added a form on the UI for deleting sensor data sources [see `PR #2095 `_] Infrastructure / Support ---------------------- diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 62a427321..2727a11da 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -59,7 +59,7 @@ DurationField, PlanningDurationField, ) -from flexmeasures.data.schemas import AssetIdField +from flexmeasures.data.schemas import AssetIdField, SourceIdField from flexmeasures.api.common.schemas.search import SearchFilterField from flexmeasures.data.schemas.scheduling import GetScheduleSchema from flexmeasures.data.schemas.units import UnitField @@ -1367,15 +1367,33 @@ def delete(self, id: int, sensor: Sensor): @route("//data", methods=["DELETE"]) @use_kwargs({"sensor": SensorIdField(data_key="id")}, location="path") + @use_kwargs( + { + "source": SourceIdField(load_default=None), + "start": AwareDateTimeField(load_default=None), + "until": AwareDateTimeField(load_default=None), + }, + location="json", + ) @permission_required_for_context("delete", ctx_arg_name="sensor") @as_json - def delete_data(self, id: int, sensor: Sensor): + def delete_data( + self, + id: int, + sensor: Sensor, + source=None, + start=None, + until=None, + ): """ .. :quickref: Sensors; Delete sensor data --- delete: summary: Delete sensor data - description: This endpoint deletes all data for a sensor. + description: > + This endpoint deletes data for a sensor. + Optionally, filter by source, start time and/or until time. + A missing source means all sources are deleted. security: - ApiKeyAuth: [] parameters: @@ -1384,6 +1402,24 @@ def delete_data(self, id: int, sensor: Sensor): description: ID of the sensor to delete data for. required: true schema: SensorId + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + source: + type: integer + description: ID of the data source to delete data for. If not provided, data from all sources is deleted. + start: + type: string + format: date-time + description: Only delete data with event start at or after this datetime (ISO 8601). + until: + type: string + format: date-time + description: Only delete data with event start before this datetime (ISO 8601). responses: 204: description: SENSOR_DATA_DELETED @@ -1398,10 +1434,25 @@ def delete_data(self, id: int, sensor: Sensor): tags: - Sensors """ - db.session.execute(delete(TimedBelief).filter_by(sensor_id=sensor.id)) + query = delete(TimedBelief).where(TimedBelief.sensor_id == sensor.id) + if source is not None: + query = query.where(TimedBelief.source_id == source.id) + if start is not None: + query = query.where(TimedBelief.event_start >= start) + if until is not None: + query = query.where(TimedBelief.event_start < until) + db.session.execute(query) + + audit_message = f"Deleted data for sensor '{sensor.name}': {sensor.id}" + if source is not None: + audit_message += f", source: {source.id}" + if start is not None: + audit_message += f", from: {start}" + if until is not None: + audit_message += f", until: {until}" AssetAuditLog.add_record( sensor.generic_asset, - f"Deleted data for sensor '{sensor.name}': {sensor.id}", + audit_message, ) db.session.commit() @@ -1414,6 +1465,7 @@ def delete_data(self, id: int, sensor: Sensor): "sort_keys": fields.Boolean(data_key="sort", load_default=True), "event_start_time": fields.Str(load_default=None), "event_end_time": fields.Str(load_default=None), + "fresh": fields.Boolean(load_default=False), }, location="query", ) @@ -1426,6 +1478,7 @@ def get_stats( event_start_time: str, event_end_time: str, sort_keys: bool, + fresh: bool, ): """ .. :quickref: Sensors; Get sensor stats @@ -1454,7 +1507,12 @@ def get_stats( format: date-time - in: query name: sort_keys - description: Whether to sort the stats by keys. + description: Whether to sort the stats by keys (defaults to true). + schema: + type: boolean + - in: query + name: fresh + description: Whether to compute fresh data, bypassing any cached results (defaults to false). schema: type: boolean responses: @@ -1487,9 +1545,14 @@ def get_stats( tags: - Sensors """ - return ( - get_sensor_stats(sensor, event_start_time, event_end_time, sort_keys), + get_sensor_stats( + sensor, + event_start_time, + event_end_time, + sort_keys, + from_cache=not fresh, + ), 200, ) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_delete_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensors_delete_data_fresh_db.py new file mode 100644 index 000000000..49c804c62 --- /dev/null +++ b/flexmeasures/api/v3_0/tests/test_sensors_delete_data_fresh_db.py @@ -0,0 +1,152 @@ +"""Tests for DELETE /api/v3_0/sensors//data with source, start and until filters. + +These tests use fresh_db (function-scoped) to ensure data isolation between tests, +since each test mutates the sensor data. +""" + +from __future__ import annotations + +import pytest + +from flask import url_for +from sqlalchemy import select + +from flexmeasures.data.models.time_series import TimedBelief +from flexmeasures import Sensor +from flexmeasures.api.v3_0.tests.utils import check_audit_log_event + + +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_delete_sensor_data_by_source( + client, setup_api_fresh_test_data, requesting_user, fresh_db +): + """Deleting sensor data with a source filter only removes beliefs from that source.""" + existing_sensor = setup_api_fresh_test_data["some gas sensor"] + existing_sensor_id = existing_sensor.id + + # Collect distinct source ids for this sensor + all_beliefs = fresh_db.session.scalars( + select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id) + ).all() + assert len(all_beliefs) > 0 + source_ids = list({b.source_id for b in all_beliefs}) + assert len(source_ids) >= 2, "Need at least two sources for this test" + + # Pick one source to delete + source_id_to_delete = source_ids[0] + + # Delete sensor data for that source only + delete_data_response = client.delete( + url_for("SensorAPI:delete_data", id=existing_sensor_id), + json={"source": source_id_to_delete}, + ) + assert delete_data_response.status_code == 204 + + remaining_beliefs = fresh_db.session.scalars( + select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id) + ).all() + + # Beliefs from the deleted source should be gone + deleted_source_beliefs = [ + b for b in remaining_beliefs if b.source_id == source_id_to_delete + ] + assert deleted_source_beliefs == [] + + # Beliefs from other sources should remain + other_beliefs = [b for b in remaining_beliefs if b.source_id != source_id_to_delete] + assert len(other_beliefs) > 0 + + deleted_sensor = fresh_db.session.get(Sensor, existing_sensor_id) + assert deleted_sensor is not None, "Sensor itself should not be deleted" + + check_audit_log_event( + db=fresh_db, + event=f"Deleted data for sensor '{existing_sensor.name}': {existing_sensor.id}, source: {source_id_to_delete}", + user=requesting_user, + asset=existing_sensor.generic_asset, + ) + + +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_delete_sensor_data_by_start( + client, setup_api_fresh_test_data, requesting_user, fresh_db +): + """Deleting sensor data with a start filter only removes beliefs at or after that time.""" + existing_sensor = setup_api_fresh_test_data["some gas sensor"] + existing_sensor_id = existing_sensor.id + + all_beliefs = fresh_db.session.scalars( + select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id) + ).all() + assert len(all_beliefs) >= 2 + + # Use the second distinct event_start as the cutoff: beliefs at or after it should be deleted + event_starts = sorted({b.event_start for b in all_beliefs}) + cutoff = event_starts[1] + + delete_data_response = client.delete( + url_for("SensorAPI:delete_data", id=existing_sensor_id), + json={"start": cutoff.isoformat()}, + ) + assert delete_data_response.status_code == 204 + + remaining_beliefs = fresh_db.session.scalars( + select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id) + ).all() + + # All remaining beliefs should have event_start < cutoff + for b in remaining_beliefs: + assert b.event_start < cutoff + + deleted_sensor = fresh_db.session.get(Sensor, existing_sensor_id) + assert deleted_sensor is not None, "Sensor itself should not be deleted" + + check_audit_log_event( + db=fresh_db, + event=f"Deleted data for sensor '{existing_sensor.name}': {existing_sensor.id}, from: {cutoff}", + user=requesting_user, + asset=existing_sensor.generic_asset, + ) + + +@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True) +def test_delete_sensor_data_by_until( + client, setup_api_fresh_test_data, requesting_user, fresh_db +): + """Deleting sensor data with an until filter only removes beliefs before that time.""" + existing_sensor = setup_api_fresh_test_data["some gas sensor"] + existing_sensor_id = existing_sensor.id + + all_beliefs = fresh_db.session.scalars( + select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id) + ).all() + assert len(all_beliefs) >= 2 + + # Use the last distinct event_start as the until cutoff: + # beliefs strictly before it should be deleted + event_starts = sorted({b.event_start for b in all_beliefs}) + cutoff = event_starts[-1] + + delete_data_response = client.delete( + url_for("SensorAPI:delete_data", id=existing_sensor_id), + json={"until": cutoff.isoformat()}, + ) + assert delete_data_response.status_code == 204 + + remaining_beliefs = fresh_db.session.scalars( + select(TimedBelief).filter(TimedBelief.sensor_id == existing_sensor_id) + ).all() + + # All remaining beliefs should have event_start >= cutoff + for b in remaining_beliefs: + assert b.event_start >= cutoff + + deleted_sensor = fresh_db.session.get(Sensor, existing_sensor_id) + assert deleted_sensor is not None, "Sensor itself should not be deleted" + + check_audit_log_event( + db=fresh_db, + event=f"Deleted data for sensor '{existing_sensor.name}': {existing_sensor.id}, until: {cutoff}", + user=requesting_user, + asset=existing_sensor.generic_asset, + ) diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index 82a32391a..a0c5a88ba 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -580,7 +580,11 @@ def _get_sensor_stats( def get_sensor_stats( - sensor: Sensor, event_start_time: str, event_end_time: str, sort_keys: bool = True + sensor: Sensor, + event_start_time: str, + event_end_time: str, + sort_keys: bool = True, + from_cache: bool = True, ) -> dict: """Get stats for a sensor. @@ -591,7 +595,7 @@ def get_sensor_stats( bucket = round(time.time() / _SENSOR_STATS_TTL) key = (sensor.id, event_end_time, event_start_time, sort_keys, bucket) - if key in _sensor_stats_cache: + if from_cache and key in _sensor_stats_cache: return _sensor_stats_cache[key] result = _get_sensor_stats(sensor, event_end_time, event_start_time, sort_keys) diff --git a/flexmeasures/ui/static/js/flexmeasures.js b/flexmeasures/ui/static/js/flexmeasures.js index d42c21d8a..d0a3408e2 100644 --- a/flexmeasures/ui/static/js/flexmeasures.js +++ b/flexmeasures/ui/static/js/flexmeasures.js @@ -649,7 +649,7 @@ function updateStatsTable(stats, tableBody) { }); } -function loadSensorStats(sensor_id, event_start_time="", event_end_time="") { +function loadSensorStats(sensor_id, event_start_time="", event_end_time="", fresh=false) { const spinner = document.getElementById('spinner-run-simulation'); const dropdownContainer = document.getElementById('sourceKeyDropdownContainer'); const tableBody = document.getElementById('statsTableBody'); @@ -664,7 +664,11 @@ function loadSensorStats(sensor_id, event_start_time="", event_end_time="") { if (toggleStatsCheckbox.checked) { queryParams = `?sort=false&event_start_time=${event_start_time}&event_end_time=${event_end_time}` } - + //add a cache buster to ensure we get the latest data after an upload + if (fresh === true) { + queryParams += `&fresh=true`; + } + // Enable all the default behaviors on every API call. dropdownMenu.innerHTML = ''; noDataWarning.classList.add('d-none'); @@ -708,6 +712,39 @@ function loadSensorStats(sensor_id, event_start_time="", event_end_time="") { const firstSourceKey = getLatestBeliefName(data); dropdownButton.textContent = firstSourceKey; updateStatsTable(data[firstSourceKey], tableBody); + + // Populate the "Delete data" source dropdown if it exists on the page, + // re-using the stats data already fetched to avoid a duplicate API call. + const deleteSourceSelect = document.getElementById('deleteDataSource'); + if (deleteSourceSelect) { + // Keep only the "All sources" placeholder option, then add sources from stats + deleteSourceSelect.innerHTML = ''; + Object.keys(data).forEach(sourceKey => { + const idMatch = sourceKey.match(/\(ID:\s*(\d+)\)$/); + if (!idMatch) { return; } + const option = document.createElement('option'); + option.value = idMatch[1]; + option.textContent = sourceKey; + deleteSourceSelect.appendChild(option); + }); + } + + // Notify the "Delete data" panel of the overall first/last event times + // across all sources so the "Select all data" link can populate the inputs. + const firstEventDates = Object.values(data) + .map(d => new Date(d["First event start"])) + .filter(d => !isNaN(d.getTime())); + const lastEventDates = Object.values(data) + .map(d => new Date(d["Last event end"])) + .filter(d => !isNaN(d.getTime())); + if (firstEventDates.length > 0 && lastEventDates.length > 0) { + document.dispatchEvent(new CustomEvent('sensorDataRangeAvailable', { + detail: { + firstEventStart: new Date(Math.min(...firstEventDates)), + lastEventEnd: new Date(Math.max(...lastEventDates)) + } + })); + } } else { // If the stats table is empty, make the properties table full width noDataWarning.classList.remove('d-none'); diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 37a900274..e3e562b0d 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.31.0" + "version": "0.31.1" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -300,7 +300,7 @@ "/api/v3_0/sensors/{id}/data": { "delete": { "summary": "Delete sensor data", - "description": "This endpoint deletes all data for a sensor.", + "description": "This endpoint deletes data for a sensor. Optionally, filter by source, start time and/or until time. A missing source means all sources are deleted.\n", "security": [ { "ApiKeyAuth": [] @@ -316,6 +316,32 @@ "required": true } ], + "requestBody": { + "required": false, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "integer", + "description": "ID of the data source to delete data for. If not provided, data from all sources is deleted." + }, + "start": { + "type": "string", + "format": "date-time", + "description": "Only delete data with event start at or after this datetime (ISO 8601)." + }, + "until": { + "type": "string", + "format": "date-time", + "description": "Only delete data with event start before this datetime (ISO 8601)." + } + } + } + } + } + }, "responses": { "204": { "description": "SENSOR_DATA_DELETED" @@ -805,7 +831,15 @@ { "in": "query", "name": "sort_keys", - "description": "Whether to sort the stats by keys.", + "description": "Whether to sort the stats by keys (defaults to true).", + "schema": { + "type": "boolean" + } + }, + { + "in": "query", + "name": "fresh", + "description": "Whether to compute fresh data, bypassing any cached results (defaults to false).", "schema": { "type": "boolean" } diff --git a/flexmeasures/ui/templates/includes/graphs.html b/flexmeasures/ui/templates/includes/graphs.html index 30470f250..62164b790 100644 --- a/flexmeasures/ui/templates/includes/graphs.html +++ b/flexmeasures/ui/templates/includes/graphs.html @@ -234,11 +234,11 @@ document.getElementById('toggleStatsCheckbox').addEventListener('change', function () { if (this.checked) { loadSensorStats({{ sensor.id }}, storeStartDate.toISOString(), storeEndDate.toISOString()); - } - else { - loadSensorStats({{ sensor.id }}); - } - }); + } + else { + loadSensorStats({{ sensor.id }}, "", "", true); + } + }); {% endif %} // Keypress Listener @@ -281,7 +281,26 @@ }; } document.addEventListener('sensorsToShowUpdated', reloadChartData); - document.addEventListener('newDataAvailable', reloadChartData); + {% if active_page == "sensors" %} + async function reloadSensorStats() { + const toggleStats = document.getElementById('toggleStatsCheckbox'); + if (toggleStats) { + if (toggleStats.checked) { + loadSensorStats({{ sensor.id }}, storeStartDate.toISOString(), storeEndDate.toISOString(), true); + } else { + loadSensorStats({{ sensor.id }}, "", "", true); + } + } + } + document.addEventListener('newDataAvailable', function() { + reloadChartData(); + reloadSensorStats(); + }); + document.addEventListener('dataDeleted', function() { + reloadChartData(); + reloadSensorStats(); + }); + {% endif %} {% if event_starts_after and event_ends_before %} $("#spinner").show(); @@ -445,14 +464,20 @@ var queryStartDate = (startDate != null) ? (startDate.toISOString()) : (null); var queryEndDate = (endDate != null) ? (endDate.toISOString()) : (null); + // Notify the "Delete data" panel of the new graph date range so its inputs stay in sync. + // detail: { startDate: Date, endDate: Date } – endDate has +1 day applied (exclusive end). + document.dispatchEvent(new CustomEvent('graphDateRangeChanged', { + detail: { startDate: startDate, endDate: endDate } + })); + {% if active_page == "sensors" %} const toggleStats = document.getElementById('toggleStatsCheckbox'); if (toggleStats.checked) { loadSensorStats({{ sensor.id }}, storeStartDate.toISOString(), storeEndDate.toISOString()); - } - else { - loadSensorStats({{ sensor.id }}); - } + } + else { + loadSensorStats({{ sensor.id }}, "", ""); + } {% endif %} stopReplay() diff --git a/flexmeasures/ui/templates/includes/toasts.html b/flexmeasures/ui/templates/includes/toasts.html index b44d232c6..d934f5fa0 100644 --- a/flexmeasures/ui/templates/includes/toasts.html +++ b/flexmeasures/ui/templates/includes/toasts.html @@ -38,7 +38,7 @@ toastInstance.show(); }); } - + function maybeHideCloseToastBtn() { const remainingToasts = toastStack.querySelectorAll(".toast"); if (remainingToasts.length === 0) { @@ -136,12 +136,6 @@ }; toast.addEventListener("hidden.bs.toast", handleClose); - toast.querySelector(".btn-close").addEventListener("click", function() { - const instance = new bootstrap.Toast(toast); - instance.dispose(); // Close immediately on click - toast.remove(); - maybeHideCloseToastBtn(); - }); }; })(); diff --git a/flexmeasures/ui/templates/sensors/index.html b/flexmeasures/ui/templates/sensors/index.html index bc5979143..c918ba301 100644 --- a/flexmeasures/ui/templates/sensors/index.html +++ b/flexmeasures/ui/templates/sensors/index.html @@ -196,6 +196,43 @@

Upload {{ sensor.name }} data

{% endif %} + + {% if user_can_delete_sensor %} +
+
Delete data
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ Select all data +
+ + +
+
+
+
+
+ {% endif %}