From 1b267f86493f17456ba5422b9b224180e0d7bc4d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:55:26 +0000 Subject: [PATCH 01/13] Initial plan From 4dc94ca70e5d08ca4f4b47c100ad012a105b7c02 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:04:06 +0000 Subject: [PATCH 02/13] Fix KeyError 'sensors' crash on asset status page for old sensors_to_show format Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/09f0ee2e-71f7-4389-a447-23f20fee1364 Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --- .../schemas/tests/test_sensor_data_schema.py | 27 +++++++++++++++++++ flexmeasures/data/services/sensors.py | 8 +++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index b1590937a8..c391ea230f 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -361,6 +361,33 @@ def test_get_status_no_status_specs( assert expected_stale_reason in sensor_status["reason"] +def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_sensors): + """ + Regression test: asset status page crashed with KeyError 'sensors' when sensors_to_show + contained the old format with a singular 'sensor' key, e.g. {"title": "Prices", "sensor": 42}. + """ + asset = add_weather_sensors["asset"] + wind_sensor, temperature_sensor = ( + add_weather_sensors["wind"], + add_weather_sensors["temperature"], + ) + + # Use the old format: one entry with plural "sensors" and one with singular "sensor" + asset.sensors_to_show = [ + {"title": "Group", "sensors": [wind_sensor.id, temperature_sensor.id]}, + {"title": "Solo", "sensor": wind_sensor.id}, + ] + db.session.add(asset) + db.session.flush() + + # Should not raise a KeyError + status_data = get_asset_sensors_metadata(asset=asset) + + sensor_ids = [s["id"] for s in status_data] + assert wind_sensor.id in sensor_ids + assert temperature_sensor.id in sensor_ids + + def test_asset_sensors_metadata( db, mock_get_statuses, add_weather_sensors, add_battery_assets ): diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index 82a32391a0..10faf75fb8 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -293,14 +293,12 @@ def get_asset_sensors_metadata( # Get sensors to show using the validate_sensors_to_show method sensors_to_show = [] + from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema + validated_asset_sensors = asset.validate_sensors_to_show( suggest_default_sensors=False ) - sensor_groups = [ - sensor["sensors"] for sensor in validated_asset_sensors if sensor is not None - ] - merged_sensor_groups = sum(sensor_groups, []) - sensors_to_show.extend(merged_sensor_groups) + sensors_to_show.extend(SensorsToShowSchema.flatten(validated_asset_sensors)) sensors_list = [ *inflexible_device_sensors, From 4bc5233830c0cd8c1bda1fe173ca8a8929f14786 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:06:09 +0000 Subject: [PATCH 03/13] Move SensorsToShowSchema import to module level Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/09f0ee2e-71f7-4389-a447-23f20fee1364 Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --- flexmeasures/data/services/sensors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index 10faf75fb8..aa43937f4b 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -22,6 +22,7 @@ from flexmeasures import Sensor, Account, Asset from flexmeasures.data.models.data_sources import DataSource, DEFAULT_DATASOURCE_TYPES from flexmeasures.data.models.generic_assets import GenericAsset +from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema from flexmeasures.data.schemas.reporting import StatusSchema from flexmeasures.utils.time_utils import server_now @@ -293,8 +294,6 @@ def get_asset_sensors_metadata( # Get sensors to show using the validate_sensors_to_show method sensors_to_show = [] - from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema - validated_asset_sensors = asset.validate_sensors_to_show( suggest_default_sensors=False ) From 6980c7a6c3529b99141cf57c7d1f41ef231aed42 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:03:26 +0000 Subject: [PATCH 04/13] Fix test: flush session before reading sensor IDs, reset sensors_to_show after test Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/5ce56f2a-9e55-408b-9659-f12da4d6800a Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --- .../schemas/tests/test_sensor_data_schema.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index c391ea230f..faee850212 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -367,10 +367,11 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso contained the old format with a singular 'sensor' key, e.g. {"title": "Prices", "sensor": 42}. """ asset = add_weather_sensors["asset"] - wind_sensor, temperature_sensor = ( - add_weather_sensors["wind"], - add_weather_sensors["temperature"], - ) + wind_sensor = add_weather_sensors["wind"] + temperature_sensor = add_weather_sensors["temperature"] + + # Flush so sensor IDs are assigned before we embed them in sensors_to_show + db.session.flush() # Use the old format: one entry with plural "sensors" and one with singular "sensor" asset.sensors_to_show = [ @@ -380,12 +381,17 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso db.session.add(asset) db.session.flush() - # Should not raise a KeyError - status_data = get_asset_sensors_metadata(asset=asset) - - sensor_ids = [s["id"] for s in status_data] - assert wind_sensor.id in sensor_ids - assert temperature_sensor.id in sensor_ids + try: + # Should not raise a KeyError + status_data = get_asset_sensors_metadata(asset=asset) + + sensor_ids = [s["id"] for s in status_data] + assert wind_sensor.id in sensor_ids + assert temperature_sensor.id in sensor_ids + finally: + # Reset to avoid polluting shared module-scoped fixture state + asset.sensors_to_show = [] + db.session.flush() def test_asset_sensors_metadata( From cb81292ce99ddf0949eb76b842e82e99bd92dcea Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 20:43:31 +0200 Subject: [PATCH 05/13] Revert "Fix test: flush session before reading sensor IDs, reset sensors_to_show after test" This reverts commit 6980c7a6c3529b99141cf57c7d1f41ef231aed42. Signed-off-by: F.N. Claessen --- .../schemas/tests/test_sensor_data_schema.py | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index faee850212..c391ea230f 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -367,11 +367,10 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso contained the old format with a singular 'sensor' key, e.g. {"title": "Prices", "sensor": 42}. """ asset = add_weather_sensors["asset"] - wind_sensor = add_weather_sensors["wind"] - temperature_sensor = add_weather_sensors["temperature"] - - # Flush so sensor IDs are assigned before we embed them in sensors_to_show - db.session.flush() + wind_sensor, temperature_sensor = ( + add_weather_sensors["wind"], + add_weather_sensors["temperature"], + ) # Use the old format: one entry with plural "sensors" and one with singular "sensor" asset.sensors_to_show = [ @@ -381,17 +380,12 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso db.session.add(asset) db.session.flush() - try: - # Should not raise a KeyError - status_data = get_asset_sensors_metadata(asset=asset) - - sensor_ids = [s["id"] for s in status_data] - assert wind_sensor.id in sensor_ids - assert temperature_sensor.id in sensor_ids - finally: - # Reset to avoid polluting shared module-scoped fixture state - asset.sensors_to_show = [] - db.session.flush() + # Should not raise a KeyError + status_data = get_asset_sensors_metadata(asset=asset) + + sensor_ids = [s["id"] for s in status_data] + assert wind_sensor.id in sensor_ids + assert temperature_sensor.id in sensor_ids def test_asset_sensors_metadata( From dc6eabe9066e5944640ee36b1383a74cd982f229 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 20:51:27 +0200 Subject: [PATCH 06/13] fix: flush sensor IDs Signed-off-by: F.N. Claessen --- .../api/common/schemas/tests/test_sensor_data_schema.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index c391ea230f..6b9019b413 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -372,6 +372,9 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso add_weather_sensors["temperature"], ) + # Flush to ensure sensors have database IDs before using them. + db.session.flush() + # Use the old format: one entry with plural "sensors" and one with singular "sensor" asset.sensors_to_show = [ {"title": "Group", "sensors": [wind_sensor.id, temperature_sensor.id]}, From 4c5354ed51254466e6dc630fd341d50f1a6508d3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 21:24:50 +0200 Subject: [PATCH 07/13] feat: agent lesson learned Signed-off-by: F.N. Claessen --- AGENTS.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 540d6fa319..fd25672974 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -629,6 +629,50 @@ When conducting work: - Run the CLI commands or API calls mentioned - Verify the implementation works end-to-end +### Must Validate the Whole Test Module After Any Test Change + +**When fixing or adding a test, always run the entire test module — not just the single test.** + +Fixing one test can silently break adjacent tests in the same module, especially when: +- Fixtures are `scope="module"` and tests share mutable state (SQLAlchemy objects, in-memory dicts) +- A test modifies a shared object (e.g. `asset.sensors_to_show`) and does not clean up after itself +- The cleanup itself introduces a new failure (e.g. resetting to `None` instead of the column default `[]`) + +**Mandatory scope for test validation:** + +| Change type | Minimum scope to run | +|---|---| +| Fix a single failing test | Full test module | +| Add a new test | Full test module | +| Change a shared fixture | All modules that use that fixture | +| Change production code touched by tests | Full test suite | + +**Anti-patterns to avoid:** + +- ❌ Running only `-k test_name` and declaring success +- ❌ Assuming module-scoped fixture state is clean between tests +- ❌ Resetting shared objects to `None` when the column default is `[]` + +**Correct pattern:** + +```bash +# After fixing test_foo, always run the whole module: +python -m pytest path/to/test_module.py -v +# All tests must pass before closing. +``` + +**Lesson from session 2026-04-13:** +- Fixed `test_asset_sensors_metadata_old_sensors_to_show_format` in isolation — passed. +- Ran only `-k test_asset_sensors_metadata_old_sensors_to_show_format` — passed. +- Closed without running the full module. +- Next test `test_asset_sensors_metadata` in the same module was broken because the + regression test left `sensors_to_show = None` on a module-scoped asset fixture. +- Root cause: `validate_sensors_to_show` had a bug — `sensors_to_show = None` with + `suggest_default_sensors=False` fell through to `deserialize(None)` which raised + `ValidationError` instead of returning `[]` as documented. +- Fix required changes in both the test (reset to `[]`, the column default) and + production code (`validate_sensors_to_show` early-return guard split into two cases). + ### Must Make Atomic Commits **Never mix different types of changes in a single commit.** From 81a4fb17b942be1539067e086fcc0a9f5104784c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 21:25:10 +0200 Subject: [PATCH 08/13] fix: reset module-scoped fixture state Signed-off-by: F.N. Claessen --- .../api/common/schemas/tests/test_sensor_data_schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 6b9019b413..6f5cd10b5c 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -390,6 +390,10 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso assert wind_sensor.id in sensor_ids assert temperature_sensor.id in sensor_ids + # Reset module-scoped fixture state so later tests are not affected. + asset.sensors_to_show = [] + db.session.flush() + def test_asset_sensors_metadata( db, mock_get_statuses, add_weather_sensors, add_battery_assets From 2b53bf4958d0496d8a2933a0b0b90226b2612988 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 18:03:26 +0000 Subject: [PATCH 09/13] Fix test: flush session before reading sensor IDs, reset sensors_to_show after test Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/5ce56f2a-9e55-408b-9659-f12da4d6800a Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --- .../schemas/tests/test_sensor_data_schema.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 6f5cd10b5c..98fc41bcd9 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -367,10 +367,11 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso contained the old format with a singular 'sensor' key, e.g. {"title": "Prices", "sensor": 42}. """ asset = add_weather_sensors["asset"] - wind_sensor, temperature_sensor = ( - add_weather_sensors["wind"], - add_weather_sensors["temperature"], - ) + wind_sensor = add_weather_sensors["wind"] + temperature_sensor = add_weather_sensors["temperature"] + + # Flush so sensor IDs are assigned before we embed them in sensors_to_show + db.session.flush() # Flush to ensure sensors have database IDs before using them. db.session.flush() @@ -383,12 +384,17 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso db.session.add(asset) db.session.flush() - # Should not raise a KeyError - status_data = get_asset_sensors_metadata(asset=asset) - - sensor_ids = [s["id"] for s in status_data] - assert wind_sensor.id in sensor_ids - assert temperature_sensor.id in sensor_ids + try: + # Should not raise a KeyError + status_data = get_asset_sensors_metadata(asset=asset) + + sensor_ids = [s["id"] for s in status_data] + assert wind_sensor.id in sensor_ids + assert temperature_sensor.id in sensor_ids + finally: + # Reset to avoid polluting shared module-scoped fixture state + asset.sensors_to_show = [] + db.session.flush() # Reset module-scoped fixture state so later tests are not affected. asset.sensors_to_show = [] From aaaacda0aaace7f6a6846eccbd5f5c91c646d8e8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 21:49:03 +0200 Subject: [PATCH 10/13] Revert "Fix test: flush session before reading sensor IDs, reset sensors_to_show after test" This reverts commit 2b53bf4958d0496d8a2933a0b0b90226b2612988. Signed-off-by: F.N. Claessen --- .../schemas/tests/test_sensor_data_schema.py | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 98fc41bcd9..6f5cd10b5c 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -367,11 +367,10 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso contained the old format with a singular 'sensor' key, e.g. {"title": "Prices", "sensor": 42}. """ asset = add_weather_sensors["asset"] - wind_sensor = add_weather_sensors["wind"] - temperature_sensor = add_weather_sensors["temperature"] - - # Flush so sensor IDs are assigned before we embed them in sensors_to_show - db.session.flush() + wind_sensor, temperature_sensor = ( + add_weather_sensors["wind"], + add_weather_sensors["temperature"], + ) # Flush to ensure sensors have database IDs before using them. db.session.flush() @@ -384,17 +383,12 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso db.session.add(asset) db.session.flush() - try: - # Should not raise a KeyError - status_data = get_asset_sensors_metadata(asset=asset) - - sensor_ids = [s["id"] for s in status_data] - assert wind_sensor.id in sensor_ids - assert temperature_sensor.id in sensor_ids - finally: - # Reset to avoid polluting shared module-scoped fixture state - asset.sensors_to_show = [] - db.session.flush() + # Should not raise a KeyError + status_data = get_asset_sensors_metadata(asset=asset) + + sensor_ids = [s["id"] for s in status_data] + assert wind_sensor.id in sensor_ids + assert temperature_sensor.id in sensor_ids # Reset module-scoped fixture state so later tests are not affected. asset.sensors_to_show = [] From bb181df586c0e226dac34ceba9a8f37d7f5f41df Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 21:51:34 +0200 Subject: [PATCH 11/13] delete: redundant flushes Signed-off-by: F.N. Claessen --- .../api/common/schemas/tests/test_sensor_data_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 6f5cd10b5c..9804562231 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -381,7 +381,6 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso {"title": "Solo", "sensor": wind_sensor.id}, ] db.session.add(asset) - db.session.flush() # Should not raise a KeyError status_data = get_asset_sensors_metadata(asset=asset) @@ -392,7 +391,6 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso # Reset module-scoped fixture state so later tests are not affected. asset.sensors_to_show = [] - db.session.flush() def test_asset_sensors_metadata( From 043eaac72787f6f7ce2bca9faf9dbcb62cc221fc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 21:52:15 +0200 Subject: [PATCH 12/13] refactor: cleaner looking code Signed-off-by: F.N. Claessen --- .../api/common/schemas/tests/test_sensor_data_schema.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 9804562231..dfda1ba1f9 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -367,10 +367,8 @@ def test_asset_sensors_metadata_old_sensors_to_show_format(db, add_weather_senso contained the old format with a singular 'sensor' key, e.g. {"title": "Prices", "sensor": 42}. """ asset = add_weather_sensors["asset"] - wind_sensor, temperature_sensor = ( - add_weather_sensors["wind"], - add_weather_sensors["temperature"], - ) + wind_sensor = add_weather_sensors["wind"] + temperature_sensor = add_weather_sensors["temperature"] # Flush to ensure sensors have database IDs before using them. db.session.flush() From af58c6d87530d028d2cf40b96e2101831ec5d35c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 21:52:42 +0200 Subject: [PATCH 13/13] docs: remove space Signed-off-by: F.N. Claessen --- .../api/common/schemas/tests/test_sensor_data_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index dfda1ba1f9..52dae05cad 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -395,7 +395,7 @@ def test_asset_sensors_metadata( db, mock_get_statuses, add_weather_sensors, add_battery_assets ): """ - Test the function to build status meta data structure, using a weather station asset. + Test the function to build status metadata structure, using a weather station asset. We include the sensor of a different asset (a battery) via the flex context (as production price, does not make too much sense actually). One sensor which the asset already includes is also set in the context as inflexible device,