From 4b377797b86b8d63ac7c98bdfb6787a9019dc0f5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:04:41 +0000 Subject: [PATCH 01/63] Initial plan From 64f791a17c332ddec0b5a993a170506aedf8616b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:10:38 +0000 Subject: [PATCH 02/63] data_sources: add account_id field to DataSource model - Add account_id column (FK to account.id, nullable) to DataSource - Add account relationship (backref 'data_sources' on Account) - Update __table_args__ UniqueConstraint to include account_id - Set self.account from user.account in __init__ when user is provided --- flexmeasures/data/models/data_sources.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index e0488023f5..8b31cbd57a 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -271,7 +271,9 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): __tablename__ = "data_source" __table_args__ = ( - db.UniqueConstraint("name", "user_id", "model", "version", "attributes_hash"), + db.UniqueConstraint( + "name", "user_id", "account_id", "model", "version", "attributes_hash" + ), ) # The type of data source (e.g. user, forecaster or scheduler) @@ -284,6 +286,10 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): ) user = db.relationship("User", backref=db.backref("data_source", lazy=True)) + # The account this data source belongs to (populated from user.account for user-type sources) + account_id = db.Column(db.Integer, db.ForeignKey("account.id"), nullable=True) + account = db.relationship("Account", backref=db.backref("data_sources", lazy=True)) + attributes = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default={}) attributes_hash = db.Column(db.LargeBinary(length=256)) @@ -316,6 +322,7 @@ def __init__( name = user.username type = "user" self.user = user + self.account = user.account elif user is None and type == "user": raise TypeError("A data source cannot have type 'user' but no user set.") self.type = type From 67459217d7d06f605ce63a4ccc5b3739f83be6a1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:10:44 +0000 Subject: [PATCH 03/63] migration: add account_id to data_source table (9877450113f6) - Add account_id column (nullable, FK to account.id) - Data migration: populate account_id from user's account where user_id IS NOT NULL - Update UniqueConstraint to include account_id column - down_revision: 8b62f8129f34 --- ...7450113f6_add_account_id_to_data_source.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py diff --git a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py new file mode 100644 index 0000000000..719249d210 --- /dev/null +++ b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py @@ -0,0 +1,89 @@ +"""Add account_id to data_source table + +Revision ID: 9877450113f6 +Revises: 8b62f8129f34 +Create Date: 2025-01-15 00:00:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "9877450113f6" +down_revision = "8b62f8129f34" +branch_labels = None +depends_on = None + +# Minimal table definitions for the data migration (SQLAlchemy Core only, no ORM) +t_data_source = sa.Table( + "data_source", + sa.MetaData(), + sa.Column("id", sa.Integer), + sa.Column("user_id", sa.Integer), + sa.Column("account_id", sa.Integer), +) + +t_fm_user = sa.Table( + "fm_user", + sa.MetaData(), + sa.Column("id", sa.Integer), + sa.Column("account_id", sa.Integer), +) + + +def upgrade(): + # 1. Add the account_id column (nullable) + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.add_column(sa.Column("account_id", sa.Integer(), nullable=True)) + batch_op.create_foreign_key( + op.f("data_source_account_id_account_fkey"), + "account", + ["account_id"], + ["id"], + ) + + # 2. Data migration: populate account_id from the related user's account + connection = op.get_bind() + rows = connection.execute( + sa.select(t_data_source.c.id, t_data_source.c.user_id).where( + t_data_source.c.user_id.isnot(None) + ) + ).fetchall() + + for ds_id, user_id in rows: + user_row = connection.execute( + sa.select(t_fm_user.c.account_id).where(t_fm_user.c.id == user_id) + ).fetchone() + if user_row is not None and user_row[0] is not None: + connection.execute( + sa.update(t_data_source) + .where(t_data_source.c.id == ds_id) + .values(account_id=user_row[0]) + ) + + # 3. Drop old UniqueConstraint and recreate it with account_id included + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.drop_constraint("data_source_name_key", type_="unique") + batch_op.create_unique_constraint( + "data_source_name_key", + ["name", "user_id", "account_id", "model", "version", "attributes_hash"], + ) + + +def downgrade(): + # 1. Restore the original UniqueConstraint without account_id + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.drop_constraint("data_source_name_key", type_="unique") + batch_op.create_unique_constraint( + "data_source_name_key", + ["name", "user_id", "model", "version", "attributes_hash"], + ) + + # 2. Drop the account_id column and its FK + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.drop_constraint( + op.f("data_source_account_id_account_fkey"), type_="foreignkey" + ) + batch_op.drop_column("account_id") From 78fb04f73584034d57450619ed35bd8a4120d15b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:11:38 +0000 Subject: [PATCH 04/63] migration: replace N+1 loop with single UPDATE...FROM for account_id backfill --- ...7450113f6_add_account_id_to_data_source.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py index 719249d210..ab156e4605 100644 --- a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py +++ b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py @@ -44,24 +44,14 @@ def upgrade(): ["id"], ) - # 2. Data migration: populate account_id from the related user's account + # 2. Data migration: populate account_id from the related user's account. + # Use a single UPDATE ... FROM to avoid N+1 queries. connection = op.get_bind() - rows = connection.execute( - sa.select(t_data_source.c.id, t_data_source.c.user_id).where( - t_data_source.c.user_id.isnot(None) - ) - ).fetchall() - - for ds_id, user_id in rows: - user_row = connection.execute( - sa.select(t_fm_user.c.account_id).where(t_fm_user.c.id == user_id) - ).fetchone() - if user_row is not None and user_row[0] is not None: - connection.execute( - sa.update(t_data_source) - .where(t_data_source.c.id == ds_id) - .values(account_id=user_row[0]) - ) + connection.execute( + sa.update(t_data_source) + .where(t_data_source.c.user_id == t_fm_user.c.id) + .values(account_id=t_fm_user.c.account_id) + ) # 3. Drop old UniqueConstraint and recreate it with account_id included with op.batch_alter_table("data_source", schema=None) as batch_op: From 65efbdd01b242ac41ff412d6c92f8dd6ece3fc80 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:14:16 +0000 Subject: [PATCH 05/63] tests/sensor_data: verify account_id is set on data source when posting sensor data Context: - DataSource.__init__ now sets self.account = user.account when a user is provided - The POST sensor data endpoint calls get_or_create_source(current_user) - Need to verify account_id is populated correctly on the resulting data source Change: - Added test_post_sensor_data_sets_account_id_on_data_source - Posts sensor data as test_supplier_user_4@seita.nl - Asserts data_source.account_id == user.account_id --- .../v3_0/tests/test_sensor_data_fresh_db.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index 3912bc0cca..e4b68ef05c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -9,6 +9,7 @@ from flexmeasures import Source from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor from flexmeasures.data.models.time_series import TimedBelief +from flexmeasures.data.models.user import User @pytest.mark.parametrize( @@ -162,3 +163,35 @@ def test_post_sensor_instantaneous_data_round( assert data.reset_index().event_start[0] == pd.Timestamp( "2021-06-06 22:00:00+0000", tz="UTC" ) + + +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_post_sensor_data_sets_account_id_on_data_source( + client, + setup_api_fresh_test_data, + requesting_user, + db, +): + """When sensor data is posted, the resulting data source should have account_id + set to the posting user's account_id. + """ + sensor = setup_api_fresh_test_data["some gas sensor"] + post_data = make_sensor_data_request_for_gas_sensor( + num_values=6, unit="m³/h", include_a_null=False + ) + response = client.post( + url_for("SensorAPI:post_data", id=sensor.id), + json=post_data, + ) + assert response.status_code == 200 + + user = db.session.execute( + select(User).filter_by(email="test_supplier_user_4@seita.nl") + ).scalar_one() + data_source = db.session.execute( + select(Source).filter_by(user=user) + ).scalar_one_or_none() + assert data_source is not None + assert data_source.account_id == user.account_id From 85778e47cfba807bf27605b38e1cddb56c9bc77f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:17:40 +0000 Subject: [PATCH 06/63] migration: use correlated subquery for account_id data migration; add changelog entry Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/7969297e-88c2-4ace-99c9-ec74109f03c8 --- documentation/changelog.rst | 1 + .../9877450113f6_add_account_id_to_data_source.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index ccd08e4861..5e1be218ea 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -17,6 +17,7 @@ New features Infrastructure / Support ---------------------- +* Add ``account_id`` field to the ``data_source`` table, populated from the related user's account (including data migration and automatic assignment when posting sensor data) [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management [see `PR #1973 `_] * Migrate from ``make`` to ``poe`` for running tasks [see `PR #1973 `_] diff --git a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py index ab156e4605..aa460b2ff8 100644 --- a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py +++ b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py @@ -45,12 +45,16 @@ def upgrade(): ) # 2. Data migration: populate account_id from the related user's account. - # Use a single UPDATE ... FROM to avoid N+1 queries. + # Use a correlated subquery to avoid N+1 queries and ensure portability. connection = op.get_bind() connection.execute( sa.update(t_data_source) - .where(t_data_source.c.user_id == t_fm_user.c.id) - .values(account_id=t_fm_user.c.account_id) + .values( + account_id=sa.select(t_fm_user.c.account_id) + .where(t_fm_user.c.id == t_data_source.c.user_id) + .scalar_subquery() + ) + .where(t_data_source.c.user_id.isnot(None)) ) # 3. Drop old UniqueConstraint and recreate it with account_id included From 06bbec9f480870ea6f690c7afbc2afb975e13500 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:25:04 +0000 Subject: [PATCH 07/63] data_sources: fix account_id assignment to avoid SAWarning and handle unflushed users Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/7969297e-88c2-4ace-99c9-ec74109f03c8 --- flexmeasures/data/models/data_sources.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 8b31cbd57a..6a775332b2 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -322,7 +322,12 @@ def __init__( name = user.username type = "user" self.user = user - self.account = user.account + # Prefer the FK column directly (avoids triggering a lazy load/autoflush). + # Fall back to the account relationship for users not yet flushed to the DB + # (where account_id may not be set on the column yet). + self.account_id = user.account_id or ( + user.account.id if user.account else None + ) elif user is None and type == "user": raise TypeError("A data source cannot have type 'user' but no user set.") self.type = type From 8b020340dae9b3dd47421620255d190951f3be88 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:29:01 +0000 Subject: [PATCH 08/63] agents/architecture: add migration checklist and DataSource account_id invariants Context: - PR #2058 added account_id FK to DataSource table - No migration checklist existed for reviewing/writing Alembic migrations - New domain invariant (user-type DataSources now have account_id) was undocumented - Agent failed to self-update during the PR session Change: - Added Alembic migration checklist with safe backfill pattern - Added DataSource entity to Domain Knowledge with account_id invariant - Documented invariants #5 and #6 for DataSource UniqueConstraint and account_id - Added Lessons Learned section with PR #2058 case study --- .../agents/architecture-domain-specialist.md | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/.github/agents/architecture-domain-specialist.md b/.github/agents/architecture-domain-specialist.md index d295024663..7454a15de0 100644 --- a/.github/agents/architecture-domain-specialist.md +++ b/.github/agents/architecture-domain-specialist.md @@ -38,9 +38,43 @@ This agent owns the integrity of models (e.g. assets, sensors, data sources, sch - [ ] **Acyclic asset trees**: Verify no changes break the `parent_asset_id != id` constraint - [ ] **Asset hierarchy**: Ensure parent-child relationships maintain referential integrity - [ ] **Account ownership**: Check that all assets have `account_id` set correctly +- [ ] **DataSource account_id**: Verify user-type DataSources have `account_id` populated from `user.account_id` (see invariant below) - [ ] **Sensor-Asset binding**: Validate sensors are properly linked to assets - [ ] **TimedBelief structure**: Ensure (event_start, belief_time, source, cumulative_probability) integrity +### Alembic Migration Changes + +When reviewing or writing Alembic migrations, check: + +- [ ] **down_revision correct**: Verify `down_revision` matches the actual current head (run `flask db heads` to check) +- [ ] **FK naming convention**: FK constraints follow the pattern `___fkey` +- [ ] **Data backfill**: Bulk UPDATE used (correlated subquery or UPDATE...FROM), not N+1 Python loop +- [ ] **Downgrade path**: `downgrade()` fully reverses `upgrade()` including constraint drops/recreates +- [ ] **No ORM in migrations**: Use `sa.Table` + `sa.MetaData()` with SQLAlchemy Core only; never import ORM models +- [ ] **Nullable new columns**: New FK columns should be `nullable=True` to avoid breaking existing rows +- [ ] **batch_alter_table**: Use `op.batch_alter_table()` for all ALTER TABLE operations (required for SQLite compat in test environments) +- [ ] **UniqueConstraint names**: Verify constraint name matches the existing DB constraint name exactly (check with `\d
` in psql) + +**Pattern: Safe data migration in Alembic** + +```python +# ✅ Correct: SQLAlchemy Core table stubs, bulk correlated subquery update +t_data_source = sa.Table("data_source", sa.MetaData(), sa.Column("id", sa.Integer), ...) +t_fm_user = sa.Table("fm_user", sa.MetaData(), sa.Column("id", sa.Integer), ...) + +connection.execute( + sa.update(t_data_source) + .values(account_id=sa.select(t_fm_user.c.account_id) + .where(t_fm_user.c.id == t_data_source.c.user_id) + .scalar_subquery()) + .where(t_data_source.c.user_id.isnot(None)) +) + +# ❌ Wrong: N+1 Python loop over ORM objects +for source in DataSource.query.filter_by(...): # Don't import ORM! + source.account_id = source.user.account_id # Triggers N queries +``` + ### Flex-context & flex-model - [ ] **Flex-context inheritance**: Verify `get_flex_context()` parent walk logic is preserved @@ -285,6 +319,16 @@ fields_to_remove = ["as_job"] # ❌ Wrong format - **Location**: `/flexmeasures/data/models/data_sources.py` - **Purpose**: Forecasters and reporters subclass `DataGenerator` to couple configured instances to unique data sources (schedulers are not yet subclassing `DataGenerator`) +#### DataSource +- **Location**: `flexmeasures/data/models/data_sources.py` +- **Key fields**: `id`, `name`, `type`, `user_id`, `account_id`, `model`, `version`, `attributes`, `attributes_hash` +- **Relationships**: + - `user` → User (via `user_id`, nullable — only for `type="user"` sources) + - `account` → Account (via `account_id`, nullable — populated from user's account for user-type sources) + - `sensors` ↔ Sensor (via `timed_belief` join table, viewonly) +- **`account_id` invariant**: For user-type DataSources, `account_id` is always set from the creating user's account (see invariant #6 below). For non-user sources (forecasters, schedulers, reporters), `account_id` remains `None`. +- **UniqueConstraint**: `(name, user_id, account_id, model, version, attributes_hash)` — added `account_id` in migration `9877450113f6` + ### Critical Invariants 1. **Acyclic Asset Trees** @@ -308,7 +352,18 @@ fields_to_remove = ["as_job"] # ❌ Wrong format - Role-based permissions (ACCOUNT_ADMIN, CONSULTANT) - Audit logging for all mutations -5. **Timezone Awareness** +5. **DataSource UniqueConstraint includes `account_id`** + - Since migration `9877450113f6` (PR #2058), `account_id` is part of the unique constraint + - Code that creates or deduplicates DataSources must include `account_id` in lookup logic + - See `get_or_create_source()` in `flexmeasures/data/services/data_sources.py` + +6. **User-type DataSources always have `account_id` populated** + - Invariant added in PR #2058: when a `DataSource` is created with `user=`, `account_id` is set from `user.account_id` + - Fallback for unflushed users: `user.account.id` is used if `user.account_id` is not yet set + - Non-user DataSources (forecasters, schedulers, reporters created without a user) have `account_id=None` + - Implication: filtering DataSources by `account_id` only works for user-type sources + +7. **Timezone Awareness** - All datetime objects MUST be timezone-aware - Sensors have explicit `timezone` field @@ -487,3 +542,12 @@ After each assignment: Change: - Added guidance on ``` + +### Lessons Learned + +**Session 2026-03-24 (PR #2058 — add account_id to DataSource)**: + +- **New domain invariant**: User-type DataSources now have `account_id` populated. Document new FK relationships and invariants in the Domain Knowledge section immediately. +- **Migration checklist**: Added an explicit Alembic migration checklist after reviewing the migration for this PR. Key patterns: correlated subquery for bulk backfill, SQLAlchemy Core stubs (no ORM imports), `batch_alter_table` for all ALTER operations, exact constraint name matching. +- **Missed API Specialist coordination**: The PR changed endpoint behavior (POST sensor data sets account_id on the created data source). The API Specialist should have been engaged to verify backward compatibility. When domain model changes affect how endpoints behave or what they return, flag for API Specialist review. +- **Self-improvement failure**: Despite having explicit self-improvement requirements, no agent updated its instructions during this PR session. This was caught by the Coordinator post-hoc. The agent must update its own instructions as the LAST step of every assignment, not skip it. From d7f7120e2969c2be994b1d67d81f29073fe72729 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:29:47 +0000 Subject: [PATCH 09/63] agents/test-specialist: add DataSource property testing pattern and lessons Context: - PR #2058 added a test for account_id on DataSource after API POST - No guidance existed for testing data source properties after API calls - Agent failed to self-update during the PR session Change: - Added 'Testing DataSource Properties After API Calls' section with pattern - Added Lessons Learned section documenting PR #2058 self-improvement failure --- .github/agents/test-specialist.md | 39 +++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/agents/test-specialist.md b/.github/agents/test-specialist.md index 390778918f..3c1cb28e41 100644 --- a/.github/agents/test-specialist.md +++ b/.github/agents/test-specialist.md @@ -302,6 +302,30 @@ flask db current - **Assertions**: Use descriptive assertion messages for failures - **Mocking**: Use pytest fixtures and mocking when testing external dependencies +### Testing DataSource Properties After API Calls + +When writing tests that verify data source properties (e.g. `account_id`, `user`, `type`) after an API call: + +1. **Use `fresh_db` fixture** — tests that POST data and then query the resulting data source are modifying the DB and must use the function-scoped `fresh_db` fixture. Place these tests in a `_fresh_db` module. + +2. **Query by user, not just name** — data sources created by the same user across test runs may collide; use `filter_by(user=user)` or `filter_by(user_id=user.id)` for precision. + +3. **Pattern** (from `test_post_sensor_data_sets_account_id_on_data_source`): + ```python + # Fetch the user that made the request + user = db.session.execute( + select(User).filter_by(email="test_supplier_user_4@seita.nl") + ).scalar_one() + # Fetch the data source created for that user + data_source = db.session.execute( + select(Source).filter_by(user=user) + ).scalar_one_or_none() + assert data_source is not None + assert data_source.account_id == user.account_id + ``` + +4. **Check both existence and value** — don't just assert `data_source is not None`; also assert the specific field value you're testing. + ## Understanding Test Design Intent (CRITICAL) **Before changing a test, understand WHY it's designed that way.** @@ -482,14 +506,9 @@ After each assignment: - Added guidance on ``` -Example: +### Lessons Learned -``` -agents/test-specialist: learned to verify claims with actual test runs -Context: -- Session #456 claimed tests passed but they were never actually run -- Led to bug slipping through to production -Change: -- Added "Actually Run Tests" section with verification steps -- Emphasized checking test output and coverage -``` \ No newline at end of file +**Session 2026-03-24 (PR #2058 — add account_id to DataSource)**: + +- **Self-improvement failure**: Despite having explicit instructions to update this agent file after each assignment, no update was made during this PR session. This was caught by the Coordinator post-hoc. The agent must treat instruction updates as the LAST mandatory step of any assignment. +- **DataSource property testing**: Added guidance in "Testing DataSource Properties After API Calls" above. When testing properties set by the API on a data source (like `account_id`), use `fresh_db`, query by user to avoid ambiguity, and assert both existence and the specific field value. \ No newline at end of file From 20a4e156e1ef28e35d78b69d5420de4cbcead113 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:30:24 +0000 Subject: [PATCH 10/63] agents/review-lead: add agent selection checklist; document repeated Coordinator failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context: - PR #2058 session: Coordinator was not invoked (3rd session in a row) - API Specialist was not engaged despite endpoint behavior change - No agents updated their instructions (same failure as 2026-02-06) Change: - Added 'Agent Selection Checklist' mapping code change types to required agents - Documented PR #2058 as lessons learned (3 distinct failures) - Reinforced that Coordinator must ALWAYS be last agent in every session - Clarified that endpoint behavior changes → API Specialist must be engaged --- .github/agents/review-lead.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/agents/review-lead.md b/.github/agents/review-lead.md index 24f4e455b2..96f2b5d413 100644 --- a/.github/agents/review-lead.md +++ b/.github/agents/review-lead.md @@ -631,8 +631,40 @@ Track and document when the Review Lead: - **Prevention**: Investigate production code first; understand test design intent; look for schema migrations - **Key insight**: "Failing tests often reveal production bugs, not test bugs" +**Specific lesson learned (2026-03-24 — PR #2058, add account_id to DataSource)**: + +- **Failure 1**: Coordinator was not invoked despite (a) explicit user prompt instruction and (b) the "MUST always run the Coordinator" requirement in these instructions. This is now the **third session in a row** where Coordinator invocation was skipped. The requirement is clearly not being followed. + - **Root cause hypothesis**: The Coordinator step is listed last, but sessions end before it's reached. It needs to be treated as a non-negotiable gate, not an optional final step. + - **Reinforcement**: Added "Agent Selection Checklist" below to force explicit reasoning about which agents to run. + +- **Failure 2**: API Specialist was not engaged for a change that modified endpoint behavior. The POST sensor data endpoint now sets `account_id` on the resulting data source. This is a side-effect change that the API Specialist should have reviewed. + - **Rule added**: Any PR that changes what data is stored or returned by an endpoint → engage API Specialist. + +- **Failure 3**: No agent (including Review Lead) updated their own instructions after the session. This is the same failure pattern from 2026-02-06. Three sessions now. This is a systemic problem. + - **Root cause**: Self-improvement is treated as optional/forgettable. It must be the **last commit** of every session, non-negotiable. + Update this file to prevent repeating the same mistakes. +### Agent Selection Checklist + +Use this checklist when selecting agents for a PR. Check each category: + +| Code Change Type | Agent to Engage | +|------------------|----------------| +| Domain model changes (`models/`) | Architecture Specialist | +| Alembic migrations | Architecture Specialist | +| Test additions/changes | Test Specialist | +| Endpoint behavior changes (stored/returned data) | **API Specialist** | +| API schema changes | API Specialist | +| CLI command changes | API Specialist | +| Performance-critical paths | Performance Specialist | +| Time/datetime/timezone code | Data & Time Specialist | +| Documentation changes | Documentation Specialist | +| CI/CD, pre-commit, GitHub Actions | Tooling & CI Specialist | +| Agent instruction changes | **Coordinator** (always at end) | + +**The Coordinator must ALWAYS be the last agent run, in every session, no exceptions.** + ### Continuous Improvement The Review Lead should: From fff71b20e7c1af609d026825c1bde88a2622b531 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:31:01 +0000 Subject: [PATCH 11/63] agents/coordinator: document PR #2058 persistent self-improvement failure pattern Context: - PR #2058 repeated the same 3 failures from sessions 2026-02-06 and 2026-02-08 - Coordinator not invoked, no agent self-updates, API Specialist not engaged - Same failures now documented 3 times in Review Lead instructions with no change in behavior Change: - Added 'Additional Pattern Discovered (2026-03-24)' with root cause analysis - Documented missed API Specialist engagement for endpoint behavior change - Flagged governance escalation if Coordinator invocation fails a 4th time - Noted code observation: or-pattern for account_id and empty Initial plan commit --- .github/agents/coordinator.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/agents/coordinator.md b/.github/agents/coordinator.md index 334aee19eb..bfb178d9dc 100644 --- a/.github/agents/coordinator.md +++ b/.github/agents/coordinator.md @@ -448,3 +448,35 @@ The Coordinator has identified these recurring issues: Review Lead should now invoke Coordinator as subagent. These patterns must not repeat. Agent instructions have been updated to prevent recurrence. + +### Additional Pattern Discovered (2026-03-24) + +**Pattern**: Persistent self-improvement failure and missing API Specialist agent selection + +**Session**: PR #2058 — Add `account_id` to DataSource table + +**Observation**: After three sessions now, the same two failures recur: +1. Coordinator is not invoked at end of session (despite MUST requirement in Review Lead instructions) +2. No agent updates its own instructions (despite MUST requirement in all agents) + +**Root cause analysis**: +- "Coordinator invocation" and "self-improvement" are both documented as mandatory last steps +- But the session ends before they are reached — they are treated as optional epilogue, not gating requirements +- The Review Lead agent selection is ad-hoc, with no explicit checklist forcing API Specialist engagement when endpoints change + +**What was missed in PR #2058**: +- API Specialist not engaged: POST sensor data now sets `account_id` on the resulting data source — this is an endpoint behavior change that should be reviewed for backward compatibility +- Zero agent instruction updates across all three participating agents (Architecture Specialist, Test Specialist, Review Lead) +- No Coordinator invocation despite explicit user request in the original prompt + +**Solutions implemented**: +- Architecture Specialist: Added Alembic migration checklist + DataSource domain invariants +- Test Specialist: Added DataSource property testing pattern + lessons learned +- Review Lead: Added Agent Selection Checklist mapping code change types to required agents; documented 3rd recurrence of these failures +- Coordinator (this file): Documented case study + +**Governance escalation**: The Review Lead's "Must Always Run Coordinator" requirement has now been documented in three sessions without being followed. If it fails a fourth time, consider structural changes — e.g., making Coordinator invocation the FIRST step of a session rather than the last, so it sets context rather than being a forgotten epilogue. + +**Code observation from PR #2058 worth tracking**: +- `user.account_id or (user.account.id if user.account else None)` — the `or` pattern is fragile for `account_id=0` (unrealistic but worth noting). Prefer `user.account_id if user.account_id is not None else (user.account.id if user.account else None)` for strict correctness. +- Empty "Initial plan" commit adds git history noise. When orchestrating agents, the first commit should be functional code, not a planning marker. From 4191ae3582d0d6d12780c5051a95918853eaaddf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 22:32:27 +0000 Subject: [PATCH 12/63] data_sources: use explicit is not None check for account_id assignment Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/7969297e-88c2-4ace-99c9-ec74109f03c8 --- flexmeasures/data/models/data_sources.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index 6a775332b2..bee911a13b 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -325,9 +325,10 @@ def __init__( # Prefer the FK column directly (avoids triggering a lazy load/autoflush). # Fall back to the account relationship for users not yet flushed to the DB # (where account_id may not be set on the column yet). - self.account_id = user.account_id or ( - user.account.id if user.account else None - ) + if user.account_id is not None: + self.account_id = user.account_id + elif user.account is not None: + self.account_id = user.account.id elif user is None and type == "user": raise TypeError("A data source cannot have type 'user' but no user set.") self.type = type From 38db656d8737577392375d0fb5d0869b85743c71 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 25 Mar 2026 00:25:10 +0100 Subject: [PATCH 13/63] feat: improve test by using a user that is not already registered as a data source before POSTing sensor data Signed-off-by: F.N. Claessen --- .../v3_0/tests/test_sensor_data_fresh_db.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index e4b68ef05c..16ab046d11 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -165,12 +165,11 @@ def test_post_sensor_instantaneous_data_round( ) -@pytest.mark.parametrize( - "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True -) +@pytest.mark.parametrize("requesting_user", ["improper_user@seita.nl"], indirect=True) def test_post_sensor_data_sets_account_id_on_data_source( client, setup_api_fresh_test_data, + setup_user_without_data_source, requesting_user, db, ): @@ -181,17 +180,22 @@ def test_post_sensor_data_sets_account_id_on_data_source( post_data = make_sensor_data_request_for_gas_sensor( num_values=6, unit="m³/h", include_a_null=False ) + + # Make sure the user is not yet registered as a data source + data_source = db.session.execute( + select(Source).filter_by(user=setup_user_without_data_source) + ).scalar_one_or_none() + assert data_source is None + response = client.post( url_for("SensorAPI:post_data", id=sensor.id), json=post_data, ) assert response.status_code == 200 - user = db.session.execute( - select(User).filter_by(email="test_supplier_user_4@seita.nl") - ).scalar_one() + # Make sure the user is now registered as a data source with account_id set data_source = db.session.execute( - select(Source).filter_by(user=user) + select(Source).filter_by(user=setup_user_without_data_source) ).scalar_one_or_none() assert data_source is not None - assert data_source.account_id == user.account_id + assert data_source.account_id == setup_user_without_data_source.account_id From 67eba6ee6b333bf74773e34a84fff09b331ff994 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 25 Mar 2026 00:30:39 +0100 Subject: [PATCH 14/63] fix: set approximate create datetime based on git info, assuming local time (UTC+1) Signed-off-by: F.N. Claessen --- .../versions/9877450113f6_add_account_id_to_data_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py index aa460b2ff8..dde9f1c0bf 100644 --- a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py +++ b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py @@ -2,7 +2,7 @@ Revision ID: 9877450113f6 Revises: 8b62f8129f34 -Create Date: 2025-01-15 00:00:00.000000 +Create Date: 2026-03-24 22:10:00.000000 """ From b7a4a5c13760e858c5e61f78308e4e4ffd070a05 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 25 Mar 2026 00:32:17 +0100 Subject: [PATCH 15/63] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index 16ab046d11..f297e97ef6 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -9,7 +9,6 @@ from flexmeasures import Source from flexmeasures.api.v3_0.tests.utils import make_sensor_data_request_for_gas_sensor from flexmeasures.data.models.time_series import TimedBelief -from flexmeasures.data.models.user import User @pytest.mark.parametrize( From 44cff24ab856563d05dd5dbd781c869b55aee871 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 25 Mar 2026 00:35:52 +0100 Subject: [PATCH 16/63] feat: add migration check to architecture agent instructions Signed-off-by: F.N. Claessen --- .github/agents/architecture-domain-specialist.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/agents/architecture-domain-specialist.md b/.github/agents/architecture-domain-specialist.md index 7454a15de0..19aedcc553 100644 --- a/.github/agents/architecture-domain-specialist.md +++ b/.github/agents/architecture-domain-specialist.md @@ -54,6 +54,7 @@ When reviewing or writing Alembic migrations, check: - [ ] **Nullable new columns**: New FK columns should be `nullable=True` to avoid breaking existing rows - [ ] **batch_alter_table**: Use `op.batch_alter_table()` for all ALTER TABLE operations (required for SQLite compat in test environments) - [ ] **UniqueConstraint names**: Verify constraint name matches the existing DB constraint name exactly (check with `\d
` in psql) +- [ ] **Create Date**: Verify the current datetime is correct (e.g. a `Create Date: 2025-01-15 00:00:00.000000` is highly unlikely to be true) **Pattern: Safe data migration in Alembic** From 95653f1ea68f5c43aa6c00d23b364e5d7ad6e6bc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 25 Mar 2026 00:39:15 +0100 Subject: [PATCH 17/63] docs: add db upgrade warning and update agent instructions accordingly Signed-off-by: F.N. Claessen --- .github/agents/architecture-domain-specialist.md | 1 + documentation/changelog.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/agents/architecture-domain-specialist.md b/.github/agents/architecture-domain-specialist.md index 19aedcc553..cf7042cf98 100644 --- a/.github/agents/architecture-domain-specialist.md +++ b/.github/agents/architecture-domain-specialist.md @@ -55,6 +55,7 @@ When reviewing or writing Alembic migrations, check: - [ ] **batch_alter_table**: Use `op.batch_alter_table()` for all ALTER TABLE operations (required for SQLite compat in test environments) - [ ] **UniqueConstraint names**: Verify constraint name matches the existing DB constraint name exactly (check with `\d
` in psql) - [ ] **Create Date**: Verify the current datetime is correct (e.g. a `Create Date: 2025-01-15 00:00:00.000000` is highly unlikely to be true) +- [ ] **Changelog**: Add a db upgrade warning in the changelog under the relevant release section (see previous sections for styling) **Pattern: Safe data migration in Alembic** diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 5e1be218ea..b367ffc582 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,8 @@ FlexMeasures Changelog v0.32.0 | April XX, 2026 ============================ +.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). + New features ------------- * Support fetching a schedule in a different unit still compatible to the sensor unit [see `PR #1993 `_] From 4e22e439c310af33637a0c6e201b9f69340ba748 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 03:09:34 +0000 Subject: [PATCH 18/63] data_sources: drop FK constraints on user_id and account_id for data lineage preservation Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/004f0bb9-133d-45c1-9e06-2fedbf25b258 --- flexmeasures/cli/tests/test_data_delete.py | 27 +++++++++++- ...f6_drop_fk_constraints_from_data_source.py | 44 +++++++++++++++++++ flexmeasures/data/models/data_sources.py | 26 ++++++++--- flexmeasures/data/tests/test_user_services.py | 17 +++++++ 4 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py diff --git a/flexmeasures/cli/tests/test_data_delete.py b/flexmeasures/cli/tests/test_data_delete.py index 44ace78500..e73640100d 100644 --- a/flexmeasures/cli/tests/test_data_delete.py +++ b/flexmeasures/cli/tests/test_data_delete.py @@ -9,14 +9,28 @@ def test_delete_account( fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db, app ): - """Check account is deleted + old audit log entries get affected_account_id set to None.""" + """Check account is deleted + old audit log entries get affected_account_id set to None. + Also check that data source lineage is preserved: account_id is NOT nullified after account deletion. + """ from flexmeasures.cli.data_delete import delete_account + from flexmeasures.data.models.data_sources import DataSource prosumer: User = find_user_by_email("test_prosumer_user@seita.nl") prosumer_account_id = prosumer.account_id num_accounts = fresh_db.session.scalar(select(func.count()).select_from(Account)) + # Find data sources belonging to the account's users (for lineage check after deletion) + data_sources_before = fresh_db.session.scalars( + select(DataSource).filter_by(account_id=prosumer_account_id) + ).all() + assert ( + len(data_sources_before) > 0 + ), "Data sources linked to the account should exist before deletion." + data_source_ids_and_account_ids = [ + (ds.id, ds.account_id) for ds in data_sources_before + ] + # Add creation audit log record user_creation_audit_log = AuditLog( event="User Test Prosumer User created test", @@ -47,3 +61,14 @@ def test_delete_account( .one_or_none() ) assert user_creation_audit_log.affected_account_id is None + + # Check that data source lineage is preserved: account_id is NOT nullified after account deletion + for ds_id, original_account_id in data_source_ids_and_account_ids: + data_source = fresh_db.session.get(DataSource, ds_id) + assert ( + data_source is not None + ), f"Data source {ds_id} should still exist after account deletion." + assert data_source.account_id == original_account_id, ( + f"Data source {ds_id} account_id should be preserved (not nullified) " + "after account deletion for lineage purposes." + ) diff --git a/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py b/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py new file mode 100644 index 0000000000..35bbb45acc --- /dev/null +++ b/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py @@ -0,0 +1,44 @@ +"""Drop FK constraints from data_source for data lineage preservation + +When users or accounts are deleted, we want to preserve the historical +user_id and account_id values in data_source rows for lineage purposes, +rather than cascade-deleting or nullifying them. + +Revision ID: a1b2c3d4e5f6 +Revises: 9877450113f6 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "a1b2c3d4e5f6" +down_revision = "9877450113f6" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.drop_constraint( + op.f("data_source_account_id_account_fkey"), type_="foreignkey" + ) + batch_op.drop_constraint("data_sources_user_id_fkey", type_="foreignkey") + + +def downgrade(): + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.create_foreign_key( + "data_sources_user_id_fkey", + "fm_user", + ["user_id"], + ["id"], + ) + batch_op.create_foreign_key( + op.f("data_source_account_id_account_fkey"), + "account", + ["account_id"], + ["id"], + ) diff --git a/flexmeasures/data/models/data_sources.py b/flexmeasures/data/models/data_sources.py index bee911a13b..32fbccaa9e 100644 --- a/flexmeasures/data/models/data_sources.py +++ b/flexmeasures/data/models/data_sources.py @@ -280,15 +280,27 @@ class DataSource(db.Model, tb.BeliefSourceDBMixin): # just a string, but preferably one of DEFAULT_DATASOURCE_TYPES type = db.Column(db.String(80), default="") - # The id of the user source (can link e.g. to fm_user table) - user_id = db.Column( - db.Integer, db.ForeignKey("fm_user.id"), nullable=True, unique=True + # The id of the user source (can link e.g. to fm_user table). + # No DB-level FK so that deleting a user preserves the lineage reference in this column. + user_id = db.Column(db.Integer, nullable=True, unique=True) + user = db.relationship( + "User", + primaryjoin="DataSource.user_id == User.id", + foreign_keys="[DataSource.user_id]", + backref=db.backref("data_source", lazy=True, passive_deletes="all"), + passive_deletes="all", ) - user = db.relationship("User", backref=db.backref("data_source", lazy=True)) - # The account this data source belongs to (populated from user.account for user-type sources) - account_id = db.Column(db.Integer, db.ForeignKey("account.id"), nullable=True) - account = db.relationship("Account", backref=db.backref("data_sources", lazy=True)) + # The account this data source belongs to (populated from user.account for user-type sources). + # No DB-level FK so that deleting an account preserves the lineage reference in this column. + account_id = db.Column(db.Integer, nullable=True) + account = db.relationship( + "Account", + primaryjoin="DataSource.account_id == Account.id", + foreign_keys="[DataSource.account_id]", + backref=db.backref("data_sources", lazy=True, passive_deletes="all"), + passive_deletes="all", + ) attributes = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default={}) diff --git a/flexmeasures/data/tests/test_user_services.py b/flexmeasures/data/tests/test_user_services.py index eb08734b5f..ef4f22f7b6 100644 --- a/flexmeasures/data/tests/test_user_services.py +++ b/flexmeasures/data/tests/test_user_services.py @@ -133,11 +133,21 @@ def test_create_invalid_user( def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db, app): """Check that deleting a user does not lead to deleting their organisation's (asset/sensor/beliefs) data. Also check that an audit log entry is created + old audit log entries get affected_user_id set to None. + Also check that the data source's user_id is preserved (not nullified) after user deletion for lineage. """ prosumer: User = find_user_by_email("test_prosumer_user@seita.nl") + prosumer_id = prosumer.id prosumer_account_id = prosumer.account_id num_users_before = fresh_db.session.scalar(select(func.count(User.id))) + # Find the data source linked to the prosumer user (for lineage check after deletion) + data_source_before = fresh_db.session.execute( + select(DataSource).filter_by(user_id=prosumer_id) + ).scalar_one_or_none() + assert ( + data_source_before is not None + ), "A data source linked to the prosumer user should exist before deletion." + # Find assets belonging to the user's organisation asset_query = select(GenericAsset).filter_by(account_id=prosumer_account_id) assets_before = fresh_db.session.scalars(asset_query).all() @@ -190,3 +200,10 @@ def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db fresh_db.session.refresh(user_creation_audit_log) assert user_creation_audit_log.affected_user_id is None + + # Check that data source lineage is preserved: user_id is NOT nullified after user deletion + fresh_db.session.expire(data_source_before) + fresh_db.session.refresh(data_source_before) + assert ( + data_source_before.user_id == prosumer_id + ), "Data source user_id should be preserved (not nullified) after user deletion for lineage purposes." From 5c29f56259c438558f3a9bfb5161691f017968c0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 03:20:22 +0000 Subject: [PATCH 19/63] tests/data-lineage: also assert account_id and user_id preservation in delete tests Context: - PR #2058 dropped FK constraints on DataSource.user_id and account_id to preserve lineage when users/accounts are deleted - The new tests only checked one of the two lineage fields per test Change: - test_delete_user: also assert data_source.account_id is preserved after user deletion (user deletion should not nullify account_id) - test_delete_account: also save and assert data_source.user_id per data source; account deletion internally calls delete_user so both user_id and account_id lineage must survive the full deletion chain --- flexmeasures/cli/tests/test_data_delete.py | 13 +++++++++---- flexmeasures/data/tests/test_user_services.py | 9 ++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/flexmeasures/cli/tests/test_data_delete.py b/flexmeasures/cli/tests/test_data_delete.py index e73640100d..daa3cc315b 100644 --- a/flexmeasures/cli/tests/test_data_delete.py +++ b/flexmeasures/cli/tests/test_data_delete.py @@ -27,8 +27,8 @@ def test_delete_account( assert ( len(data_sources_before) > 0 ), "Data sources linked to the account should exist before deletion." - data_source_ids_and_account_ids = [ - (ds.id, ds.account_id) for ds in data_sources_before + data_source_ids_and_lineage = [ + (ds.id, ds.user_id, ds.account_id) for ds in data_sources_before ] # Add creation audit log record @@ -62,8 +62,8 @@ def test_delete_account( ) assert user_creation_audit_log.affected_account_id is None - # Check that data source lineage is preserved: account_id is NOT nullified after account deletion - for ds_id, original_account_id in data_source_ids_and_account_ids: + # Check that data source lineage is preserved: account_id and user_id are NOT nullified after account deletion + for ds_id, original_user_id, original_account_id in data_source_ids_and_lineage: data_source = fresh_db.session.get(DataSource, ds_id) assert ( data_source is not None @@ -72,3 +72,8 @@ def test_delete_account( f"Data source {ds_id} account_id should be preserved (not nullified) " "after account deletion for lineage purposes." ) + if original_user_id is not None: + assert data_source.user_id == original_user_id, ( + f"Data source {ds_id} user_id should be preserved (not nullified) " + "after account deletion for lineage purposes." + ) diff --git a/flexmeasures/data/tests/test_user_services.py b/flexmeasures/data/tests/test_user_services.py index ef4f22f7b6..5e431681ed 100644 --- a/flexmeasures/data/tests/test_user_services.py +++ b/flexmeasures/data/tests/test_user_services.py @@ -147,6 +147,10 @@ def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db assert ( data_source_before is not None ), "A data source linked to the prosumer user should exist before deletion." + data_source_account_id_before = data_source_before.account_id + assert ( + data_source_account_id_before is not None + ), "The data source linked to the prosumer user should have an account_id before deletion." # Find assets belonging to the user's organisation asset_query = select(GenericAsset).filter_by(account_id=prosumer_account_id) @@ -201,9 +205,12 @@ def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db fresh_db.session.refresh(user_creation_audit_log) assert user_creation_audit_log.affected_user_id is None - # Check that data source lineage is preserved: user_id is NOT nullified after user deletion + # Check that data source lineage is preserved: user_id and account_id are NOT nullified after user deletion fresh_db.session.expire(data_source_before) fresh_db.session.refresh(data_source_before) assert ( data_source_before.user_id == prosumer_id ), "Data source user_id should be preserved (not nullified) after user deletion for lineage purposes." + assert ( + data_source_before.account_id == data_source_account_id_before + ), "Data source account_id should be preserved (not nullified) after user deletion for lineage purposes." From 97efde0846e38ee35fdcb1c6f924be4cdfb8d7a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 03:29:51 +0000 Subject: [PATCH 20/63] docs: expand PR #2058 changelog entry to cover FK drop and lineage preservation Context: - The existing entry only mentioned adding account_id to data_source - PR #2058 also drops FK constraints on user_id and account_id to preserve historical references when users/accounts are deleted (data lineage) Change: - Changelog now states both the column addition AND the behavior change: 'historical user/account IDs are no longer nullified when users or accounts are deleted' --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b367ffc582..56135a070e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -19,7 +19,7 @@ New features Infrastructure / Support ---------------------- -* Add ``account_id`` field to the ``data_source`` table, populated from the related user's account (including data migration and automatic assignment when posting sensor data) [see `PR #2058 `_] +* Add ``account_id`` field to the ``data_source`` table, populated from the related user's account (including data migration and automatic assignment when posting sensor data); also drop FK constraints on ``data_source.user_id`` and ``data_source.account_id`` to preserve data lineage (historical user/account IDs are no longer nullified when users or accounts are deleted) [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management [see `PR #1973 `_] * Migrate from ``make`` to ``poe`` for running tasks [see `PR #1973 `_] From 6abc8d90ff0c2e2f3851d6853c21e098b9702325 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 03:29:58 +0000 Subject: [PATCH 21/63] coordinator: document no-FK lineage pattern and changelog completeness lesson Context: - PR #2058 dropped FK constraints on DataSource.user_id and account_id for data lineage preservation (a deliberate design decision) - The original changelog entry omitted the FK drop and behavior change - No existing agent pattern covered 'intentional FK removal for lineage' Change: - Added checklist for no-FK lineage pattern (passive_deletes, tests, changelog) - Added lesson: changelog must describe behavior changes, not just schema changes - Documented FK drop design decision for future reviewers --- .github/agents/coordinator.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/agents/coordinator.md b/.github/agents/coordinator.md index bfb178d9dc..8a8d892150 100644 --- a/.github/agents/coordinator.md +++ b/.github/agents/coordinator.md @@ -480,3 +480,23 @@ These patterns must not repeat. Agent instructions have been updated to prevent **Code observation from PR #2058 worth tracking**: - `user.account_id or (user.account.id if user.account else None)` — the `or` pattern is fragile for `account_id=0` (unrealistic but worth noting). Prefer `user.account_id if user.account_id is not None else (user.account.id if user.account else None)` for strict correctness. - Empty "Initial plan" commit adds git history noise. When orchestrating agents, the first commit should be functional code, not a planning marker. + +### Additional Pattern Discovered (2026-03-25) + +**Pattern**: No-FK columns for data lineage preservation + +**Session**: PR #2058 continued — Drop FK constraints on `data_source.user_id` and `data_source.account_id` + +**Design decision documented**: +FlexMeasures now intentionally drops DB-level FK constraints on `DataSource.user_id` and `DataSource.account_id` so that historical lineage references survive user/account deletion. The ORM uses `passive_deletes="all"` to prevent auto-nullification. + +**Checklist implication for future PRs**: +When reviewing schema changes that affect FK constraints: +- [ ] If a FK is dropped intentionally for lineage: verify `passive_deletes="all"` on the ORM relationship AND its backref +- [ ] Verify tests check that the orphaned column values are NOT nullified after parent deletion +- [ ] Verify changelog describes the *behavior change* (lineage preservation), not just the schema change (column added) + +**Changelog completeness check** — lessons from this session: +- The initial changelog entry for PR #2058 only described adding `account_id`; it omitted the FK drop and behavior change +- When a migration both adds a column AND changes deletion semantics (e.g., drops a FK), the changelog must cover BOTH aspects +- Coordinator caught this and updated the entry to read: "...also drop FK constraints on `data_source.user_id` and `data_source.account_id` to preserve data lineage (historical user/account IDs are no longer nullified when users or accounts are deleted)" From ebfba1d79e317f35bbd9cf05298bf8f5cedc84ba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 03:30:58 +0000 Subject: [PATCH 22/63] coordinator: clarify account_id fallback pattern references actual implementation The note previously described the 'or' anti-pattern as a concern, but the final code already uses 'if account_id is not None' correctly. Clarified to point to the implemented solution. --- .github/agents/coordinator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/agents/coordinator.md b/.github/agents/coordinator.md index 8a8d892150..bd9317de9c 100644 --- a/.github/agents/coordinator.md +++ b/.github/agents/coordinator.md @@ -478,7 +478,7 @@ These patterns must not repeat. Agent instructions have been updated to prevent **Governance escalation**: The Review Lead's "Must Always Run Coordinator" requirement has now been documented in three sessions without being followed. If it fails a fourth time, consider structural changes — e.g., making Coordinator invocation the FIRST step of a session rather than the last, so it sets context rather than being a forgotten epilogue. **Code observation from PR #2058 worth tracking**: -- `user.account_id or (user.account.id if user.account else None)` — the `or` pattern is fragile for `account_id=0` (unrealistic but worth noting). Prefer `user.account_id if user.account_id is not None else (user.account.id if user.account else None)` for strict correctness. +- An early draft used `user.account_id or (user.account.id if user.account else None)` — the `or` pattern is fragile for `account_id=0` (unrealistic but worth noting). The final implementation correctly uses `if user.account_id is not None` (see `data_sources.py` lines 340-343) — this is the right pattern to follow. - Empty "Initial plan" commit adds git history noise. When orchestrating agents, the first commit should be functional code, not a planning marker. ### Additional Pattern Discovered (2026-03-25) From c64b4ee82c50c6157a943d478bf0ad7b15b428f9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 13:38:50 +0100 Subject: [PATCH 23/63] feat: add test covering CLI command to delete user Signed-off-by: F.N. Claessen --- flexmeasures/cli/tests/test_data_delete.py | 72 ++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/flexmeasures/cli/tests/test_data_delete.py b/flexmeasures/cli/tests/test_data_delete.py index daa3cc315b..1198b4e114 100644 --- a/flexmeasures/cli/tests/test_data_delete.py +++ b/flexmeasures/cli/tests/test_data_delete.py @@ -77,3 +77,75 @@ def test_delete_account( f"Data source {ds_id} user_id should be preserved (not nullified) " "after account deletion for lineage purposes." ) + + +def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db, app): + """Check user is deleted + old audit log entries get affected_user_id set to None. + Also check that data source lineage is preserved: user_id is NOT nullified after user deletion. + """ + from flexmeasures.cli.data_delete import delete_a_user + from flexmeasures.data.models.data_sources import DataSource + + prosumer: User = find_user_by_email("test_prosumer_user@seita.nl") + prosumer_id = prosumer.id + prosumer_email = prosumer.email + prosumer_account_id = prosumer.account_id + + num_users = fresh_db.session.scalar(select(func.count()).select_from(User)) + + # Find data sources belonging to the user (for lineage check after deletion) + data_source_before = fresh_db.session.execute( + select(DataSource).filter_by(user_id=prosumer_id) + ).scalar_one_or_none() + if data_source_before is not None: + data_source_id = data_source_before.id + data_source_user_id_before = data_source_before.user_id + data_source_account_id_before = data_source_before.account_id + else: + data_source_id = None + + # Add creation audit log record + user_creation_audit_log = AuditLog( + event="User Test Prosumer User created test", + affected_user_id=prosumer_id, + affected_account_id=prosumer_account_id, + ) + fresh_db.session.add(user_creation_audit_log) + fresh_db.session.commit() + + # Delete the user via CLI + cli_input = { + "email": prosumer_email, + } + runner = app.test_cli_runner() + result = runner.invoke(delete_a_user, to_flags(cli_input), input="y\n") + check_command_ran_without_error(result) + + # Check user is deleted + assert find_user_by_email(prosumer_email) is None + assert ( + fresh_db.session.scalar(select(func.count()).select_from(User)) == num_users - 1 + ) + + # Check that old audit log entries get affected_user_id set to None + user_creation_audit_log_after = ( + fresh_db.session.query(AuditLog) + .filter_by(event="User Test Prosumer User created test") + .one_or_none() + ) + assert user_creation_audit_log_after.affected_user_id is None + + # Check that data source lineage is preserved: user_id is NOT nullified after user deletion + if data_source_id is not None: + data_source_after = fresh_db.session.get(DataSource, data_source_id) + assert ( + data_source_after is not None + ), f"Data source {data_source_id} should still exist after user deletion." + assert data_source_after.user_id == data_source_user_id_before, ( + f"Data source {data_source_id} user_id should be preserved (not nullified) " + "after user deletion for lineage purposes." + ) + assert data_source_after.account_id == data_source_account_id_before, ( + f"Data source {data_source_id} account_id should be preserved (not nullified) " + "after user deletion for lineage purposes." + ) From 57c1f8eb56d3eb4f919081977667ea89927e4677 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 13:43:31 +0100 Subject: [PATCH 24/63] feat: let audit log entries retain the account ID and user IDs after either is deleted Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- flexmeasures/cli/tests/test_data_delete.py | 18 +++++++-- ...onstraint_from_audit_log_active_user_id.py | 38 +++++++++++++++++++ flexmeasures/data/models/audit_log.py | 33 +++++++++++++--- flexmeasures/data/tests/test_user_services.py | 5 ++- 5 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 69c2c66507..cb69038390 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -19,7 +19,7 @@ New features Infrastructure / Support ---------------------- -* Add ``account_id`` field to the ``data_source`` table, populated from the related user's account (including data migration and automatic assignment when posting sensor data); also drop FK constraints on ``data_source.user_id`` and ``data_source.account_id`` to preserve data lineage (historical user/account IDs are no longer nullified when users or accounts are deleted) [see `PR #2058 `_] +* Preserve data lineage by removing cascading deletes from user and account references in audit logs and data sources; when users or accounts are deleted, their IDs are retained in historical records for traceability and compliance [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management [see `PR #1973 `_] * Migrate from ``make`` to ``poe`` for running tasks [see `PR #1973 `_] diff --git a/flexmeasures/cli/tests/test_data_delete.py b/flexmeasures/cli/tests/test_data_delete.py index 1198b4e114..9b15d13a30 100644 --- a/flexmeasures/cli/tests/test_data_delete.py +++ b/flexmeasures/cli/tests/test_data_delete.py @@ -60,7 +60,14 @@ def test_delete_account( .filter_by(event="User Test Prosumer User created test") .one_or_none() ) - assert user_creation_audit_log.affected_account_id is None + assert user_creation_audit_log.affected_account_id == prosumer_account_id, ( + "Audit log affected_account_id should be preserved (not nullified) " + "after account deletion for lineage purposes." + ) + assert user_creation_audit_log.affected_user_id == prosumer.id, ( + "Audit log affected_user_id should be preserved (not nullified) " + "after account deletion for lineage purposes." + ) # Check that data source lineage is preserved: account_id and user_id are NOT nullified after account deletion for ds_id, original_user_id, original_account_id in data_source_ids_and_lineage: @@ -80,7 +87,7 @@ def test_delete_account( def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db, app): - """Check user is deleted + old audit log entries get affected_user_id set to None. + """Check user is deleted + old audit log entries get affected_user_id preserved. Also check that data source lineage is preserved: user_id is NOT nullified after user deletion. """ from flexmeasures.cli.data_delete import delete_a_user @@ -127,13 +134,16 @@ def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db fresh_db.session.scalar(select(func.count()).select_from(User)) == num_users - 1 ) - # Check that old audit log entries get affected_user_id set to None + # Check that old audit log entries preserve affected_user_id (not set to None) user_creation_audit_log_after = ( fresh_db.session.query(AuditLog) .filter_by(event="User Test Prosumer User created test") .one_or_none() ) - assert user_creation_audit_log_after.affected_user_id is None + assert user_creation_audit_log_after.affected_user_id == prosumer_id, ( + "Audit log affected_user_id should be preserved (not nullified) " + "after user deletion for lineage purposes." + ) # Check that data source lineage is preserved: user_id is NOT nullified after user deletion if data_source_id is not None: diff --git a/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py b/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py new file mode 100644 index 0000000000..66216bee5e --- /dev/null +++ b/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py @@ -0,0 +1,38 @@ +"""Drop FK constraint from audit_log.active_user_id for data lineage preservation + +When users are deleted, we want to preserve the historical active_user_id +values in audit_log rows for lineage purposes, rather than cascade-deleting +or nullifying them. + +Revision ID: b2c3d4e5f6a7 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-27 00:00:00.000000 + +""" + +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "b2c3d4e5f6a7" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("audit_log", schema=None) as batch_op: + batch_op.drop_constraint( + op.f("audit_log_active_user_id_fm_user_fkey"), type_="foreignkey" + ) + + +def downgrade(): + with op.batch_alter_table("audit_log", schema=None) as batch_op: + batch_op.create_foreign_key( + op.f("audit_log_active_user_id_fm_user_fkey"), + "fm_user", + ["active_user_id"], + ["id"], + ondelete="SET NULL", + ) diff --git a/flexmeasures/data/models/audit_log.py b/flexmeasures/data/models/audit_log.py index bf1c615fbd..9a54048cce 100644 --- a/flexmeasures/data/models/audit_log.py +++ b/flexmeasures/data/models/audit_log.py @@ -30,14 +30,35 @@ class AuditLog(db.Model, AuthModelMixin): event_datetime = Column(DateTime()) event = Column(String(255)) active_user_name = Column(String(255)) - active_user_id = Column( - "active_user_id", Integer(), ForeignKey("fm_user.id", ondelete="SET NULL") + # No DB-level FK with cascade for active_user_id so that deleting a user preserves the lineage reference in this column. + active_user_id = Column("active_user_id", Integer(), nullable=True) + # No DB-level FK with cascade for affected_user_id so that deleting a user preserves the lineage reference in this column. + affected_user_id = Column("affected_user_id", Integer(), nullable=True) + # No DB-level FK with cascade for affected_account_id so that deleting an account preserves the lineage reference in this column. + affected_account_id = Column("affected_account_id", Integer(), nullable=True) + + # Relationships to navigate to User and Account without database-level FK constraints + # This allows audit logs to maintain references to deleted users/accounts for lineage purposes + active_user = db.relationship( + "User", + primaryjoin="AuditLog.active_user_id == User.id", + foreign_keys="[AuditLog.active_user_id]", + backref=db.backref("active_audit_logs", lazy=True, passive_deletes="all"), + passive_deletes="all", ) - affected_user_id = Column( - "affected_user_id", Integer(), ForeignKey("fm_user.id", ondelete="SET NULL") + affected_user = db.relationship( + "User", + primaryjoin="AuditLog.affected_user_id == User.id", + foreign_keys="[AuditLog.affected_user_id]", + backref=db.backref("affected_audit_logs", lazy=True, passive_deletes="all"), + passive_deletes="all", ) - affected_account_id = Column( - "affected_account_id", Integer(), ForeignKey("account.id", ondelete="SET NULL") + affected_account = db.relationship( + "Account", + primaryjoin="AuditLog.affected_account_id == Account.id", + foreign_keys="[AuditLog.affected_account_id]", + backref=db.backref("affected_audit_logs", lazy=True, passive_deletes="all"), + passive_deletes="all", ) @classmethod diff --git a/flexmeasures/data/tests/test_user_services.py b/flexmeasures/data/tests/test_user_services.py index 5e431681ed..41715d362d 100644 --- a/flexmeasures/data/tests/test_user_services.py +++ b/flexmeasures/data/tests/test_user_services.py @@ -203,7 +203,10 @@ def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db assert user_deletion_audit_log.active_user_id is None fresh_db.session.refresh(user_creation_audit_log) - assert user_creation_audit_log.affected_user_id is None + assert user_creation_audit_log.affected_user_id == prosumer_id, ( + "Audit log affected_user_id should be preserved (not nullified) " + "after user deletion for lineage purposes." + ) # Check that data source lineage is preserved: user_id and account_id are NOT nullified after user deletion fresh_db.session.expire(data_source_before) From e1d6cdcdffc76c043b1587e88904b7925381ae1e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 13:54:29 +0100 Subject: [PATCH 25/63] docs: rewrite changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index cb69038390..07ef29cbab 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -19,7 +19,7 @@ New features Infrastructure / Support ---------------------- -* Preserve data lineage by removing cascading deletes from user and account references in audit logs and data sources; when users or accounts are deleted, their IDs are retained in historical records for traceability and compliance [see `PR #2058 `_] +* Support coupling data sources to accounts, and preserve user ID and account ID references in audit logs and data sources for traceability and compliance [see `PR #2058 `_] * Stop creating new toy assets when restarting the docker-compose stack [see `PR #2018 `_] * Migrate from ``pip`` to ``uv`` for dependency management [see `PR #1973 `_] * Migrate from ``make`` to ``poe`` for running tasks [see `PR #1973 `_] From 7ec2cb1eb0445e716088d2cf4ed02bb208c8fb7b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 14:10:03 +0100 Subject: [PATCH 26/63] fix: dropped foreign key name Signed-off-by: F.N. Claessen --- ...f6_drop_fk_constraints_from_data_source.py | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py b/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py index 35bbb45acc..79070d5498 100644 --- a/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py +++ b/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py @@ -11,6 +11,7 @@ """ from alembic import op +from sqlalchemy import inspect # revision identifiers, used by Alembic. @@ -21,23 +22,36 @@ def upgrade(): + # Inspect existing FK constraints to handle different database states gracefully + bind = op.get_bind() + inspector = inspect(bind) + existing_fks = inspector.get_foreign_keys("data_source") + existing_fk_names = [fk["name"] for fk in existing_fks] + with op.batch_alter_table("data_source", schema=None) as batch_op: - batch_op.drop_constraint( - op.f("data_source_account_id_account_fkey"), type_="foreignkey" - ) - batch_op.drop_constraint("data_sources_user_id_fkey", type_="foreignkey") + # Drop the account_id FK if it exists + if "data_source_account_id_account_fkey" in existing_fk_names: + batch_op.drop_constraint( + "data_source_account_id_account_fkey", type_="foreignkey" + ) + + # Drop the user_id FK if it exists (may have auto-generated name) + for fk_name in existing_fk_names: + if "user_id" in fk_name: + batch_op.drop_constraint(fk_name, type_="foreignkey") + break def downgrade(): with op.batch_alter_table("data_source", schema=None) as batch_op: batch_op.create_foreign_key( - "data_sources_user_id_fkey", + "data_source_user_id_fkey", "fm_user", ["user_id"], ["id"], ) batch_op.create_foreign_key( - op.f("data_source_account_id_account_fkey"), + "data_source_account_id_account_fkey", "account", ["account_id"], ["id"], From 0fefb2ef36e8abf36c7984d8cb4937098b9b8b95 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 15:27:01 +0100 Subject: [PATCH 27/63] fix: when deleting a user, still log the ID of the deleted user Signed-off-by: F.N. Claessen --- flexmeasures/data/services/users.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/services/users.py b/flexmeasures/data/services/users.py index f33dbfd87f..d802919999 100644 --- a/flexmeasures/data/services/users.py +++ b/flexmeasures/data/services/users.py @@ -222,11 +222,6 @@ def delete_user(user: User): if hasattr(current_user, "id") and user.id == current_user.id: raise Exception("You cannot delete yourself.") - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - user_datastore.delete_user(user) - db.session.execute(delete(User).filter_by(id=user.id)) - current_app.logger.info("Deleted %s." % user) - active_user_id, active_user_name = None, None if hasattr(current_user, "id"): active_user_id, active_user_name = current_user.id, current_user.username @@ -235,7 +230,12 @@ def delete_user(user: User): event=f"User {user.username} deleted", active_user_id=active_user_id, active_user_name=active_user_name, - affected_user_id=None, # add the audit log record even if the user is gone + affected_user_id=user.id, # add the audit log record even if the user will be gone affected_account_id=user.account_id, ) db.session.add(user_audit_log) + + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) + user_datastore.delete_user(user) + db.session.execute(delete(User).filter_by(id=user.id)) + current_app.logger.info("Deleted %s." % user) From 6bcf9c6d222a0924b59ebaa21f85bbae471aa306 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 15:34:44 +0100 Subject: [PATCH 28/63] feat: allow tying a newly CLI-created data source to an account Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 12 +++++++++++- flexmeasures/data/services/data_sources.py | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 066c85e49d..42a753c080 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -516,13 +516,23 @@ def add_initial_structure(): type=str, help=f"Type of source (free, but FlexMeasures has support for {DEFAULT_DATASOURCE_TYPES}).", ) -def add_source(name: str, model: str, version: str, source_type: str): +@click.option( + "--account", + "account", + required=False, + type=AccountIdField(), + help=f"Organisation account associated with the source.", +) +def add_source( + name: str, model: str, version: str, source_type: str, account: Account | None +): """Add a data source.""" source = get_or_create_source( source=name, model=model, version=version, source_type=source_type, + account=account, ) db.session.commit() click.secho(f"Added source {source.__repr__()}", **MsgStyle.SUCCESS) diff --git a/flexmeasures/data/services/data_sources.py b/flexmeasures/data/services/data_sources.py index c2f260806b..d8260b022e 100644 --- a/flexmeasures/data/services/data_sources.py +++ b/flexmeasures/data/services/data_sources.py @@ -6,7 +6,7 @@ from sqlalchemy import select from typing import Type, TypeVar -from flexmeasures import User, Source +from flexmeasures import Account, Source, User from flexmeasures.data import db from flexmeasures.data.models.data_sources import DataSource, DataGenerator from flexmeasures.data.models.user import is_user @@ -22,6 +22,7 @@ def get_or_create_source( model: str | None = None, version: str | None = None, attributes: dict | None = None, + account: Account | None = None, flush: bool = True, ) -> DataSource: if is_user(source): @@ -35,6 +36,8 @@ def get_or_create_source( query = query.filter( DataSource.attributes_hash == DataSource.hash_attributes(attributes) ) + if account is not None: + query = query.filter(DataSource.account == account) if is_user(source): query = query.filter(DataSource.user == source) elif isinstance(source, str): @@ -54,6 +57,7 @@ def get_or_create_source( version=version, type=source_type, attributes=attributes, + account=account, ) current_app.logger.info(f"Setting up {_source} as new data source...") db.session.add(_source) From faa876f69ea8eac978ceba3aa04bb2450da60d0c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 15:43:11 +0100 Subject: [PATCH 29/63] fix: add missing drop_constraints Signed-off-by: F.N. Claessen --- ...onstraint_from_audit_log_active_user_id.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py b/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py index 66216bee5e..749219c46f 100644 --- a/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py +++ b/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py @@ -25,10 +25,30 @@ def upgrade(): batch_op.drop_constraint( op.f("audit_log_active_user_id_fm_user_fkey"), type_="foreignkey" ) + batch_op.drop_constraint( + op.f("audit_log_affected_user_id_fm_user_fkey"), type_="foreignkey" + ) + batch_op.drop_constraint( + op.f("audit_log_affected_account_id_account_fkey"), type_="foreignkey" + ) def downgrade(): with op.batch_alter_table("audit_log", schema=None) as batch_op: + batch_op.create_foreign_key( + op.f("audit_log_affected_account_id_account_fkey"), + "account", + ["affected_account_id"], + ["id"], + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + op.f("audit_log_affected_user_id_fm_user_fkey"), + "fm_user", + ["affected_user_id"], + ["id"], + ondelete="SET NULL", + ) batch_op.create_foreign_key( op.f("audit_log_active_user_id_fm_user_fkey"), "fm_user", From 317c687f8dc06338df797db52b6cad4008e37756 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 15:44:24 +0100 Subject: [PATCH 30/63] chore: minimize diff Signed-off-by: F.N. Claessen --- flexmeasures/data/services/users.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/services/users.py b/flexmeasures/data/services/users.py index d802919999..ce96ba13b6 100644 --- a/flexmeasures/data/services/users.py +++ b/flexmeasures/data/services/users.py @@ -222,6 +222,11 @@ def delete_user(user: User): if hasattr(current_user, "id") and user.id == current_user.id: raise Exception("You cannot delete yourself.") + user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) + user_datastore.delete_user(user) + db.session.execute(delete(User).filter_by(id=user.id)) + current_app.logger.info("Deleted %s." % user) + active_user_id, active_user_name = None, None if hasattr(current_user, "id"): active_user_id, active_user_name = current_user.id, current_user.username @@ -234,8 +239,3 @@ def delete_user(user: User): affected_account_id=user.account_id, ) db.session.add(user_audit_log) - - user_datastore = SQLAlchemySessionUserDatastore(db.session, User, Role) - user_datastore.delete_user(user) - db.session.execute(delete(User).filter_by(id=user.id)) - current_app.logger.info("Deleted %s." % user) From 4401355998d2bfdbf2b8ca93210c57b26e6b7ab3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 15:46:45 +0100 Subject: [PATCH 31/63] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 42a753c080..011cc114c8 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -521,7 +521,7 @@ def add_initial_structure(): "account", required=False, type=AccountIdField(), - help=f"Organisation account associated with the source.", + help="Organisation account associated with the source.", ) def add_source( name: str, model: str, version: str, source_type: str, account: Account | None From d7a4ec1dd2237537257a9b325495db111db51fcd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 27 Mar 2026 16:25:22 +0100 Subject: [PATCH 32/63] fix: test Signed-off-by: F.N. Claessen --- flexmeasures/data/tests/test_user_services.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/tests/test_user_services.py b/flexmeasures/data/tests/test_user_services.py index 41715d362d..8d90fae4dd 100644 --- a/flexmeasures/data/tests/test_user_services.py +++ b/flexmeasures/data/tests/test_user_services.py @@ -198,7 +198,10 @@ def test_delete_user(fresh_db, setup_roles_users_fresh_db, setup_assets_fresh_db .filter_by(event="User Test Prosumer User deleted") .one_or_none() ) - assert user_deletion_audit_log.affected_user_id is None + assert user_deletion_audit_log.affected_user_id == prosumer_id, ( + "Audit log affected_user_id should be preserved (not nullified) " + "after user deletion for lineage purposes." + ) assert user_deletion_audit_log.affected_account_id == prosumer_account_id assert user_deletion_audit_log.active_user_id is None From c103b951e327e59c5d4d9559af1b2a031f86600c Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:23:47 +0100 Subject: [PATCH 33/63] feat: add source account query criterion Context: - source filtering needs to support data sources linked to accounts Change: - add a reusable query criterion for filtering by source account ids --- flexmeasures/data/queries/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/flexmeasures/data/queries/utils.py b/flexmeasures/data/queries/utils.py index c439f41da1..78afb5f1b0 100644 --- a/flexmeasures/data/queries/utils.py +++ b/flexmeasures/data/queries/utils.py @@ -85,12 +85,15 @@ def potentially_limit_assets_query_to_accounts( def get_source_criteria( cls: "Type[ts.TimedValue] | Type[ts.TimedBelief]", user_source_ids: int | list[int], + source_account_ids: int | list[int], source_types: list[str], exclude_source_types: list[str], ) -> list[BinaryExpression]: source_criteria: list[BinaryExpression] = [] if user_source_ids is not None: source_criteria.append(user_source_criterion(cls, user_source_ids)) + if source_account_ids is not None: + source_criteria.append(source_account_criterion(source_account_ids)) if source_types is not None: if user_source_ids and "user" not in source_types: source_types.append("user") @@ -131,6 +134,13 @@ def user_source_criterion( return cls.source_id.not_in(ignorable_user_source_ids) +def source_account_criterion(source_account_ids: int | list[int]) -> BinaryExpression: + """Criterion to collect only data from sources belonging to the given account IDs.""" + if not isinstance(source_account_ids, list): + source_account_ids = [source_account_ids] + return DataSource.account_id.in_(source_account_ids) + + def source_type_criterion(source_types: list[str]) -> BinaryExpression: """Criterion to collect only data from sources that are of the given type.""" return DataSource.type.in_(source_types) From bbbc771f92e2651eda22e3228145ccb7a66a4ba8 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:24:07 +0100 Subject: [PATCH 34/63] feat: thread source account filters through time series search Context: - account-based source filtering needs to reach belief queries from sensor search helpers Change: - add source_account_ids to the time-series search interfaces and pass it into source criteria generation --- flexmeasures/data/models/time_series.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index 76a3605254..db9912f786 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -395,6 +395,7 @@ def search_beliefs( # noqa: C901 DataSource | list[DataSource] | int | list[int] | str | list[str] | None ) = None, user_source_ids: int | list[int] | None = None, + source_account_ids: int | list[int] | None = None, source_types: list[str] | None = None, exclude_source_types: list[str] | None = None, use_latest_version_per_event: bool = True, @@ -419,6 +420,7 @@ def search_beliefs( # noqa: C901 :param horizons_at_most: only return beliefs with a belief horizon equal or less than this timedelta (for example, use timedelta(0) to get post knowledge time beliefs) :param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources. Without this set and a most recent parameter used (see below), the results can be of any source. :param user_source_ids: Optional list of user source ids to query only specific user sources + :param source_account_ids: Optional list of account ids to query only sources linked to specific accounts :param source_types: Optional list of source type names to query only specific source types * :param exclude_source_types: Optional list of source type names to exclude specific source types * :param use_latest_version_per_event: only return the belief from the latest version of a source, for each event @@ -442,6 +444,7 @@ def search_beliefs( # noqa: C901 horizons_at_most=horizons_at_most, source=source, user_source_ids=user_source_ids, + source_account_ids=source_account_ids, source_types=source_types, exclude_source_types=exclude_source_types, use_latest_version_per_event=use_latest_version_per_event, @@ -851,6 +854,7 @@ def search( DataSource | list[DataSource] | int | list[int] | str | list[str] | None ) = None, user_source_ids: int | list[int] | None = None, + source_account_ids: int | list[int] | None = None, source_types: list[str] | None = None, exclude_source_types: list[str] | None = None, use_latest_version_per_event: bool = True, @@ -875,6 +879,7 @@ def search( :param horizons_at_most: only return beliefs with a belief horizon equal or less than this timedelta (for example, use timedelta(0) to get post knowledge time beliefs) :param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources :param user_source_ids: Optional list of user source ids to query only specific user sources + :param source_account_ids: Optional list of account ids to query only sources linked to specific accounts :param source_types: Optional list of source type names to query only specific source types * :param exclude_source_types: Optional list of source type names to exclude specific source types * :param use_latest_version_per_event: only return the belief from the latest version of a source, for each event @@ -915,7 +920,11 @@ def search( parsed_sources = parse_source_arg(source) source_criteria = get_source_criteria( - cls, user_source_ids, source_types, exclude_source_types + cls, + user_source_ids, + source_account_ids, + source_types, + exclude_source_types, ) custom_join_targets = [] if parsed_sources else [DataSource] From dee8a7c005c53afd26822f749e4cb25e5eb286f1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:24:26 +0100 Subject: [PATCH 35/63] feat: accept source account filters in sensor data schema Context: - the GET sensor data schema needs to expose account-linked source filtering Change: - add source_account_id to the sensor data schema and pass it into belief searches - define a query schema for documenting the actual GET parameters --- flexmeasures/api/common/schemas/sensor_data.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index d1418705c2..44d290d217 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -16,6 +16,7 @@ SensorEntityAddressField, SensorIdField, ) +from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.utils.api_utils import upsample_values from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.schemas import AwareDateTimeField, DurationField, SourceIdField @@ -152,6 +153,7 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): resolution = DurationField(required=False) source = SourceIdField(required=False) + source_account = AccountIdField(required=False, data_key="source_account_id") # Optional field that can be used for extra validation type = fields.Str( @@ -202,6 +204,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: unit = sensor_data_description["unit"] resolution = sensor_data_description.get("resolution") source = sensor_data_description.get("source") + source_account = sensor_data_description.get("source_account") # Post-load configuration of event frequency if resolution is None: @@ -231,6 +234,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: horizons_at_least=horizons_at_least, horizons_at_most=horizons_at_most, source=source, + source_account_ids=source_account.id if source_account else None, beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, resolution=resolution, @@ -264,6 +268,14 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: return response +class GetSensorDataQuerySchema(SensorDataTimingDescriptionSchema): + """Document the actual query parameters for GET /sensors//data.""" + + resolution = DurationField(required=False) + source = SourceIdField(required=False) + source_account = AccountIdField(required=False, data_key="source_account_id") + + class PostSensorDataSchema(SensorDataDescriptionSchema): """ This schema includes data (values) and still describes it. From 0e50afdaf04ac173f6aa56a3a7ff1da0e81fbede Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:24:43 +0100 Subject: [PATCH 36/63] fix: document sensor data query schema correctly Context: - the GET sensor data route was pointing Swagger at the wrong query schema Change: - switch the documented query schema to the GET-specific one - clarify source and source_account_id in the endpoint docs --- flexmeasures/api/v3_0/sensors.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 4fc4ac1ae1..564815ddd8 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -31,6 +31,7 @@ from flexmeasures.api.common.schemas.sensor_data import ( # noqa F401 SensorDataDescriptionSchema, GetSensorDataSchema, + GetSensorDataQuerySchema, PostSensorDataSchema, ) from flexmeasures.api.common.schemas.sensors import SensorId # noqa F401 @@ -585,7 +586,8 @@ def get_data(self, id: int, **sensor_data_description: dict): - "resolution" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution)) - "horizon" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs)) - "prior" (the belief timing docs also apply here) - - "source" (read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources)) + - "source" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources)) + - "source_account_id" (filter by the account ID linked to data sources) An example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m³/h: @@ -602,7 +604,7 @@ def get_data(self, id: int, **sensor_data_description: dict): required: true schema: SensorId - in: query - schema: SensorDataTimingDescriptionSchema + schema: GetSensorDataQuerySchema responses: 200: From 201679faa695415efc02ec6079fdfbec888e613a Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:25:01 +0100 Subject: [PATCH 37/63] test: cover source account filtering in sensor data GET Context: - the sensor data endpoint now accepts source_account_id for source filtering Change: - add a GET sensor data test that verifies account-linked sources are filtered correctly --- .../api/v3_0/tests/test_sensor_data.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index c9c1822c7c..3b20dc02d6 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -114,6 +114,40 @@ def test_get_instantaneous_sensor_data( assert all(a == b for a, b in zip(values, [815, 818, None, None])) +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_get_sensor_data_filtered_by_source_account( + client, + setup_api_test_data: dict[str, Sensor], + setup_roles_users: dict[str, User], + requesting_user, + db, +): + """Check that GET /sensors//data can filter by the account linked to a source.""" + sensor = setup_api_test_data["some gas sensor"] + source_user = db.session.get(User, setup_roles_users["Test Supplier User"]) + assert source_user.account_id is not None + message = { + "start": "2021-05-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "m³/h", + "source_account_id": source_user.account_id, + "resolution": "PT20M", + } + response = client.get( + url_for("SensorAPI:get_data", id=sensor.id), + query_string=message, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + values = response.json["values"] + # The fixture also stores data from an accountless "Other source". + # Filtering by the user account should exclude those points. + assert all(a == b for a, b in zip(values, [91.5, 92.1, None, None])) + + @pytest.mark.parametrize( "requesting_user, status_code", [ From a12b1ae63aaa7e6fffaf9628421150d2ce1d63db Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:25:20 +0100 Subject: [PATCH 38/63] docs: clarify source filters for sensor data GET Context: - the notation docs should match the actual source filtering supported by the endpoint Change: - document source and source_account_id for GET sensor data - clarify that source type filtering is not supported there --- documentation/api/notation.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 5b6a5cab45..ee4ac6f9bd 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -274,6 +274,13 @@ For example, to obtain data originating from data source 42, include the followi Data source IDs can be found by hovering over data in charts. +For the ``GET /api/v3_0/sensors//data`` endpoint specifically, source filtering supports: + +- ``source``: filter by data source ID +- ``source_account_id``: filter by the account ID linked to data sources + +Filtering that endpoint by source type is currently not supported. + .. _units: Units From e4284fdf6798147e2d2cc1200fc0586de0b1ff8e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:25:40 +0100 Subject: [PATCH 39/63] docs: add changelog entry for sensor data source filtering Context: - the endpoint behavior and Swagger exposure changed in a user-visible way Change: - add a changelog line for source account filtering and Swagger query parameter exposure --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 07ef29cbab..b3235e4b50 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -30,6 +30,7 @@ Infrastructure / Support Bugfixes ----------- +* Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2058 `_] v0.31.2 | March 18, 2026 @@ -1558,4 +1559,3 @@ Infrastructure / Support * Start using marshmallow for input validation, also introducing ``HTTP status 422 (Unprocessable Entity)`` in the API [see `PR #25 `_] * Replace ``solarpy`` with ``pvlib`` (due to license conflict) [see `PR #16 `_] * Stop supporting the creation of new users on asset creation (to reduce complexity) [see `PR #36 `_] - From f91203e238e16ca8ccdd1eda70412bb66f62c290 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 2 Apr 2026 02:26:09 +0100 Subject: [PATCH 40/63] chore: refresh generated OpenAPI specs Context: - the sensor data schema and route docs changed the exposed GET query parameters Change: - regenerate the checked-in OpenAPI spec to match the current API surface --- flexmeasures/ui/static/openapi-specs.json | 27 ++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index c95f04cfda..ee1f7189b9 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -339,7 +339,7 @@ }, "get": { "summary": "Get sensor data", - "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > **Note:** This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n", + "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"source_account_id\" (filter by the account ID linked to data sources)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > **Note:** This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n", "security": [ { "ApiKeyAuth": [] @@ -407,6 +407,31 @@ "example": "m\u00b3/h" }, "required": true + }, + { + "in": "query", + "name": "resolution", + "schema": { + "type": "string", + "format": "duration" + }, + "required": false + }, + { + "in": "query", + "name": "source", + "schema": { + "type": "integer" + }, + "required": false + }, + { + "in": "query", + "name": "source_account_id", + "schema": { + "type": "integer" + }, + "required": false } ], "responses": { From 8bb5ebeba24f8ee4f02f11d0497994e96ac2cb95 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 2 Apr 2026 15:17:11 +0200 Subject: [PATCH 41/63] agents: rename Review Lead to Lead across all agent instructions Update all agent documentation to use 'Lead' instead of 'Review Lead', reflecting the consolidated governance model where lead.md has been removed and its responsibilities are now documented in AGENTS.md. Changes: - AGENTS.md: All 31 'Review Lead' references updated to 'Lead' - coordinator.md: Updated monitoring patterns to reference Lead - test-specialist.md: Updated coordination examples to reference Lead - tooling-ci-specialist.md: Updated coordination examples to reference Lead This ensures consistent terminology across all agent instructions. --- .github/agents/coordinator.md | 88 +++++++++--------- .github/agents/test-specialist.md | 12 +-- .github/agents/tooling-ci-specialist.md | 12 +-- AGENTS.md | 118 ++++++++++++------------ 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/agents/coordinator.md b/.github/agents/coordinator.md index 58a3b95698..b3e0bbc871 100644 --- a/.github/agents/coordinator.md +++ b/.github/agents/coordinator.md @@ -292,30 +292,30 @@ Agents should escalate to the Coordinator when: - Encourage agent autonomy and expertise - Provide actionable feedback via review comments -### Review Lead Delegation Pattern Monitoring +### Lead Delegation Pattern Monitoring -**The Coordinator MUST verify Review Lead delegation patterns during governance reviews.** +**The Coordinator MUST verify Lead delegation patterns during governance reviews.** -**Context:** Review Lead has a recurring failure mode of working solo instead of delegating to specialists (observed in session 2026-02-08). +**Context:** Lead has a recurring failure mode of working solo instead of delegating to specialists (observed in session 2026-02-08). **What to check:** -When reviewing a session where Review Lead was involved: +When reviewing a session where Lead was involved: -- [ ] **Delegation occurred**: Did Review Lead invoke appropriate specialists? -- [ ] **No solo execution**: Did Review Lead make code/API/docs changes itself? -- [ ] **Git commit author check**: Are there Review Lead commits with production code? -- [ ] **Request interpretation**: Did Review Lead parse user intent correctly? +- [ ] **Delegation occurred**: Did Lead invoke appropriate specialists? +- [ ] **No solo execution**: Did Lead make code/API/docs changes itself? +- [ ] **Git commit author check**: Are there Lead commits with production code? +- [ ] **Request interpretation**: Did Lead parse user intent correctly? - [ ] **Regression indicators**: Any signs of "too simple to delegate" thinking? **Red flags (immediate governance concern):** -- 🚩 Review Lead commits containing code changes (should be specialist commits) -- 🚩 Review Lead commits containing test changes (should be Test Specialist) -- 🚩 Review Lead commits containing doc changes (should be Documentation Specialist) +- 🚩 Lead commits containing code changes (should be specialist commits) +- 🚩 Lead commits containing test changes (should be Test Specialist) +- 🚩 Lead commits containing doc changes (should be Documentation Specialist) - 🚩 User says "You are regressing" or "You must handle requests as a team" - 🚩 Session closed without specialist involvement on implementation tasks -- 🚩 Review Lead justifies solo work with "too simple to delegate" +- 🚩 Lead justifies solo work with "too simple to delegate" **Verification commands:** @@ -323,27 +323,27 @@ When reviewing a session where Review Lead was involved: # Check who made commits git log --oneline --all --since="1 day ago" --format="%h %an %s" -# Check Review Lead commit types -git log --author="Review Lead" --oneline -10 +# Check Lead commit types +git log --author="Lead" --oneline -10 -# Look for code changes by Review Lead (should be empty or synthesis only) -git log --author="Review Lead" --stat -5 +# Look for code changes by Lead (should be empty or synthesis only) +git log --author="Lead" --stat -5 ``` **When delegation failure detected:** 1. **Document in session review** - What was the failure? -2. **Check Review Lead instructions** - Were they followed? +2. **Check Lead instructions** - Were they followed? 3. **Identify gap** - What prevented proper delegation? 4. **Recommend fix** - How to prevent recurrence? -5. **Update Review Lead instructions** - Add enforcement mechanism +5. **Update Lead instructions** - Add enforcement mechanism 6. **Verify fix works** - Test with hypothetical scenario **Escalation pattern:** -If Review Lead repeatedly violates delegation requirements: +If Lead repeatedly violates delegation requirements: - This is a systemic issue requiring Coordinator intervention -- Review Lead instructions need stronger enforcement +- Lead instructions need stronger enforcement - Consider adding mandatory checkpoints before work execution - May need explicit blockers to prevent solo execution @@ -351,20 +351,20 @@ If Review Lead repeatedly violates delegation requirements: | Pattern | Indicator | Action | |---------|-----------|--------| -| Solo execution | Review Lead makes code commits | Flag as regression | -| "Too simple" trap | Review Lead justifies not delegating | Update instructions with example | -| Request misinterpretation | Review Lead confirms instead of implements | Strengthen request parsing guidance | +| Solo execution | Lead makes code commits | Flag as regression | +| "Too simple" trap | Lead justifies not delegating | Update instructions with example | +| Request misinterpretation | Lead confirms instead of implements | Strengthen request parsing guidance | | Delegation omission | Specialists not invoked on implementation | Verify Session Close Checklist followed | **Success indicators:** -- ✅ Review Lead invoked appropriate specialists +- ✅ Lead invoked appropriate specialists - ✅ Specialists made the actual changes -- ✅ Review Lead synthesized findings +- ✅ Lead synthesized findings - ✅ Team-based execution pattern maintained - ✅ Session Close Checklist verified delegation -**This monitoring ensures Review Lead maintains its orchestration role and doesn't regress to solo execution.** +**This monitoring ensures Lead maintains its orchestration role and doesn't regress to solo execution.** ## Self-Improvement Notes @@ -551,25 +551,25 @@ Lead should now invoke Coordinator as subagent. - Governance process shown to be optional (dangerous precedent) **Solution implemented**: -1. ✅ Added mandatory "Session Close Checklist" to Review Lead (commit 3ad8908) +1. ✅ Added mandatory "Session Close Checklist" to Lead (commit 3ad8908) 2. ✅ Added "Full Test Suite Requirement" to Test Specialist (commit 8d67f3c) 3. ✅ Added "Pre-commit Hook Enforcement" to Tooling & CI Specialist (commit dfe67e8) 4. ✅ Added "Session Close Verification" pattern to Coordinator (this commit) **Structural changes**: -- Review Lead now has comprehensive checklist before closing any session +- Lead now has comprehensive checklist before closing any session - Test Specialist must execute full suite, not just feature-specific tests - Tooling & CI Specialist must verify pre-commit execution -- Coordinator enforces Review Lead checklist completion +- Coordinator enforces Lead checklist completion **New Coordinator pattern (Pattern #7)**: When invoked for governance review, Coordinator must verify: -- [ ] Review Lead followed session close checklist +- [ ] Lead followed session close checklist - [ ] No checklist items were skipped without justification - [ ] Evidence provided for each checklist item **Enforcement escalation**: -If Review Lead repeatedly closes sessions without completing checklist: +If Lead repeatedly closes sessions without completing checklist: 1. First occurrence: Document and update instructions (this session) 2. Second occurrence: Require explicit justification for skips 3. Third occurrence: Escalate to architectural solution (automated checks) @@ -592,26 +592,26 @@ These patterns must not repeat. Agent instructions have been updated to prevent **Session**: PR #2058 — Add `account_id` to DataSource table **Observation**: After three sessions now, the same two failures recur: -1. Coordinator is not invoked at end of session (despite MUST requirement in Review Lead instructions) +1. Coordinator is not invoked at end of session (despite MUST requirement in Lead instructions) 2. No agent updates its own instructions (despite MUST requirement in all agents) **Root cause analysis**: - "Coordinator invocation" and "self-improvement" are both documented as mandatory last steps - But the session ends before they are reached — they are treated as optional epilogue, not gating requirements -- The Review Lead agent selection is ad-hoc, with no explicit checklist forcing API Specialist engagement when endpoints change +- The Lead agent selection is ad-hoc, with no explicit checklist forcing API Specialist engagement when endpoints change **What was missed in PR #2058**: - API Specialist not engaged: POST sensor data now sets `account_id` on the resulting data source — this is an endpoint behavior change that should be reviewed for backward compatibility -- Zero agent instruction updates across all three participating agents (Architecture Specialist, Test Specialist, Review Lead) +- Zero agent instruction updates across all three participating agents (Architecture Specialist, Test Specialist, Lead) - No Coordinator invocation despite explicit user request in the original prompt **Solutions implemented**: - Architecture Specialist: Added Alembic migration checklist + DataSource domain invariants - Test Specialist: Added DataSource property testing pattern + lessons learned -- Review Lead: Added Agent Selection Checklist mapping code change types to required agents; documented 3rd recurrence of these failures +- Lead: Added Agent Selection Checklist mapping code change types to required agents; documented 3rd recurrence of these failures - Coordinator (this file): Documented case study -**Governance escalation**: The Review Lead's "Must Always Run Coordinator" requirement has now been documented in three sessions without being followed. If it fails a fourth time, consider structural changes — e.g., making Coordinator invocation the FIRST step of a session rather than the last, so it sets context rather than being a forgotten epilogue. +**Governance escalation**: The Lead's "Must Always Run Coordinator" requirement has now been documented in three sessions without being followed. If it fails a fourth time, consider structural changes — e.g., making Coordinator invocation the FIRST step of a session rather than the last, so it sets context rather than being a forgotten epilogue. **Code observation from PR #2058 worth tracking**: - An early draft used `user.account_id or (user.account.id if user.account else None)` — the `or` pattern is fragile for `account_id=0` (unrealistic but worth noting). The final implementation correctly uses `if user.account_id is not None` (see `data_sources.py` lines 340-343) — this is the right pattern to follow. @@ -641,7 +641,7 @@ When reviewing schema changes that affect FK constraints: **Pattern**: Systemic self-improvement failure across all agents -**Observation**: Five agents completed substantial work (Architecture, API, Test, Documentation, Review Lead): +**Observation**: Five agents completed substantial work (Architecture, API, Test, Documentation, Lead): - Created new API endpoints (3 POST endpoints) - Wrote 17 comprehensive test functions - Created 494-line feature guide documentation @@ -658,17 +658,17 @@ When reviewing schema changes that affect FK constraints: **Root causes identified**: 1. **Self-improvement not enforced**: No blocking requirement, agents treat as optional 2. **Unclear triggers**: Agents don't know when to update instructions ("after completing work" too vague) -3. **No verification**: Review Lead doesn't check if agents self-improved +3. **No verification**: Lead doesn't check if agents self-improved 4. **Invisible requirement**: Self-improvement not in task completion checklist **Secondary violations observed**: - Temporary file committed (`API_REVIEW_ANNOTATIONS.md`, 575 lines) then removed - Non-atomic commits mixing multiple concerns - Test claims without execution evidence -- Review Lead didn't invoke Coordinator despite governance request +- Lead didn't invoke Coordinator despite governance request **Solution implemented**: -1. Added self-improvement enforcement to Review Lead checklist (see below) +1. Added self-improvement enforcement to Lead checklist (see below) 2. Documented temporary file prevention patterns 3. Added test execution evidence requirement 4. Strengthened Coordinator invocation triggers @@ -681,14 +681,14 @@ When reviewing schema changes that affect FK constraints: **Future sessions**: Monitor whether self-improvement enforcement works. If pattern recurs 3+ times, escalate to architectural solution (e.g., automated checks, mandatory prompts). -**Session 2026-02-10 (Annotation API Tests)**: Pattern recurred despite Review Lead update. Test Specialist fixed 32 annotation API tests (100% passing), made excellent technical commits, but did NOT update instructions with learned lessons (permission semantics, fixture selection, error code expectations). Review Lead enforcement unclear—may not have been involved in session. **Status**: Pattern persists. Approaching threshold for architectural solution. +**Session 2026-02-10 (Annotation API Tests)**: Pattern recurred despite Lead update. Test Specialist fixed 32 annotation API tests (100% passing), made excellent technical commits, but did NOT update instructions with learned lessons (permission semantics, fixture selection, error code expectations). Lead enforcement unclear—may not have been involved in session. **Status**: Pattern persists. Approaching threshold for architectural solution. ### Enforcement Mechanism Added -**New requirement for Review Lead**: Before marking task complete, verify: +**New requirement for Lead**: Before marking task complete, verify: ```markdown -## Task Completion Checklist (Review Lead) +## Task Completion Checklist (Lead) - [ ] Code review completed and feedback addressed - [ ] Security scan completed and alerts investigated @@ -698,7 +698,7 @@ When reviewing schema changes that affect FK constraints: - [ ] No temporary analysis files committed ``` -If any agent hasn't self-improved, Review Lead must: +If any agent hasn't self-improved, Lead must: 1. Request agent update their instructions 2. Wait for update 3. Review update for quality diff --git a/.github/agents/test-specialist.md b/.github/agents/test-specialist.md index 4e638cc154..9862364645 100644 --- a/.github/agents/test-specialist.md +++ b/.github/agents/test-specialist.md @@ -118,11 +118,11 @@ If ANY test fails during full suite execution: **Click context errors**: - Check IdField decorators (`@with_appcontext` vs `@with_appcontext_if_needed()`) - Compare against SensorIdField pattern -- See Review Lead's Click context error pattern +- See Lead's Click context error pattern -### Integration with Review Lead +### Integration with Lead -The Test Specialist MUST provide evidence of full test suite execution to Review Lead. +The Test Specialist MUST provide evidence of full test suite execution to Lead. **Required evidence format:** ``` @@ -134,14 +134,14 @@ Full test suite execution: - Coverage: 87.2% (unchanged) ``` -**Review Lead verification:** -Review Lead's session close checklist includes: +**Lead verification:** +Lead's session close checklist includes: - [ ] Test Specialist confirmed full test suite execution - [ ] All tests pass (100%) - [ ] Test output captured and reviewed **Enforcement:** -Review Lead cannot close session until Test Specialist provides evidence of full test suite execution with 100% pass rate. +Lead cannot close session until Test Specialist provides evidence of full test suite execution with 100% pass rate. ## Testing Patterns for flexmeasures diff --git a/.github/agents/tooling-ci-specialist.md b/.github/agents/tooling-ci-specialist.md index fb9d178a2f..b30995e71c 100644 --- a/.github/agents/tooling-ci-specialist.md +++ b/.github/agents/tooling-ci-specialist.md @@ -163,12 +163,12 @@ Code that bypasses pre-commit: **Who runs pre-commit:** - **During code changes**: Agent making changes runs pre-commit before committing -- **Before PR close**: Review Lead verifies pre-commit execution +- **Before PR close**: Lead verifies pre-commit execution - **In PR review**: Tooling & CI Specialist validates config matches CI **Enforcement:** -- Review Lead's session close checklist includes pre-commit verification -- Review Lead cannot close session without pre-commit evidence +- Lead's session close checklist includes pre-commit verification +- Lead cannot close session without pre-commit evidence - If pre-commit fails, agent must fix all issues before proceeding #### Common Failures and Fixes @@ -206,9 +206,9 @@ black . ci/run_mypy.sh ``` -#### Integration with Review Lead +#### Integration with Lead -**Review Lead checklist items:** +**Lead checklist items:** - [ ] Pre-commit hooks installed - [ ] All hooks pass: `pre-commit run --all-files` - [ ] Zero failures from flake8, black, mypy @@ -219,7 +219,7 @@ ci/run_mypy.sh - Or confirm: "Pre-commit verified: all hooks passed" **Enforcement:** -Review Lead MUST verify pre-commit execution before closing session. +Lead MUST verify pre-commit execution before closing session. ### Agent Environment Setup diff --git a/AGENTS.md b/AGENTS.md index 108b0832c3..a752091904 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ This avoids "agent spam" and ensures unified results. ## Quick Navigation for Critical Sections -**Before starting ANY session, Review Lead MUST consult:** +**Before starting ANY session, Lead MUST consult:** 1. **Parse user intent** → Section 1.1 (Request Interpretation) 2. **Check delegation requirements** → Section 2.1 (Mandatory Delegation Triggers) @@ -110,7 +110,7 @@ The Lead: ### 1.1. Parse User Intent (FIRST STEP - ALWAYS DO THIS) -**Before selecting agents or doing ANY work, Review Lead MUST verify understanding.** +**Before selecting agents or doing ANY work, Lead MUST verify understanding.** This prevents misinterpreting requests and working on the wrong thing. @@ -162,11 +162,11 @@ Or do you want me to [Y] instead?" User: "migrate endpoints to /api/v3_0/accounts//annotations" ❌ **Wrong interpretation:** User wants confirmation of their migration -→ Review Lead confirms work, doesn't do migration +→ Lead confirms work, doesn't do migration → User: "That was rather useless... you basically ignored my request" ✅ **Correct interpretation:** "migrate" = implementation verb = action request -→ Review Lead delegates to specialists to DO the migration +→ Lead delegates to specialists to DO the migration → Test Specialist, API Specialist, Documentation Specialist all participate * * * @@ -197,9 +197,9 @@ Notably: ### 2.1. Delegation Requirements (NON-NEGOTIABLE) -**The Review Lead MUST NEVER work alone on implementation tasks.** +**The Lead MUST NEVER work alone on implementation tasks.** -This is the most critical anti-pattern to avoid: Review Lead working solo instead of delegating. +This is the most critical anti-pattern to avoid: Lead working solo instead of delegating. **Mandatory Delegation Triggers:** @@ -230,9 +230,9 @@ This is the most critical anti-pattern to avoid: Review Lead working solo instea - ✅ ALL endpoint changes → Test + API + Documentation Specialists - ✅ ALL agent/process changes → Coordinator governance -**Review Lead's role in implementation:** +**Lead's role in implementation:** -The Review Lead: +The Lead: - ✅ Orchestrates specialists - ✅ Synthesizes their findings - ✅ Manages coordination @@ -255,13 +255,13 @@ Correct pattern: - [ ] Test Specialist made code changes and verified tests ✅ - [ ] API Specialist reviewed backward compatibility ✅ - [ ] Documentation Specialist updated docs ✅ -- [ ] Review Lead synthesized findings ✅ +- [ ] Lead synthesized findings ✅ **Example from session 2026-02-08 (failure):** User: "migrate endpoints to /api/v3_0/accounts//annotations" -❌ **What Review Lead did:** +❌ **What Lead did:** - Migrated AccountAPI, AssetAPI, SensorAPI endpoints ALONE - Updated test URLs ALONE - Ran pre-commit hooks ALONE @@ -270,7 +270,7 @@ User: "migrate endpoints to /api/v3_0/accounts//annotations" ❌ **Result:** User: "You are regressing. You must handle my requests as a team" -✅ **What Review Lead should have done:** +✅ **What Lead should have done:** ```python # Delegate to Test Specialist task(agent_type="test-specialist", @@ -424,9 +424,9 @@ The coordinator will: ### Must Enforce Agent Self-Improvement (CRITICAL) -**The Review Lead MUST ensure all participating agents update their instructions.** +**The Lead MUST ensure all participating agents update their instructions.** -This is the Review Lead's responsibility from the coordinator's governance review. When agents complete work, the Review Lead must: +This is the Lead's responsibility from the coordinator's governance review. When agents complete work, the Lead must: #### 1. Identify Participating Agents After work is complete, identify which agents contributed: @@ -487,11 +487,11 @@ If an agent doesn't update or provides insufficient update: - Instructions stay current and relevant **Example from Session 2026-02-10:** -- 5 agents participated (Architecture, API, Test, Documentation, Review Lead) -- Only Review Lead updated instructions initially +- 5 agents participated (Architecture, API, Test, Documentation, Lead) +- Only Lead updated instructions initially - Coordinator flagged 100% failure rate -- Review Lead should have prompted all 4 other agents -- Review Lead should have verified updates before closing +- Lead should have prompted all 4 other agents +- Lead should have verified updates before closing ### Must Not Create PRs Prematurely @@ -944,13 +944,13 @@ Before completing an assignment and closing the session: ### Regression Prevention (CRITICAL) -**The Review Lead can backslide to solo execution mode.** +**The Lead can backslide to solo execution mode.** This is the primary failure pattern observed in session 2026-02-08. **What regression looks like:** -When Review Lead starts working alone instead of delegating to specialists: +When Lead starts working alone instead of delegating to specialists: - Writing code directly - Updating tests without Test Specialist - Modifying docs without Documentation Specialist @@ -967,9 +967,9 @@ When Review Lead starts working alone instead of delegating to specialists: **Regression indicators (how to detect):** -- 🚩 Review Lead making code commits (should be specialist commits) -- 🚩 Review Lead updating tests (should be Test Specialist) -- 🚩 Review Lead modifying docs (should be Documentation Specialist) +- 🚩 Lead making code commits (should be specialist commits) +- 🚩 Lead updating tests (should be Test Specialist) +- 🚩 Lead modifying docs (should be Documentation Specialist) - 🚩 User says "You are regressing" - 🚩 User says "You must handle my requests as a team" - 🚩 Session closes without specialist involvement @@ -1013,40 +1013,40 @@ Ask these questions before ANY work execution: **The correct workflow:** 1. User requests implementation -2. Review Lead parses intent (section 1.1) -3. Review Lead identifies required specialists (section 2.1) -4. **Review Lead delegates to specialists** ← THIS IS THE JOB +2. Lead parses intent (section 1.1) +3. Lead identifies required specialists (section 2.1) +4. **Lead delegates to specialists** ← THIS IS THE JOB 5. Specialists do the actual work -6. Review Lead synthesizes findings -7. Review Lead runs session close checklist +6. Lead synthesizes findings +7. Lead runs session close checklist **Example from session 2026-02-08 (regression case study):** **Request:** "migrate endpoints to /api/v3_0/accounts//annotations" -**What Review Lead did (WRONG):** +**What Lead did (WRONG):** ``` -✗ Review Lead migrated AccountAPI endpoints -✗ Review Lead updated AssetAPI endpoints -✗ Review Lead modified SensorAPI endpoints -✗ Review Lead changed test URLs -✗ Review Lead ran pre-commit hooks +✗ Lead migrated AccountAPI endpoints +✗ Lead updated AssetAPI endpoints +✗ Lead modified SensorAPI endpoints +✗ Lead changed test URLs +✗ Lead ran pre-commit hooks ✗ NO specialist involvement ``` **User response:** "You are regressing. You must handle my requests as a team" -**What Review Lead should have done (CORRECT):** +**What Lead should have done (CORRECT):** ``` -✓ Review Lead parsed intent: Implementation request -✓ Review Lead identified specialists needed: +✓ Lead parsed intent: Implementation request +✓ Lead identified specialists needed: - Test Specialist (test URL updates) - API Specialist (backward compatibility) - Documentation Specialist (doc updates) -✓ Review Lead delegated to each specialist +✓ Lead delegated to each specialist ✓ Specialists did the actual work -✓ Review Lead synthesized findings +✓ Lead synthesized findings ✓ Team-based execution ``` @@ -1054,7 +1054,7 @@ Ask these questions before ANY work execution: "Simple task" is a cognitive trap. **NO task is too simple to delegate.** -The Review Lead's job is orchestration, not execution. +The Lead's job is orchestration, not execution. ### Learning from Failures @@ -1105,7 +1105,7 @@ Track and document when the Lead: **Specific lesson learned (2026-02-10 follow-up)**: - **Session**: Implementing coordinator's governance review recommendations -- **Failure**: Review Lead updated own instructions but didn't ensure other agents did the same +- **Failure**: Lead updated own instructions but didn't ensure other agents did the same - **What went wrong**: Didn't take ownership of follow-through on coordinator recommendations - **Impact**: 4 out of 5 participating agents didn't update their instructions (80% failure rate) - **Root cause**: No enforcement mechanism; assumed agents would self-update without prompting @@ -1116,7 +1116,7 @@ Track and document when the Lead: 3. Verify updates are substantive and committed 4. Re-prompt if necessary 5. Don't close session until all agents have updated -- **Key insight**: "Review Lead owns follow-through on coordinator recommendations" +- **Key insight**: "Lead owns follow-through on coordinator recommendations" - **Test execution learning**: Test specialist couldn't run tests because PostgreSQL setup was skipped; must follow copilot-setup-steps.yml workflow **Specific lesson learned (2026-02-10 test fixes)**: @@ -1152,24 +1152,24 @@ Track and document when the Lead: **Specific lesson learned (2026-02-08 endpoint migration)**: - **Session**: Annotation API endpoint migration (flat to nested RESTful pattern) -- **Failures identified**: Review Lead worked solo instead of delegating to specialists +- **Failures identified**: Lead worked solo instead of delegating to specialists - **Root cause**: Treated "simple" endpoint URL changes as not requiring delegation - **Impact**: User intervention required ("You are regressing. You must handle my requests as a team") - **Failure pattern**: 1. User: "migrate endpoints to /api/v3_0/accounts//annotations" - 2. Review Lead misunderstood as confirmation request (Failure #1) + 2. Lead misunderstood as confirmation request (Failure #1) 3. User corrected: "That was rather useless... you basically ignored my request" - 4. Review Lead did entire migration alone without delegation (Failure #2): + 4. Lead did entire migration alone without delegation (Failure #2): - Migrated AccountAPI, AssetAPI, SensorAPI endpoints - Updated test URLs - Ran pre-commit hooks - NO delegation to Test/API/Documentation specialists 5. User: "You are regressing. You must handle my requests as a team" - 6. Review Lead then properly delegated after explicit user checklist + 6. Lead then properly delegated after explicit user checklist - **Key insights**: - "Simple task" is a cognitive trap that triggers solo execution mode - - NO task is too simple to delegate - delegation is the Review Lead's core job - - Regression pattern: Review Lead forgets team-based model under time pressure + - NO task is too simple to delegate - delegation is the Lead's core job + - Regression pattern: Lead forgets team-based model under time pressure - Request interpretation MUST happen before work starts - **Prevention**: Added sections to this file: 1. **Request Interpretation** (Section 1.1) - Parse intent before work @@ -1177,25 +1177,25 @@ Track and document when the Lead: 3. **Regression Prevention** - How to detect and correct backsliding 4. **Delegation Verification** - Session close checklist item 5. **Quick Navigation** - Prominent links to critical sections -- **Verification**: Review Lead must now answer "Am I working solo?" before ANY execution +- **Verification**: Lead must now answer "Am I working solo?" before ANY execution Update this file to prevent repeating the same mistakes. ## Session Close Checklist (MANDATORY) -**Before closing ANY session, the Review Lead MUST verify ALL items in this checklist.** +**Before closing ANY session, the Lead MUST verify ALL items in this checklist.** This is non-negotiable. Skipping items without explicit justification and user approval is a governance failure. ### Delegation Verification (CRITICAL - NEW) -**Before closing session, verify Review Lead did NOT work solo:** +**Before closing session, verify Lead did NOT work solo:** - [ ] **Task type identified**: Code/API/docs/time/performance/governance changes -- [ ] **Specialists involved**: Appropriate specialists were invoked (not Review Lead alone) +- [ ] **Specialists involved**: Appropriate specialists were invoked (not Lead alone) - [ ] **Evidence of delegation**: Show task() calls that invoked specialists -- [ ] **No solo execution**: Review Lead did NOT make code/API/docs changes itself +- [ ] **No solo execution**: Lead did NOT make code/API/docs changes itself - [ ] **Synthesis provided**: Combined specialist findings into unified output **Evidence required:** @@ -1205,7 +1205,7 @@ List which specialists were invoked and what each did: ✓ Test Specialist - Updated test URLs, verified 32 tests pass ✓ API Specialist - Verified backward compatibility ✓ Documentation Specialist - Updated API docs with new structure -✓ Review Lead - Synthesized findings, managed coordination +✓ Lead - Synthesized findings, managed coordination ``` **FORBIDDEN patterns (immediate governance failure):** @@ -1213,14 +1213,14 @@ List which specialists were invoked and what each did: - ❌ "I handled it myself" (regression to solo mode) - ❌ "Too simple to delegate" (invalid justification) - ❌ "No specialists needed" (delegation always needed for code/API/docs) -- ❌ Review Lead commits containing code changes (should be specialist commits) -- ❌ Review Lead commits containing test changes (should be Test Specialist) -- ❌ Review Lead commits containing doc changes (should be Documentation Specialist) +- ❌ Lead commits containing code changes (should be specialist commits) +- ❌ Lead commits containing test changes (should be Test Specialist) +- ❌ Lead commits containing doc changes (should be Documentation Specialist) **Git commit check:** ```bash -git log --oneline -10 --author="Review Lead" +git log --oneline -10 --author="Lead" ``` Should show ONLY: @@ -1299,7 +1299,7 @@ This is a regression (see Regression Prevention section). You MUST: ### Enforcement -**The Review Lead MUST NOT close a session until ALL checklist items are verified.** +**The Lead MUST NOT close a session until ALL checklist items are verified.** If you cannot complete an item: 1. Document why in session notes @@ -1309,7 +1309,7 @@ If you cannot complete an item: If you close without completing checklist: - This is a governance failure - Coordinator will document it -- Review Lead instructions will be updated to prevent recurrence +- Lead instructions will be updated to prevent recurrence ### Continuous Improvement From 86d3ebb0e77e734d65a2564b85c23fd3a6f52b01 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 2 Apr 2026 15:26:44 +0200 Subject: [PATCH 42/63] db: fix merge migration to properly resolve conflicting heads Update b2e07f0dafa1_merge.py migration to specify both parent revisions that are being merged: - 3f4a6f9d2b11 (increase_audit_event_length_to_500) - b2c3d4e5f6a7 (Drop FK constraint from audit_log.active_user_id) This resolves the multiple heads issue and creates a proper mergepoint in the migration history. The migration now correctly shows as a (mergepoint) in the history output. --- .../migrations/versions/b2e07f0dafa1_merge.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py diff --git a/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py b/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py new file mode 100644 index 0000000000..8875f38e5c --- /dev/null +++ b/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py @@ -0,0 +1,25 @@ +"""merge + +Revision ID: b2e07f0dafa1 +Revises: +Create Date: 2026-04-02 15:21:56.334943 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "b2e07f0dafa1" +down_revision = ("3f4a6f9d2b11", "b2c3d4e5f6a7") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From 7d5fa60947479640f125dec846d6b870c9353bc5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 2 Apr 2026 15:33:26 +0200 Subject: [PATCH 43/63] agents: remove merge conflict markers from coordinator.md Clean up leftover git merge conflict markers (<<<<<<< HEAD, =======, >>>>>>> origin/main) that were not properly resolved. Both pattern sections (2026-03-24 and 2026-02-10) have been preserved as they document complementary learning from different sessions. --- .github/agents/coordinator.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/agents/coordinator.md b/.github/agents/coordinator.md index b3e0bbc871..2731a33bfa 100644 --- a/.github/agents/coordinator.md +++ b/.github/agents/coordinator.md @@ -584,7 +584,6 @@ If Lead repeatedly closes sessions without completing checklist: These patterns must not repeat. Agent instructions have been updated to prevent recurrence. -<<<<<<< HEAD ### Additional Pattern Discovered (2026-03-24) **Pattern**: Persistent self-improvement failure and missing API Specialist agent selection @@ -705,4 +704,3 @@ If any agent hasn't self-improved, Lead must: 4. Then mark task complete **This makes self-improvement blocking, not optional.** ->>>>>>> origin/main From 4724959678efc2a38ef3508bfeb57271c133f7f9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 2 Apr 2026 15:36:52 +0200 Subject: [PATCH 44/63] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py b/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py index 8875f38e5c..7daab17abf 100644 --- a/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py +++ b/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py @@ -1,15 +1,11 @@ """merge Revision ID: b2e07f0dafa1 -Revises: +Revises: 3f4a6f9d2b11, b2c3d4e5f6a7 Create Date: 2026-04-02 15:21:56.334943 """ -from alembic import op -import sqlalchemy as sa - - # revision identifiers, used by Alembic. revision = "b2e07f0dafa1" down_revision = ("3f4a6f9d2b11", "b2c3d4e5f6a7") From 60a4b502d196fb2c998081c6244e3a95c81a03c3 Mon Sep 17 00:00:00 2001 From: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Date: Sun, 5 Apr 2026 20:24:35 +0200 Subject: [PATCH 45/63] docs: clarify why we add a creation audit log record in a test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Nicolas Höning Signed-off-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> --- flexmeasures/cli/tests/test_data_delete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/cli/tests/test_data_delete.py b/flexmeasures/cli/tests/test_data_delete.py index 9b15d13a30..7fd9a2b558 100644 --- a/flexmeasures/cli/tests/test_data_delete.py +++ b/flexmeasures/cli/tests/test_data_delete.py @@ -31,7 +31,7 @@ def test_delete_account( (ds.id, ds.user_id, ds.account_id) for ds in data_sources_before ] - # Add creation audit log record + # Add creation audit log record, as that has not automatically been done when setting up test data user_creation_audit_log = AuditLog( event="User Test Prosumer User created test", affected_user_id=prosumer.id, From 3adfed6a7ff2a4583b674dd5a610a09953fdb550 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 02:31:29 +0100 Subject: [PATCH 46/63] docs: fix pr number in changelog entry Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index a368160a7c..4f3c233692 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -36,7 +36,7 @@ Infrastructure / Support Bugfixes ----------- -* Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2058 `_] +* Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_] v0.31.2 | March 18, 2026 From c865dbb8704a0a9b598a92a49bc65d53277648e3 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 02:53:27 +0100 Subject: [PATCH 47/63] feat: rename sensor data account filter parameter --- flexmeasures/api/common/schemas/sensor_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 44d290d217..33ce852b22 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -153,7 +153,7 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): class GetSensorDataSchema(SensorDataDescriptionSchema): resolution = DurationField(required=False) source = SourceIdField(required=False) - source_account = AccountIdField(required=False, data_key="source_account_id") + account = AccountIdField(required=False) # Optional field that can be used for extra validation type = fields.Str( @@ -204,7 +204,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: unit = sensor_data_description["unit"] resolution = sensor_data_description.get("resolution") source = sensor_data_description.get("source") - source_account = sensor_data_description.get("source_account") + account = sensor_data_description.get("account") # Post-load configuration of event frequency if resolution is None: @@ -234,7 +234,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: horizons_at_least=horizons_at_least, horizons_at_most=horizons_at_most, source=source, - source_account_ids=source_account.id if source_account else None, + source_account_ids=account.id if account else None, beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, resolution=resolution, @@ -273,7 +273,7 @@ class GetSensorDataQuerySchema(SensorDataTimingDescriptionSchema): resolution = DurationField(required=False) source = SourceIdField(required=False) - source_account = AccountIdField(required=False, data_key="source_account_id") + account = AccountIdField(required=False) class PostSensorDataSchema(SensorDataDescriptionSchema): From 2734a9702a30d24d9193f9bc54ef958ab02aa971 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 02:53:45 +0100 Subject: [PATCH 48/63] feat: align sensor data docs with account filter parameter --- documentation/api/notation.rst | 2 +- flexmeasures/api/v3_0/sensors.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index ee4ac6f9bd..dd956c8c23 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -277,7 +277,7 @@ Data source IDs can be found by hovering over data in charts. For the ``GET /api/v3_0/sensors//data`` endpoint specifically, source filtering supports: - ``source``: filter by data source ID -- ``source_account_id``: filter by the account ID linked to data sources +- ``account``: filter by the account ID linked to data sources Filtering that endpoint by source type is currently not supported. diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 76491d856c..f8235c7c9b 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -590,7 +590,7 @@ def get_data(self, id: int, **sensor_data_description: dict): - "horizon" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs)) - "prior" (the belief timing docs also apply here) - "source" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources)) - - "source_account_id" (filter by the account ID linked to data sources) + - "account" (filter by the account ID linked to data sources) An example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m³/h: From 62de0cb26154f9c60dd977ce562bbc53cb812142 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 02:54:05 +0100 Subject: [PATCH 49/63] test: update sensor data account filter request --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 3b20dc02d6..c7f3579ce7 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -133,7 +133,7 @@ def test_get_sensor_data_filtered_by_source_account( "duration": "PT1H20M", "horizon": "PT0H", "unit": "m³/h", - "source_account_id": source_user.account_id, + "account": source_user.account_id, "resolution": "PT20M", } response = client.get( From 5f57b7238c6f18cc895c8919385a370620a8ba13 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 02:54:22 +0100 Subject: [PATCH 50/63] feat: regenerate OpenAPI specs for account filter parameter --- flexmeasures/ui/static/openapi-specs.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 159f416dc9..b954a3df90 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "0.31.0" + "version": "0.32.0" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -339,7 +339,7 @@ }, "get": { "summary": "Get sensor data", - "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"source_account_id\" (filter by the account ID linked to data sources)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > **Note:** This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n", + "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"account\" (filter by the account ID linked to data sources)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > Note: This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n", "security": [ { "ApiKeyAuth": [] @@ -427,7 +427,7 @@ }, { "in": "query", - "name": "source_account_id", + "name": "account", "schema": { "type": "integer" }, From 5457e44f1306db4f80c2bb12887c053b764c2f14 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 02:57:51 +0100 Subject: [PATCH 51/63] feat: use kwargs in source criteria call --- flexmeasures/data/models/time_series.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/time_series.py b/flexmeasures/data/models/time_series.py index db9912f786..176bdbdaec 100644 --- a/flexmeasures/data/models/time_series.py +++ b/flexmeasures/data/models/time_series.py @@ -920,11 +920,11 @@ def search( parsed_sources = parse_source_arg(source) source_criteria = get_source_criteria( - cls, - user_source_ids, - source_account_ids, - source_types, - exclude_source_types, + cls=cls, + user_source_ids=user_source_ids, + source_account_ids=source_account_ids, + source_types=source_types, + exclude_source_types=exclude_source_types, ) custom_join_targets = [] if parsed_sources else [DataSource] From 20cca6af0f194b1ff34da882a5b88908fd2d82ac Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 03:01:30 +0100 Subject: [PATCH 52/63] feat: add metadata to sensor data query schema Signed-off-by: Mohamed Belhsan Hmida --- .../api/common/schemas/sensor_data.py | 24 ++++++++++++++++--- flexmeasures/ui/static/openapi-specs.json | 10 ++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 33ce852b22..8aa66857ac 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -271,9 +271,27 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: class GetSensorDataQuerySchema(SensorDataTimingDescriptionSchema): """Document the actual query parameters for GET /sensors//data.""" - resolution = DurationField(required=False) - source = SourceIdField(required=False) - account = AccountIdField(required=False) + resolution = DurationField( + required=False, + metadata=dict( + description="Resolution of the returned sensor data in ISO 8601 duration format.", + example="PT15M", + ), + ) + source = SourceIdField( + required=False, + metadata=dict( + description="Filter by a specific data source ID.", + example=42, + ), + ) + account = AccountIdField( + required=False, + metadata=dict( + description="Filter by the account linked to data sources.", + example=3, + ), + ) class PostSensorDataSchema(SensorDataDescriptionSchema): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b954a3df90..695bb6d63e 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -411,8 +411,10 @@ { "in": "query", "name": "resolution", + "description": "Resolution of the returned sensor data in ISO 8601 duration format.", "schema": { "type": "string", + "example": "PT15M", "format": "duration" }, "required": false @@ -420,16 +422,20 @@ { "in": "query", "name": "source", + "description": "Filter by a specific data source ID.", "schema": { - "type": "integer" + "type": "integer", + "example": 42 }, "required": false }, { "in": "query", "name": "account", + "description": "Filter by the account linked to data sources.", "schema": { - "type": "integer" + "type": "integer", + "example": 3 }, "required": false } From 28c0d946144082d2951986f419edc8a78d5848a6 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 03:17:11 +0100 Subject: [PATCH 53/63] feat: share sensor data query filters in a mixin Signed-off-by: Mohamed Belhsan Hmida --- .../api/common/schemas/sensor_data.py | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 8aa66857ac..6f10a5380b 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -150,10 +150,33 @@ def check_schema_unit_against_sensor_unit(self, data, **kwargs): ) -class GetSensorDataSchema(SensorDataDescriptionSchema): - resolution = DurationField(required=False) - source = SourceIdField(required=False) - account = AccountIdField(required=False) +class GetSensorDataFilterSchemaMixin: + """Shared filters for GET sensor data request parsing and docs.""" + + resolution = DurationField( + required=False, + metadata=dict( + description="Resolution of the returned sensor data in ISO 8601 duration format.", + example="PT15M", + ), + ) + source = SourceIdField( + required=False, + metadata=dict( + description="Filter by a specific data source ID.", + example=42, + ), + ) + account = AccountIdField( + required=False, + metadata=dict( + description="Filter by the account linked to data sources.", + example=3, + ), + ) + + +class GetSensorDataSchema(GetSensorDataFilterSchemaMixin, SensorDataDescriptionSchema): # Optional field that can be used for extra validation type = fields.Str( @@ -268,31 +291,11 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: return response -class GetSensorDataQuerySchema(SensorDataTimingDescriptionSchema): +class GetSensorDataQuerySchema( + GetSensorDataFilterSchemaMixin, SensorDataTimingDescriptionSchema +): """Document the actual query parameters for GET /sensors//data.""" - resolution = DurationField( - required=False, - metadata=dict( - description="Resolution of the returned sensor data in ISO 8601 duration format.", - example="PT15M", - ), - ) - source = SourceIdField( - required=False, - metadata=dict( - description="Filter by a specific data source ID.", - example=42, - ), - ) - account = AccountIdField( - required=False, - metadata=dict( - description="Filter by the account linked to data sources.", - example=3, - ), - ) - class PostSensorDataSchema(SensorDataDescriptionSchema): """ From 9b61895beed410320fdcb4cbdb73fa4b060c6b93 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 7 Apr 2026 04:57:48 +0100 Subject: [PATCH 54/63] feat: add source account ids to search schemas Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/io.py | 2 ++ flexmeasures/data/schemas/reporting/__init__.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/flexmeasures/data/schemas/io.py b/flexmeasures/data/schemas/io.py index afecb4b5b9..4187eab5db 100644 --- a/flexmeasures/data/schemas/io.py +++ b/flexmeasures/data/schemas/io.py @@ -1,6 +1,7 @@ from marshmallow import fields, Schema, post_load, post_dump from flexmeasures.data.schemas.sensors import SensorIdField +from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.data.schemas import AwareDateTimeField, DurationField from flexmeasures.data.schemas.sources import DataSourceIdField from flask import current_app @@ -35,6 +36,7 @@ class Input(Schema): horizons_at_most = DurationField() user_source_ids = fields.List(DataSourceIdField()) + source_account_ids = fields.List(AccountIdField()) source_types = fields.List(fields.Str()) exclude_source_types = fields.List(fields.Str()) most_recent_beliefs_only = fields.Boolean() diff --git a/flexmeasures/data/schemas/reporting/__init__.py b/flexmeasures/data/schemas/reporting/__init__.py index 94257599ef..b22180930f 100644 --- a/flexmeasures/data/schemas/reporting/__init__.py +++ b/flexmeasures/data/schemas/reporting/__init__.py @@ -1,5 +1,6 @@ from marshmallow import Schema, fields, validate +from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.data.schemas.sources import DataSourceIdField from flexmeasures.data.schemas import AwareDateTimeField, DurationField @@ -55,6 +56,7 @@ class BeliefsSearchConfigSchema(Schema): horizons_at_most = DurationField() source = DataSourceIdField() + source_account_ids = fields.List(AccountIdField()) source_types = fields.List(fields.Str()) exclude_source_types = fields.List(fields.Str()) From bf5bf7571be379d8e5fffc7227002ca64e1b029a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Apr 2026 10:58:40 +0000 Subject: [PATCH 55/63] migration: merge drop-FK migration into add-account-id migration (9877450113f6) Remove the intermediate a1b2c3d4e5f6 migration that was dropping the account_id FK (only created within this PR) and the user_id FK. Instead: - Don't create the account_id FK in 9877450113f6 in the first place - Absorb the user_id FK drop into 9877450113f6 - Update b2c3d4e5f6a7 to chain off 9877450113f6 directly - Delete a1b2c3d4e5f6 Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/4d9d8a92-d4fd-4635-bebf-1965c35667b3 Co-authored-by: nhoening <1042336+nhoening@users.noreply.github.com> --- ...7450113f6_add_account_id_to_data_source.py | 44 +++++++++----- ...f6_drop_fk_constraints_from_data_source.py | 58 ------------------- ...onstraint_from_audit_log_active_user_id.py | 2 +- 3 files changed, 32 insertions(+), 72 deletions(-) delete mode 100644 flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py diff --git a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py index dde9f1c0bf..227f0e751e 100644 --- a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py +++ b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py @@ -1,4 +1,9 @@ -"""Add account_id to data_source table +"""Add account_id to data_source table and drop FK constraints for lineage preservation + +Adds account_id to data_source (without a DB-level FK constraint, so that referenced +accounts can be deleted while the historical account_id value is preserved for lineage). +Also drops the existing user_id FK constraint for the same reason: when a user is deleted, +the data_source.user_id should remain intact rather than being cascaded or nullified. Revision ID: 9877450113f6 Revises: 8b62f8129f34 @@ -8,6 +13,7 @@ from alembic import op import sqlalchemy as sa +from sqlalchemy import inspect # revision identifiers, used by Alembic. @@ -34,15 +40,9 @@ def upgrade(): - # 1. Add the account_id column (nullable) + # 1. Add the account_id column (nullable, no DB-level FK so lineage is preserved) with op.batch_alter_table("data_source", schema=None) as batch_op: batch_op.add_column(sa.Column("account_id", sa.Integer(), nullable=True)) - batch_op.create_foreign_key( - op.f("data_source_account_id_account_fkey"), - "account", - ["account_id"], - ["id"], - ) # 2. Data migration: populate account_id from the related user's account. # Use a correlated subquery to avoid N+1 queries and ensure portability. @@ -65,9 +65,30 @@ def upgrade(): ["name", "user_id", "account_id", "model", "version", "attributes_hash"], ) + # 4. Drop the user_id FK constraint so that deleting a user preserves the lineage + # reference in data_source rows (no cascade, no SET NULL). + bind = op.get_bind() + inspector = inspect(bind) + existing_fks = inspector.get_foreign_keys("data_source") + existing_fk_names = [fk["name"] for fk in existing_fks] + with op.batch_alter_table("data_source", schema=None) as batch_op: + for fk_name in existing_fk_names: + if "user_id" in fk_name: + batch_op.drop_constraint(fk_name, type_="foreignkey") + break + def downgrade(): - # 1. Restore the original UniqueConstraint without account_id + # 1. Re-add the user_id FK constraint + with op.batch_alter_table("data_source", schema=None) as batch_op: + batch_op.create_foreign_key( + "data_source_user_id_fkey", + "fm_user", + ["user_id"], + ["id"], + ) + + # 2. Restore the original UniqueConstraint without account_id with op.batch_alter_table("data_source", schema=None) as batch_op: batch_op.drop_constraint("data_source_name_key", type_="unique") batch_op.create_unique_constraint( @@ -75,9 +96,6 @@ def downgrade(): ["name", "user_id", "model", "version", "attributes_hash"], ) - # 2. Drop the account_id column and its FK + # 3. Drop the account_id column with op.batch_alter_table("data_source", schema=None) as batch_op: - batch_op.drop_constraint( - op.f("data_source_account_id_account_fkey"), type_="foreignkey" - ) batch_op.drop_column("account_id") diff --git a/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py b/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py deleted file mode 100644 index 79070d5498..0000000000 --- a/flexmeasures/data/migrations/versions/a1b2c3d4e5f6_drop_fk_constraints_from_data_source.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Drop FK constraints from data_source for data lineage preservation - -When users or accounts are deleted, we want to preserve the historical -user_id and account_id values in data_source rows for lineage purposes, -rather than cascade-deleting or nullifying them. - -Revision ID: a1b2c3d4e5f6 -Revises: 9877450113f6 -Create Date: 2026-03-25 00:00:00.000000 - -""" - -from alembic import op -from sqlalchemy import inspect - - -# revision identifiers, used by Alembic. -revision = "a1b2c3d4e5f6" -down_revision = "9877450113f6" -branch_labels = None -depends_on = None - - -def upgrade(): - # Inspect existing FK constraints to handle different database states gracefully - bind = op.get_bind() - inspector = inspect(bind) - existing_fks = inspector.get_foreign_keys("data_source") - existing_fk_names = [fk["name"] for fk in existing_fks] - - with op.batch_alter_table("data_source", schema=None) as batch_op: - # Drop the account_id FK if it exists - if "data_source_account_id_account_fkey" in existing_fk_names: - batch_op.drop_constraint( - "data_source_account_id_account_fkey", type_="foreignkey" - ) - - # Drop the user_id FK if it exists (may have auto-generated name) - for fk_name in existing_fk_names: - if "user_id" in fk_name: - batch_op.drop_constraint(fk_name, type_="foreignkey") - break - - -def downgrade(): - with op.batch_alter_table("data_source", schema=None) as batch_op: - batch_op.create_foreign_key( - "data_source_user_id_fkey", - "fm_user", - ["user_id"], - ["id"], - ) - batch_op.create_foreign_key( - "data_source_account_id_account_fkey", - "account", - ["account_id"], - ["id"], - ) diff --git a/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py b/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py index 749219c46f..6eac719b54 100644 --- a/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py +++ b/flexmeasures/data/migrations/versions/b2c3d4e5f6a7_drop_fk_constraint_from_audit_log_active_user_id.py @@ -15,7 +15,7 @@ # revision identifiers, used by Alembic. revision = "b2c3d4e5f6a7" -down_revision = "a1b2c3d4e5f6" +down_revision = "9877450113f6" branch_labels = None depends_on = None From e26d199bdfec611d2cd5f26b4da7acd83322f949 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Mon, 13 Apr 2026 01:51:10 +0100 Subject: [PATCH 56/63] feat: load source account ids as integers Context: - shared search schemas were deserializing source_account_ids to Account objects - downstream source filtering expects integer account ids for SQL criteria --- flexmeasures/data/schemas/io.py | 3 +-- flexmeasures/data/schemas/reporting/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/io.py b/flexmeasures/data/schemas/io.py index 4187eab5db..bcdb47e563 100644 --- a/flexmeasures/data/schemas/io.py +++ b/flexmeasures/data/schemas/io.py @@ -1,7 +1,6 @@ from marshmallow import fields, Schema, post_load, post_dump from flexmeasures.data.schemas.sensors import SensorIdField -from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.data.schemas import AwareDateTimeField, DurationField from flexmeasures.data.schemas.sources import DataSourceIdField from flask import current_app @@ -36,7 +35,7 @@ class Input(Schema): horizons_at_most = DurationField() user_source_ids = fields.List(DataSourceIdField()) - source_account_ids = fields.List(AccountIdField()) + source_account_ids = fields.List(fields.Int()) source_types = fields.List(fields.Str()) exclude_source_types = fields.List(fields.Str()) most_recent_beliefs_only = fields.Boolean() diff --git a/flexmeasures/data/schemas/reporting/__init__.py b/flexmeasures/data/schemas/reporting/__init__.py index b22180930f..ff2f64987e 100644 --- a/flexmeasures/data/schemas/reporting/__init__.py +++ b/flexmeasures/data/schemas/reporting/__init__.py @@ -1,6 +1,5 @@ from marshmallow import Schema, fields, validate -from flexmeasures.data.schemas.account import AccountIdField from flexmeasures.data.schemas.sources import DataSourceIdField from flexmeasures.data.schemas import AwareDateTimeField, DurationField @@ -56,7 +55,7 @@ class BeliefsSearchConfigSchema(Schema): horizons_at_most = DurationField() source = DataSourceIdField() - source_account_ids = fields.List(AccountIdField()) + source_account_ids = fields.List(fields.Int()) source_types = fields.List(fields.Str()) exclude_source_types = fields.List(fields.Str()) From 880efae856937cffc39c69af822c978014575822 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Mon, 13 Apr 2026 01:53:42 +0100 Subject: [PATCH 57/63] Update flexmeasures/data/queries/utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/data/queries/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/queries/utils.py b/flexmeasures/data/queries/utils.py index 78afb5f1b0..aede33672c 100644 --- a/flexmeasures/data/queries/utils.py +++ b/flexmeasures/data/queries/utils.py @@ -84,10 +84,10 @@ def potentially_limit_assets_query_to_accounts( def get_source_criteria( cls: "Type[ts.TimedValue] | Type[ts.TimedBelief]", - user_source_ids: int | list[int], - source_account_ids: int | list[int], - source_types: list[str], - exclude_source_types: list[str], + user_source_ids: int | list[int] | None = None, + source_account_ids: int | list[int] | None = None, + source_types: list[str] | None = None, + exclude_source_types: list[str] | None = None, ) -> list[BinaryExpression]: source_criteria: list[BinaryExpression] = [] if user_source_ids is not None: From 1d39eae5ce10cfc195ec1e478f4e3afc25d824ac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 5 Apr 2026 20:29:12 +0200 Subject: [PATCH 58/63] docs: make this lengthy comment only once Signed-off-by: F.N. Claessen --- flexmeasures/data/models/audit_log.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/data/models/audit_log.py b/flexmeasures/data/models/audit_log.py index 84c9c45fd2..587e0df77f 100644 --- a/flexmeasures/data/models/audit_log.py +++ b/flexmeasures/data/models/audit_log.py @@ -30,11 +30,9 @@ class AuditLog(db.Model, AuthModelMixin): event_datetime = Column(DateTime()) event = Column(String(500)) active_user_name = Column(String(255)) - # No DB-level FK with cascade for active_user_id so that deleting a user preserves the lineage reference in this column. + # No DB-level FK with cascade for any user_id or account_id so that deleting a user preserves the lineage reference in this column. active_user_id = Column("active_user_id", Integer(), nullable=True) - # No DB-level FK with cascade for affected_user_id so that deleting a user preserves the lineage reference in this column. affected_user_id = Column("affected_user_id", Integer(), nullable=True) - # No DB-level FK with cascade for affected_account_id so that deleting an account preserves the lineage reference in this column. affected_account_id = Column("affected_account_id", Integer(), nullable=True) # Relationships to navigate to User and Account without database-level FK constraints From 2503d95bf1d0243ea1c76cbb0b04c64d50b1d4d9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sun, 5 Apr 2026 20:34:07 +0200 Subject: [PATCH 59/63] docs: add explanation for defining foreign keys in the db.relationship Signed-off-by: F.N. Claessen --- flexmeasures/data/models/audit_log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/models/audit_log.py b/flexmeasures/data/models/audit_log.py index 587e0df77f..86c0b250f6 100644 --- a/flexmeasures/data/models/audit_log.py +++ b/flexmeasures/data/models/audit_log.py @@ -37,6 +37,8 @@ class AuditLog(db.Model, AuthModelMixin): # Relationships to navigate to User and Account without database-level FK constraints # This allows audit logs to maintain references to deleted users/accounts for lineage purposes + # The foreign_keys= parameter inside db.relationship(...) is a SQLAlchemy ORM hint only — it has zero effect on the database schema. + # It's needed here because SQLAlchemy can't automatically infer which column is the "FK side" of the join when there's no actual ForeignKey() in the column definition active_user = db.relationship( "User", primaryjoin="AuditLog.active_user_id == User.id", From d43b41d914e7b0bd7cf8b30a9ad54621ba5da542 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 12:12:02 +0200 Subject: [PATCH 60/63] fix: merge db revisions Signed-off-by: F.N. Claessen --- .../migrations/versions/2c6a4c8127c1_merge.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py diff --git a/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py b/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py new file mode 100644 index 0000000000..c40b018e63 --- /dev/null +++ b/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py @@ -0,0 +1,21 @@ +"""merge + +Revision ID: 2c6a4c8127c1 +Revises: +Create Date: 2026-04-13 12:10:52.322376 + +""" + +# revision identifiers, used by Alembic. +revision = "2c6a4c8127c1" +down_revision = ("b2e07f0dafa1", "e26d02ed1621") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass From 74ed3dc63b63a38b429372a5a5a07b9448f43f6c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 12:24:35 +0200 Subject: [PATCH 61/63] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py b/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py index c40b018e63..0e3e6356dd 100644 --- a/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py +++ b/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py @@ -1,7 +1,7 @@ """merge Revision ID: 2c6a4c8127c1 -Revises: +Revises: b2e07f0dafa1, e26d02ed1621 Create Date: 2026-04-13 12:10:52.322376 """ From 16655e4313e5133d2aad28272461b116687a9da5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:52:20 +0000 Subject: [PATCH 62/63] migration: rebase add-account-id migration onto e26d02ed1621 (tip of main) Change 9877450113f6.down_revision from 8b62f8129f34 to e26d02ed1621 so the PR's two real migrations (9877450113f6, b2c3d4e5f6a7) chain linearly after the tip of the main branch, making the two merge-only files (b2e07f0dafa1 and 2c6a4c8127c1) completely obsolete. Delete b2e07f0dafa1_merge.py and 2c6a4c8127c1_merge.py. Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/94c3070e-a701-4a97-a895-9ea1064e3c20 Co-authored-by: Flix6x <30658763+Flix6x@users.noreply.github.com> --- .../migrations/versions/2c6a4c8127c1_merge.py | 21 ------------------- ...7450113f6_add_account_id_to_data_source.py | 2 +- .../migrations/versions/b2e07f0dafa1_merge.py | 21 ------------------- 3 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py delete mode 100644 flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py diff --git a/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py b/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py deleted file mode 100644 index 0e3e6356dd..0000000000 --- a/flexmeasures/data/migrations/versions/2c6a4c8127c1_merge.py +++ /dev/null @@ -1,21 +0,0 @@ -"""merge - -Revision ID: 2c6a4c8127c1 -Revises: b2e07f0dafa1, e26d02ed1621 -Create Date: 2026-04-13 12:10:52.322376 - -""" - -# revision identifiers, used by Alembic. -revision = "2c6a4c8127c1" -down_revision = ("b2e07f0dafa1", "e26d02ed1621") -branch_labels = None -depends_on = None - - -def upgrade(): - pass - - -def downgrade(): - pass diff --git a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py index 227f0e751e..f0e046d120 100644 --- a/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py +++ b/flexmeasures/data/migrations/versions/9877450113f6_add_account_id_to_data_source.py @@ -18,7 +18,7 @@ # revision identifiers, used by Alembic. revision = "9877450113f6" -down_revision = "8b62f8129f34" +down_revision = "e26d02ed1621" branch_labels = None depends_on = None diff --git a/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py b/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py deleted file mode 100644 index 7daab17abf..0000000000 --- a/flexmeasures/data/migrations/versions/b2e07f0dafa1_merge.py +++ /dev/null @@ -1,21 +0,0 @@ -"""merge - -Revision ID: b2e07f0dafa1 -Revises: 3f4a6f9d2b11, b2c3d4e5f6a7 -Create Date: 2026-04-02 15:21:56.334943 - -""" - -# revision identifiers, used by Alembic. -revision = "b2e07f0dafa1" -down_revision = ("3f4a6f9d2b11", "b2c3d4e5f6a7") -branch_labels = None -depends_on = None - - -def upgrade(): - pass - - -def downgrade(): - pass From e7139da3ffc3db37844931cca03428ccf54f1073 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 13 Apr 2026 12:44:23 +0200 Subject: [PATCH 63/63] style: punctuation / double spaces Signed-off-by: F.N. Claessen --- ...1_recompute_attributes_hash_with_sort_keys.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/migrations/versions/a5b26c3f8e91_recompute_attributes_hash_with_sort_keys.py b/flexmeasures/data/migrations/versions/a5b26c3f8e91_recompute_attributes_hash_with_sort_keys.py index 7d06e7df75..4a708932e4 100644 --- a/flexmeasures/data/migrations/versions/a5b26c3f8e91_recompute_attributes_hash_with_sort_keys.py +++ b/flexmeasures/data/migrations/versions/a5b26c3f8e91_recompute_attributes_hash_with_sort_keys.py @@ -2,22 +2,22 @@ Previously the hash was computed without sorting JSON object keys, so a PostgreSQL JSONB round-trip (which always returns keys in alphabetical order) -produced a different hash than the one stored in the database. This caused +produced a different hash than the one stored in the database. This caused get_or_create_source() to silently create duplicate DataSource rows when it was called with attributes that had been loaded back from the database. The upgrade also handles the case where the bug already produced duplicate rows (same logical content, but saved with different key-insertion-order hashes). -For each group of duplicates the newest row (highest ID) is kept as-is. Older +For each group of duplicates the newest row (highest ID) is kept as-is. Older duplicates receive a synthetic ``{"flexmeasures-hash-conflict": N}`` attribute so that their hashes remain unique without touching the timed_belief table. Downgrade note: since PostgreSQL JSONB already serialises all object keys in alphabetical order when storing, ``json.dumps(attrs)`` and ``json.dumps(attrs, sort_keys=True)`` produce identical strings for any data -that has gone through JSONB. Therefore recomputing the hash without +that has gone through JSONB. Therefore, recomputing the hash without ``sort_keys`` would yield the same bytes as the upgrade, making a downgrade -data-migration a no-op. The downgrade function is intentionally left empty. +data-migration a no-op. The downgrade function is intentionally left empty. Revision ID: a5b26c3f8e91 Revises: 8b62f8129f34 @@ -51,7 +51,7 @@ def upgrade(): Duplicate rows (same logical content, different key-order hashes) are resolved by tagging older rows with a synthetic conflict-marker attribute so - that each row still gets a unique hash. The newest row (highest id) is + that each row still gets a unique hash. The newest row (highest id) is always kept clean. """ bind = op.get_bind() @@ -64,7 +64,7 @@ def upgrade(): ).fetchall() # Group rows by their normalised unique key (name, user_id, model, version, - # sorted-attributes hash). Any group with >1 member means the bug created + # sorted-attributes hash). Any group with >1 member means the bug created # duplicate rows. groups: dict = defaultdict(list) attrs_by_id: dict = {} @@ -112,7 +112,7 @@ def downgrade(): """No data migration needed on downgrade. PostgreSQL JSONB always serialises object keys in alphabetical order when - storing. This means ``json.dumps(attrs)`` and + storing. This means ``json.dumps(attrs)`` and ``json.dumps(attrs, sort_keys=True)`` produce identical strings for any attributes that have been round-tripped through the database, so recomputing hashes without ``sort_keys`` would yield the same bytes. @@ -120,7 +120,7 @@ def downgrade(): Note: rows that were tagged with ``flexmeasures-hash-conflict`` during upgrade are NOT cleaned up here, because doing so would require knowing which rows were duplicates and which one was the "canonical" row -- that - information is not reliably recoverable. After a downgrade, those rows + information is not reliably recoverable. After a downgrade, those rows will have a slightly different ``attributes`` dict than before the upgrade, but ``get_or_create_source`` will still find them correctly via their hash. """