Merge pull request #239 from MISP/do-not-store-events-sql

refactor: do not store events/attributes/objects in sql, use opensearch

Author: righel

api/README.md

@@ -209,7 +209,7 @@ Example:
 ```
 from app.worker import tasks
 
-tasks.handle_created_attribute.delay(pulled_attribute.id, pulled_attribute.object_id, pulled_attribute.event_id)
+tasks.handle_created_attribute.delay(pulled_attribute.id, pulled_attribute.object_uuid, pulled_attribute.event_uuid)
 ```
 
 If you add a new task, you have to restart the celery `worker` container, otherwise you will get `NotRegistered('app.worker.tasks.new_task') ` exception.

api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py

@@ -0,0 +1,44 @@
+"""detach object_references FK constraints to events and objects
+
+These are the last DB-level FK constraints that reference the events and objects
+tables from an external table, blocking their eventual removal.
+
+Revision ID: c8d7e6f5a4b3
+Revises: f3e2d1c0b9a8
+Create Date: 2026-03-25 00:00:00.000000
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c8d7e6f5a4b3"
+down_revision = "f3e2d1c0b9a8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint(
+        "object_references_event_id_fkey", "object_references", type_="foreignkey"
+    )
+    op.drop_constraint(
+        "object_references_object_id_fkey", "object_references", type_="foreignkey"
+    )
+
+
+def downgrade():
+    op.create_foreign_key(
+        "object_references_event_id_fkey",
+        "object_references",
+        "events",
+        ["event_id"],
+        ["id"],
+    )
+    op.create_foreign_key(
+        "object_references_object_id_fkey",
+        "object_references",
+        "objects",
+        ["object_id"],
+        ["id"],
+    )

api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py

@@ -0,0 +1,30 @@
+"""remove event_id columns from attributes and objects tables
+
+Revision ID: d4e5f6a7b8c9
+Revises: c8d7e6f5a4b3
+Create Date: 2026-03-25 00:00:00.000000
+
+"""
+
+from alembic import op
+
+revision = "d4e5f6a7b8c9"
+down_revision = "c8d7e6f5a4b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_index("ix_attributes_event_id", table_name="attributes", if_exists=True)
+    op.drop_column("attributes", "event_id")
+    op.drop_index("ix_objects_event_id", table_name="objects", if_exists=True)
+    op.drop_column("objects", "event_id")
+
+
+def downgrade():
+    import sqlalchemy as sa
+
+    op.add_column("objects", sa.Column("event_id", sa.Integer(), nullable=True))
+    op.create_index("ix_objects_event_id", "objects", ["event_id"])
+    op.add_column("attributes", sa.Column("event_id", sa.Integer(), nullable=True))
+    op.create_index("ix_attributes_event_id", "attributes", ["event_id"])

api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py

@@ -0,0 +1,75 @@
+"""drop events table and migrate feeds.event_id to uuid string
+
+Revision ID: e1a2b3c4d5f6
+Revises: d4e5f6a7b8c9
+Create Date: 2026-03-26 00:00:00.000000
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision = "e1a2b3c4d5f6"
+down_revision = "d4e5f6a7b8c9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Replace event_id (INTEGER) with event_uuid (VARCHAR) in object_references
+    op.drop_column("object_references", "event_id")
+    op.add_column("object_references", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True))
+
+    # Replace event_id (INTEGER) with event_uuid (VARCHAR) in event_tags and attribute_tags
+    op.drop_column("event_tags", "event_id")
+    op.add_column("event_tags", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True))
+    op.drop_column("attribute_tags", "event_id")
+    op.add_column("attribute_tags", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True))
+
+    # Rename feeds.event_id (INTEGER) to feeds.event_uuid (VARCHAR for UUID string)
+    op.drop_column("feeds", "event_id")
+    op.add_column("feeds", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True))
+
+    # Drop the events table (all FK constraints were removed in prior migrations)
+    op.drop_table("events")
+
+
+def downgrade():
+    # Recreate events table (minimal schema for rollback)
+    op.create_table(
+        "events",
+        sa.Column("id", sa.Integer(), nullable=False, autoincrement=True),
+        sa.Column("org_id", sa.Integer(), nullable=False),
+        sa.Column("date", sa.Date(), nullable=False),
+        sa.Column("info", sa.String(), nullable=True),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("published", sa.Boolean(), nullable=False, server_default="false"),
+        sa.Column("attribute_count", sa.Integer(), nullable=True),
+        sa.Column("object_count", sa.Integer(), nullable=True),
+        sa.Column("orgc_id", sa.Integer(), nullable=False),
+        sa.Column("timestamp", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("sharing_group_id", sa.Integer(), nullable=True),
+        sa.Column("proposal_email_lock", sa.Boolean(), nullable=False, server_default="false"),
+        sa.Column("locked", sa.Boolean(), nullable=False, server_default="false"),
+        sa.Column("publish_timestamp", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("sighting_timestamp", sa.Integer(), nullable=True),
+        sa.Column("disable_correlation", sa.Boolean(), nullable=True),
+        sa.Column("extends_uuid", postgresql.UUID(as_uuid=True), nullable=True),
+        sa.Column("protected", sa.Boolean(), nullable=False, server_default="false"),
+        sa.Column("deleted", sa.Boolean(), nullable=False, server_default="false"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+
+    # Revert feeds.event_uuid back to event_id INTEGER
+    op.drop_column("feeds", "event_uuid")
+    op.add_column("feeds", sa.Column("event_id", sa.Integer(), nullable=True))
+
+    # Revert object_references, event_tags, attribute_tags
+    op.drop_column("object_references", "event_uuid")
+    op.add_column("object_references", sa.Column("event_id", sa.Integer(), nullable=False))
+    op.drop_column("event_tags", "event_uuid")
+    op.add_column("event_tags", sa.Column("event_id", sa.Integer(), nullable=True))
+    op.drop_column("attribute_tags", "event_uuid")
+    op.add_column("attribute_tags", sa.Column("event_id", sa.Integer(), nullable=True))

api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py

@@ -0,0 +1,61 @@
+"""detach event/attribute FK constraints from event_tags and attribute_tags
+
+These FK constraints block dropping the events and attributes tables.
+The tag junction tables will remain, referencing only the tags table.
+
+Revision ID: f3e2d1c0b9a8
+Revises: a1b2c3d4e5f6
+Create Date: 2026-03-25 00:00:00.000000
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f3e2d1c0b9a8"
+down_revision = "a1b2c3d4e5f6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint(
+        "event_tags_event_id_fkey", "event_tags", type_="foreignkey"
+    )
+    op.drop_constraint(
+        "attribute_tags_attribute_id_fkey", "attribute_tags", type_="foreignkey"
+    )
+    op.drop_constraint(
+        "attribute_tags_event_id_fkey", "attribute_tags", type_="foreignkey"
+    )
+    # Allow NULLs now that FK constraints are removed (events/attributes may not exist in SQL)
+    op.alter_column("event_tags", "event_id", nullable=True)
+    op.alter_column("attribute_tags", "event_id", nullable=True)
+    op.alter_column("attribute_tags", "attribute_id", nullable=True)
+
+
+def downgrade():
+    op.alter_column("attribute_tags", "attribute_id", nullable=False)
+    op.alter_column("attribute_tags", "event_id", nullable=False)
+    op.alter_column("event_tags", "event_id", nullable=False)
+    op.create_foreign_key(
+        "event_tags_event_id_fkey",
+        "event_tags",
+        "events",
+        ["event_id"],
+        ["id"],
+    )
+    op.create_foreign_key(
+        "attribute_tags_attribute_id_fkey",
+        "attribute_tags",
+        "attributes",
+        ["attribute_id"],
+        ["id"],
+    )
+    op.create_foreign_key(
+        "attribute_tags_event_id_fkey",
+        "attribute_tags",
+        "events",
+        ["event_id"],
+        ["id"],
+    )

api/app/models/__init__.py

@@ -1,9 +1,5 @@
-from app.models.attribute import Attribute  # noqa
-from app.models.event import Event  # noqa
 from app.models.hunt import Hunt  # noqa
 from app.models.module import ModuleSettings  # noqa
-from app.models.object import Object  # noqa
-from app.models.object_reference import ObjectReference  # noqa
 from app.models.organisation import Organisation  # noqa
 from app.models.role import Role  # noqa
 from app.models.server import Server  # noqa
@@ -12,7 +8,7 @@
     SharingGroupOrganisation,
     SharingGroupServer,
 )
-from app.models.tag import AttributeTag, EventTag, Tag  # noqa
+from app.models.tag import Tag  # noqa
 from app.models.user import User  # noqa
 
 # fixes: sqlalchemy.exc.InvalidRequestError: When initializing mapper mapped class AAA->bbb, expression 'XXXX' failed to locate a name ('XXXX'). If this is a class name, consider adding this relationship() to the <class 'app.models.bbb.AAAA'> class after both dependent classes have been defined.