From f270ab8932519890c79aed83f4985d6d8c3c0ee2 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 25 Mar 2026 09:22:01 +0100 Subject: [PATCH 01/38] chg: improve ui --- .../explore/ExploreResultsSection.vue | 240 +++++++++--------- 1 file changed, 121 insertions(+), 119 deletions(-) diff --git a/frontend/src/components/explore/ExploreResultsSection.vue b/frontend/src/components/explore/ExploreResultsSection.vue index 58571f2e..2a54ea1e 100644 --- a/frontend/src/components/explore/ExploreResultsSection.vue +++ b/frontend/src/components/explore/ExploreResultsSection.vue @@ -124,132 +124,134 @@ function onTypesChanged(types) { From 32acc67f22154fee80ffdb902ba158b96f02f111 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 25 Mar 2026 10:09:08 +0100 Subject: [PATCH 02/38] chg: update opensearch mappings, add os-sql parity tests --- api/app/tests/api/test_indexing_parity.py | 299 ++++++++++++++++++++++ api/app/worker/tasks.py | 48 +++- opensearch/mappings/misp-attributes.json | 61 +++++ opensearch/mappings/misp-events.json | 17 ++ opensearch/mappings/misp-objects.json | 12 + 5 files changed, 435 insertions(+), 2 deletions(-) create mode 100644 api/app/tests/api/test_indexing_parity.py diff --git a/api/app/tests/api/test_indexing_parity.py b/api/app/tests/api/test_indexing_parity.py new file mode 100644 index 00000000..e343fcda --- /dev/null +++ b/api/app/tests/api/test_indexing_parity.py @@ -0,0 +1,299 @@ +"""Phase 0 regression harness: verifies that index_event produces OpenSearch +documents that are fully consistent with the PostgreSQL source of truth. + +Each test creates SQL fixtures, calls index_event synchronously (full_reindex), +then fetches the indexed documents and asserts field-level parity. + +Run with: + docker compose exec api poetry run pytest tests/api/test_indexing_parity.py -v +""" +import pytest +from app.models import attribute as attribute_models +from app.models import event as event_models +from app.models import object as object_models +from app.models import object_reference as object_reference_models +from app.models import tag as tag_models +from app.tests.api_tester import ApiTester +from app.worker.tasks import index_attribute, index_event, index_object +from sqlalchemy.orm import Session + + +def _os_client(): + from app.services.opensearch import get_opensearch_client + + return get_opensearch_client() + + +class TestIndexingParity(ApiTester): + """Assert SQL ↔ OpenSearch parity for events, attributes, and objects.""" + + # ── cleanup ──────────────────────────────────────────────────────────────── + + @pytest.fixture(scope="class", autouse=True) + def cleanup(self, db: Session): + # Runs at class setup time (same pattern as ApiTester.cleanup) so that + # leftover data from a previous test class is wiped before our fixtures + # try to insert rows with the same hardcoded UUIDs / info strings. + try: + pass + finally: + os = _os_client() + for index in ("misp-events", "misp-attributes", "misp-objects"): + try: + os.delete_by_query( + index=index, + body={"query": {"match_all": {}}}, + refresh=True, + ignore=[404], + ) + except Exception: + pass + self.teardown_db(db) + + # ── event parity ────────────────────────────────────────────────────────── + + def test_event_fields_in_opensearch( + self, + db: Session, + event_1: event_models.Event, + attribute_1: attribute_models.Attribute, + object_1: object_models.Object, + object_attribute_1: attribute_models.Attribute, + tlp_white_tag: tag_models.Tag, + ): + """All scalar event fields must appear in the OS document with correct values.""" + # tag the event so we can assert tag parity too + event_tag = tag_models.EventTag( + event_id=event_1.id, tag_id=tlp_white_tag.id, local=False + ) + db.add(event_tag) + db.commit() + + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] + + assert doc["id"] == event_1.id + assert doc["uuid"] == str(event_1.uuid) + assert doc["info"] == event_1.info + assert doc["org_id"] == event_1.org_id + assert doc["orgc_id"] == event_1.orgc_id + assert doc["user_id"] == event_1.user_id + assert doc["published"] == event_1.published + assert doc["analysis"] == event_1.analysis.value + assert doc["distribution"] == event_1.distribution.value + assert doc["threat_level"] == event_1.threat_level.value + assert doc["timestamp"] == event_1.timestamp + assert doc["publish_timestamp"] == event_1.publish_timestamp + assert doc["deleted"] == event_1.deleted + assert doc["locked"] == event_1.locked + assert doc["protected"] == event_1.protected + assert doc["disable_correlation"] == event_1.disable_correlation + assert doc["proposal_email_lock"] == event_1.proposal_email_lock + assert doc.get("sighting_timestamp") == event_1.sighting_timestamp + assert doc.get("sharing_group_id") == event_1.sharing_group_id + expected_extends = ( + str(event_1.extends_uuid) if event_1.extends_uuid else None + ) + assert doc.get("extends_uuid") == expected_extends + + def test_event_tags_in_opensearch( + self, + db: Session, + event_1: event_models.Event, + tlp_white_tag: tag_models.Tag, + ): + """Tags attached to an event must appear in the misp-events document.""" + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] + + tag_names = [t["name"] for t in doc.get("tags", [])] + assert tlp_white_tag.name in tag_names + + def test_event_organisation_in_opensearch( + self, + db: Session, + event_1: event_models.Event, + ): + """The organisation nested object must be present in the misp-events document.""" + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] + + assert "organisation" in doc + assert doc["organisation"]["id"] == event_1.org_id + + # ── attribute parity ────────────────────────────────────────────────────── + + def test_standalone_attribute_fields_in_opensearch( + self, + db: Session, + event_1: event_models.Event, + attribute_1: attribute_models.Attribute, + ): + """Standalone (non-object) attribute fields must be consistent in misp-attributes.""" + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-attributes", id=str(attribute_1.uuid))["_source"] + + assert doc["id"] == attribute_1.id + assert doc["uuid"] == str(attribute_1.uuid) + assert doc["event_id"] == attribute_1.event_id + assert doc["event_uuid"] == str(event_1.uuid) + assert doc["type"] == attribute_1.type + assert doc["value"] == attribute_1.value + assert doc["category"] == attribute_1.category + assert doc["to_ids"] == attribute_1.to_ids + assert doc["deleted"] == attribute_1.deleted + assert doc["disable_correlation"] == attribute_1.disable_correlation + assert doc["timestamp"] == attribute_1.timestamp + assert doc["distribution"] == attribute_1.distribution.value + assert doc.get("sharing_group_id") == attribute_1.sharing_group_id + assert doc.get("comment") == attribute_1.comment + assert doc.get("first_seen") == attribute_1.first_seen + assert doc.get("last_seen") == attribute_1.last_seen + # standalone attributes must not carry an object_uuid + assert doc.get("object_uuid") is None + + def test_object_attribute_fields_in_opensearch( + self, + db: Session, + event_1: event_models.Event, + object_1: object_models.Object, + object_attribute_1: attribute_models.Attribute, + ): + """Object attributes must be in misp-attributes (not embedded in the object doc) + and must carry both event_uuid and object_uuid.""" + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + + # must exist as a separate document in misp-attributes + attr_doc = os.get( + index="misp-attributes", id=str(object_attribute_1.uuid) + )["_source"] + + assert attr_doc["event_uuid"] == str(event_1.uuid) + assert attr_doc["object_uuid"] == str(object_1.uuid) + assert attr_doc["value"] == object_attribute_1.value + + def test_standalone_attribute_object_uuid_via_index_attribute( + self, + db: Session, + event_1: event_models.Event, + attribute_1: attribute_models.Attribute, + ): + """index_attribute must not set object_uuid for standalone attributes.""" + index_attribute(str(attribute_1.uuid)) + + os = _os_client() + doc = os.get(index="misp-attributes", id=str(attribute_1.uuid))["_source"] + + assert doc.get("object_uuid") is None + assert doc["event_uuid"] == str(event_1.uuid) + + def test_object_attribute_object_uuid_via_index_attribute( + self, + db: Session, + event_1: event_models.Event, + object_1: object_models.Object, + object_attribute_1: attribute_models.Attribute, + ): + """index_attribute must populate object_uuid for attributes that belong to an object.""" + index_attribute(str(object_attribute_1.uuid)) + + os = _os_client() + doc = os.get( + index="misp-attributes", id=str(object_attribute_1.uuid) + )["_source"] + + assert doc["object_uuid"] == str(object_1.uuid) + assert doc["event_uuid"] == str(event_1.uuid) + + # ── object parity ───────────────────────────────────────────────────────── + + def test_object_fields_in_opensearch( + self, + db: Session, + event_1: event_models.Event, + object_1: object_models.Object, + ): + """Object scalar fields must be consistent between SQL and misp-objects.""" + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-objects", id=str(object_1.uuid))["_source"] + + assert doc["id"] == object_1.id + assert doc["uuid"] == str(object_1.uuid) + assert doc["event_uuid"] == str(event_1.uuid) + assert doc["name"] == object_1.name + assert doc["deleted"] == object_1.deleted + assert doc["timestamp"] == object_1.timestamp + assert doc["distribution"] == object_1.distribution.value + assert doc.get("sharing_group_id") == object_1.sharing_group_id + assert doc.get("meta_category") == object_1.meta_category + assert doc.get("template_uuid") == object_1.template_uuid + assert doc.get("template_version") == object_1.template_version + assert doc.get("first_seen") == object_1.first_seen + assert doc.get("last_seen") == object_1.last_seen + + def test_object_doc_does_not_contain_attributes( + self, + db: Session, + event_1: event_models.Event, + object_1: object_models.Object, + object_attribute_1: attribute_models.Attribute, + ): + """Attributes must NOT be embedded inside the misp-objects document; they + live as top-level documents in misp-attributes.""" + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-objects", id=str(object_1.uuid))["_source"] + + assert "attributes" not in doc or doc["attributes"] == [] + + def test_index_object_task_does_not_embed_attributes( + self, + db: Session, + event_1: event_models.Event, + object_1: object_models.Object, + object_attribute_1: attribute_models.Attribute, + ): + """index_object (the standalone task) must also not embed attributes.""" + index_object(str(object_1.uuid)) + + os = _os_client() + doc = os.get(index="misp-objects", id=str(object_1.uuid))["_source"] + + assert "attributes" not in doc or doc["attributes"] == [] + + # and the attribute must exist separately + attr_doc = os.get( + index="misp-attributes", id=str(object_attribute_1.uuid) + )["_source"] + assert attr_doc["object_uuid"] == str(object_1.uuid) + + # ── attribute count parity ──────────────────────────────────────────────── + + def test_attribute_count_in_event_doc( + self, + db: Session, + event_1: event_models.Event, + attribute_1: attribute_models.Attribute, + object_attribute_1: attribute_models.Attribute, + ): + """attribute_count in the misp-events document must match the SQL value.""" + db.refresh(event_1) + index_event(str(event_1.uuid), full_reindex=True) + + os = _os_client() + doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] + + assert doc["attribute_count"] == event_1.attribute_count diff --git a/api/app/worker/tasks.py b/api/app/worker/tasks.py index b927253d..da43c083 100644 --- a/api/app/worker/tasks.py +++ b/api/app/worker/tasks.py @@ -928,11 +928,16 @@ def index_attribute(attribute_uuid: str): attribute = event_schemas.Attribute.model_validate(db_attribute) event_uuid = db_attribute.event.uuid if db_attribute.event else None + object_uuid = None + if db_attribute.object_id: + db_object = objects_repository.get_object_by_id(db, db_attribute.object_id) + object_uuid = db_object.uuid if db_object else None OpenSearchClient = get_opensearch_client() attribute_raw = attribute.model_dump() attribute_raw["event_uuid"] = str(event_uuid) if event_uuid else None + attribute_raw["object_uuid"] = str(object_uuid) if object_uuid else None # convert timestamp to datetime so it can be indexed attribute_raw["@timestamp"] = datetime.fromtimestamp( @@ -1035,6 +1040,7 @@ def index_object(object_uuid: str): db_object = objects_repository.get_object_by_uuid(db, object_uuid) if db_object is None: raise Exception("Object with uuid=%s not found", object_uuid) + event_uuid = str(db_object.event.uuid) object = event_schemas.Object.model_validate(db_object) @@ -1046,7 +1052,10 @@ def index_object(object_uuid: str): object_raw["@timestamp"] = datetime.fromtimestamp( object_raw["timestamp"] ).isoformat() - object_raw["event_uuid"] = str(db_object.event.uuid) + object_raw["event_uuid"] = event_uuid + + # pop attributes and index them separately in misp-attributes (consistent with index_event) + object_attributes = object_raw.pop("attributes", []) response = OpenSearchClient.index( index="misp-objects", @@ -1061,7 +1070,42 @@ def index_object(object_uuid: str): ) raise Exception("Failed to index object.") - logger.info("indexed object uuid=%s job finished", object_uuid) + logger.info("indexed object uuid=%s", object_uuid) + + if object_attributes: + attribute_docs = [] + for attribute in object_attributes: + attribute["@timestamp"] = datetime.fromtimestamp( + attribute["timestamp"] + ).isoformat() + attribute["event_uuid"] = event_uuid + attribute["object_uuid"] = str(object_raw["uuid"]) + attribute["data"] = "" + attribute_docs.append( + { + "_id": attribute["uuid"], + "_index": "misp-attributes", + "_source": attribute, + } + ) + + success, failed = opensearch_helpers.bulk( + OpenSearchClient, attribute_docs, refresh=True + ) + if failed: + logger.error( + "Failed to index attributes of object uuid=%s. Failed: %s", + object_uuid, + failed, + ) + raise Exception("Failed to index object attributes.") + logger.info( + "indexed %s attributes of object uuid=%s", + len(attribute_docs), + object_uuid, + ) + + logger.info("indexing object uuid=%s job finished", object_uuid) return True diff --git a/opensearch/mappings/misp-attributes.json b/opensearch/mappings/misp-attributes.json index e22de228..af32638c 100644 --- a/opensearch/mappings/misp-attributes.json +++ b/opensearch/mappings/misp-attributes.json @@ -161,6 +161,67 @@ "ignore_above": 256 } } + }, + "sharing_group_id": { + "type": "long" + }, + "comment": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "first_seen": { + "type": "long" + }, + "last_seen": { + "type": "long" + }, + "object_uuid": { + "type": "keyword" + }, + "tags": { + "properties": { + "colour": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "exportable": { + "type": "boolean" + }, + "hide_tag": { + "type": "boolean" + }, + "id": { + "type": "long" + }, + "is_custom_galaxy": { + "type": "boolean" + }, + "is_galaxy": { + "type": "boolean" + }, + "local_only": { + "type": "boolean" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } } } } diff --git a/opensearch/mappings/misp-events.json b/opensearch/mappings/misp-events.json index 8e5a9fd2..1ba79718 100644 --- a/opensearch/mappings/misp-events.json +++ b/opensearch/mappings/misp-events.json @@ -115,6 +115,23 @@ "ignore_above": 256 } } + }, + "sighting_timestamp": { + "type": "long" + }, + "extends_uuid": { + "type": "keyword" + }, + "sharing_group_id": { + "type": "long" + }, + "organisation": { + "type": "object", + "dynamic": true + }, + "sharing_group": { + "type": "object", + "dynamic": true } } } diff --git a/opensearch/mappings/misp-objects.json b/opensearch/mappings/misp-objects.json index 80a784da..857392b1 100644 --- a/opensearch/mappings/misp-objects.json +++ b/opensearch/mappings/misp-objects.json @@ -67,11 +67,23 @@ } } }, + "sharing_group_id": { + "type": "long" + }, "object_references": { "properties": { "@timestamp": { "type": "date" }, + "source_uuid": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, "comment": { "type": "text", "fields": { From 74ac8267babef37079238443c15054a2a4095396 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 25 Mar 2026 10:22:00 +0100 Subject: [PATCH 03/38] chg: do opensearch operations inline for crud operations --- api/app/repositories/attributes.py | 6 +++--- api/app/repositories/events.py | 17 ++++++++++------- api/app/repositories/objects.py | 6 +++--- api/app/routers/attributes.py | 2 ++ api/app/routers/events.py | 4 +++- api/app/worker/tasks.py | 16 ++++++++-------- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/api/app/repositories/attributes.py b/api/app/repositories/attributes.py index 38b7e075..33226270 100644 --- a/api/app/repositories/attributes.py +++ b/api/app/repositories/attributes.py @@ -140,7 +140,7 @@ def create_attribute( db.refresh(db_attribute) if db_attribute is not None: - tasks.handle_created_attribute.delay( + tasks.handle_created_attribute( db_attribute.id, db_attribute.object_id, db_attribute.event_id ) @@ -291,7 +291,7 @@ def update_attribute( db.commit() db.refresh(db_attribute) - tasks.handle_updated_attribute.delay( + tasks.handle_updated_attribute( db_attribute.id, db_attribute.object_id, db_attribute.event_id ) @@ -316,7 +316,7 @@ def delete_attribute(db: Session, attribute_id: int | str) -> None: db.commit() db.refresh(db_attribute) - tasks.handle_deleted_attribute.delay( + tasks.handle_deleted_attribute( db_attribute.id, db_attribute.object_id, db_attribute.event_id ) diff --git a/api/app/repositories/events.py b/api/app/repositories/events.py index 43fb77ee..00a54a46 100644 --- a/api/app/repositories/events.py +++ b/api/app/repositories/events.py @@ -170,7 +170,7 @@ def create_event(db: Session, event: event_schemas.EventCreate) -> event_models. db.flush() db.refresh(db_event) - tasks.handle_created_event.delay(str(db_event.uuid)) + tasks.handle_created_event(str(db_event.uuid)) return db_event @@ -259,7 +259,7 @@ def update_event(db: Session, event_id: int, event: event_schemas.EventUpdate): db.commit() db.refresh(db_event) - tasks.handle_updated_event.delay(str(db_event.uuid)) + tasks.handle_updated_event(str(db_event.uuid)) return db_event @@ -282,13 +282,13 @@ def delete_event(db: Session, event_id: Union[int, UUID], force: bool = False) - event_uuid = str(db_event.uuid) db.delete(db_event) db.commit() - tasks.delete_indexed_event.delay(event_uuid) + tasks.delete_indexed_event(event_uuid) return db.commit() db.refresh(db_event) - tasks.handle_deleted_event.delay(str(db_event.uuid)) + tasks.handle_deleted_event(str(db_event.uuid)) def increment_attribute_count( @@ -535,7 +535,8 @@ def publish_event(db: Session, db_event: event_models.Event) -> event_models.Eve db.commit() db.refresh(db_event) - tasks.handle_published_event.delay(str(db_event.uuid)) + tasks.handle_published_event(str(db_event.uuid)) + tasks.index_event(str(db_event.uuid), full_reindex=False) return db_event @@ -550,7 +551,8 @@ def unpublish_event(db: Session, db_event: event_models.Event) -> event_models.E db.commit() db.refresh(db_event) - tasks.handle_unpublished_event.delay(str(db_event.uuid)) + tasks.handle_unpublished_event(str(db_event.uuid)) + tasks.index_event(str(db_event.uuid), full_reindex=False) return db_event @@ -563,9 +565,10 @@ def toggle_event_correlation( db.commit() db.refresh(db_event) - tasks.handle_toggled_event_correlation.delay( + tasks.handle_toggled_event_correlation( str(db_event.uuid), db_event.disable_correlation ) + tasks.index_event(str(db_event.uuid), full_reindex=False) return db_event diff --git a/api/app/repositories/objects.py b/api/app/repositories/objects.py index ded32873..8ca878a4 100644 --- a/api/app/repositories/objects.py +++ b/api/app/repositories/objects.py @@ -150,7 +150,7 @@ def create_object( db.refresh(db_object) - tasks.handle_created_object.delay(db_object.id, db_object.event_id) + tasks.handle_created_object(db_object.id, db_object.event_id) return db_object @@ -332,7 +332,7 @@ def update_object( db.commit() db.refresh(db_object) - tasks.handle_updated_object.delay(db_object.id, db_object.event_id) + tasks.handle_updated_object(db_object.id, db_object.event_id) return db_object @@ -359,7 +359,7 @@ def delete_object(db: Session, object_id: Union[int, UUID]) -> object_models.Obj db.commit() db.refresh(db_object) - tasks.handle_deleted_object.delay(db_object.id, db_object.event_id) + tasks.handle_deleted_object(db_object.id, db_object.event_id) return db_object diff --git a/api/app/routers/attributes.py b/api/app/routers/attributes.py index debde73d..61e16ca5 100644 --- a/api/app/routers/attributes.py +++ b/api/app/routers/attributes.py @@ -199,6 +199,7 @@ def tag_attribute( ) tags_repository.tag_attribute(db=db, attribute=attribute, tag=tag) + tasks.index_attribute(str(attribute.uuid)) return Response(status_code=status.HTTP_201_CREATED) @@ -228,5 +229,6 @@ def untag_attribute( ) tags_repository.untag_attribute(db=db, attribute=attribute, tag=tag) + tasks.index_attribute(str(attribute.uuid)) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/api/app/routers/events.py b/api/app/routers/events.py index cef8df1b..fd26917a 100644 --- a/api/app/routers/events.py +++ b/api/app/routers/events.py @@ -132,7 +132,7 @@ def create_event( event_create_request.org_id = user.org_id db_event = events_repository.create_event(db=db, event=event_create_request) - tasks.index_event.delay(str(db_event.uuid), full_reindex=True) + tasks.index_event(str(db_event.uuid), full_reindex=True) return db_event @@ -186,6 +186,7 @@ def tag_event( ) tags_repository.tag_event(db=db, event=event, tag=tag) + tasks.index_event(str(event.uuid), full_reindex=False) return Response(status_code=status.HTTP_201_CREATED) @@ -215,6 +216,7 @@ def untag_event( ) tags_repository.untag_event(db=db, event=event, tag=tag) + tasks.index_event(str(event.uuid), full_reindex=False) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/api/app/worker/tasks.py b/api/app/worker/tasks.py index da43c083..aa204deb 100644 --- a/api/app/worker/tasks.py +++ b/api/app/worker/tasks.py @@ -172,7 +172,7 @@ def handle_updated_event(event_uuid: str): db, "updated", event=db_event ) - index_event.delay(str(db_event.uuid), full_reindex=False) + index_event(str(db_event.uuid), full_reindex=False) return True @@ -190,7 +190,7 @@ def handle_deleted_event(event_uuid: str): db, "deleted", event=db_event ) - delete_indexed_event.delay(str(event_uuid)) + delete_indexed_event(str(event_uuid)) return True @@ -207,7 +207,7 @@ def handle_created_attribute(attribute_id: int, object_id: int | None, event_id: db, "created", attribute=db_attribute ) - index_attribute.delay(str(db_attribute.uuid)) + index_attribute(str(db_attribute.uuid)) return True @@ -221,7 +221,7 @@ def handle_updated_attribute(attribute_id: int, object_id: int | None, event_id: db, "updated", attribute=db_attribute ) - index_attribute.delay(str(db_attribute.uuid)) + index_attribute(str(db_attribute.uuid)) return True @@ -238,7 +238,7 @@ def handle_deleted_attribute(attribute_id: int, object_id: int | None, event_id: db, "deleted", attribute=db_attribute ) - delete_indexed_attribute.delay(str(db_attribute.uuid)) + delete_indexed_attribute(str(db_attribute.uuid)) return True @@ -255,7 +255,7 @@ def handle_created_object(object_id: int, event_id: int): db, "created", object=db_object ) - index_object.delay(str(db_object.uuid)) + index_object(str(db_object.uuid)) return True @@ -270,7 +270,7 @@ def handle_updated_object(object_id: int, event_id: int): db, "updated", object=db_object ) - index_object.delay(str(db_object.uuid)) + index_object(str(db_object.uuid)) return True @@ -287,7 +287,7 @@ def handle_deleted_object(object_id: int, event_id: int): db, "deleted", object=db_object ) - delete_indexed_object.delay(str(db_object.uuid)) + delete_indexed_object(str(db_object.uuid)) return True From 37721b5610e694c7d4e0b8d86aa453d0de881ae4 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 25 Mar 2026 11:05:42 +0100 Subject: [PATCH 04/38] add: add correlations and expanded props to attributes pydantic schema, refactor read operations to use opensearch instead of postgres --- api/app/repositories/attributes.py | 74 +++++++++++++++++++- api/app/repositories/events.py | 67 +++++++++++++++++- api/app/repositories/objects.py | 108 ++++++++++++++++++++++++++++- api/app/routers/attributes.py | 32 +++------ api/app/routers/events.py | 21 ++---- api/app/routers/objects.py | 27 +++----- api/app/schemas/attribute.py | 3 +- api/app/tests/api/test_events.py | 11 ++- api/app/tests/api_tester.py | 27 +++++++- 9 files changed, 305 insertions(+), 65 deletions(-) diff --git a/api/app/repositories/attributes.py b/api/app/repositories/attributes.py index 33226270..d6ea0f0a 100644 --- a/api/app/repositories/attributes.py +++ b/api/app/repositories/attributes.py @@ -1,5 +1,6 @@ +import math import time -from typing import Iterable, Union +from typing import Iterable, Optional, Union from uuid import UUID from app.models.event import DistributionLevel from app.services.opensearch import get_opensearch_client @@ -12,11 +13,11 @@ from app.schemas import event as event_schemas from app.worker import tasks from fastapi import HTTPException, status +from fastapi_pagination import Page, Params from fastapi_pagination.ext.sqlalchemy import paginate from pymisp import MISPAttribute, MISPTag from sqlalchemy.orm import Session from sqlalchemy.sql import select -from fastapi_pagination import Page from collections import defaultdict from opensearchpy.exceptions import NotFoundError @@ -86,6 +87,75 @@ def get_attributes( return enrich_attributes_page_with_correlations(page_results) +def get_attributes_from_opensearch( + params: Params, + event_uuid: str = None, + deleted: bool = None, + object_id: int = None, + type: str = None, +) -> Page[attribute_schemas.Attribute]: + client = get_opensearch_client() + + must_clauses = [] + if event_uuid is not None: + must_clauses.append({"term": {"event_uuid.keyword": event_uuid}}) + if deleted is not None: + must_clauses.append({"term": {"deleted": deleted}}) + if type is not None: + must_clauses.append({"term": {"type.keyword": type}}) + + # Mirror SQL behaviour: always filter by object_id (None → standalone attributes only) + if object_id is None: + must_clauses.append( + {"bool": {"must_not": [{"exists": {"field": "object_id"}}]}} + ) + else: + must_clauses.append({"term": {"object_id": object_id}}) + + query_body = { + "query": {"bool": {"must": must_clauses}}, + "from": (params.page - 1) * params.size, + "size": params.size, + "sort": [{"timestamp": {"order": "desc"}}], + } + + response = client.search(index="misp-attributes", body=query_body) + total = response["hits"]["total"]["value"] + hits = response["hits"]["hits"] + + items = [attribute_schemas.Attribute.model_validate(hit["_source"]) for hit in hits] + + pages = math.ceil(total / params.size) if params.size > 0 else 0 + attributes_page = Page( + items=items, total=total, page=params.page, size=params.size, pages=pages + ) + return enrich_attributes_page_with_correlations(attributes_page) + + +def get_attribute_from_opensearch( + attribute_id: Union[int, UUID], +) -> Optional[attribute_schemas.Attribute]: + client = get_opensearch_client() + + if isinstance(attribute_id, int): + response = client.search( + index="misp-attributes", + body={"query": {"term": {"id": attribute_id}}, "size": 1}, + ) + hits = response["hits"]["hits"] + if not hits: + return None + source = hits[0]["_source"] + else: + try: + doc = client.get(index="misp-attributes", id=str(attribute_id)) + source = doc["_source"] + except NotFoundError: + return None + + return attribute_schemas.Attribute.model_validate(source) + + def get_attribute_by_id( db: Session, attribute_id: int ) -> Union[attribute_models.Attribute, None]: diff --git a/api/app/repositories/events.py b/api/app/repositories/events.py index 00a54a46..35c9c652 100644 --- a/api/app/repositories/events.py +++ b/api/app/repositories/events.py @@ -1,8 +1,9 @@ import logging +import math import time from datetime import datetime from uuid import UUID -from typing import Union, Iterable +from typing import Optional, Union, Iterable from app.worker import tasks from app.services.opensearch import get_opensearch_client from app.services.vulnerability_lookup import lookup as vulnerability_lookup @@ -17,7 +18,9 @@ import app.schemas.attribute as attribute_schemas import app.schemas.vulnerability as vulnerability_schemas from fastapi import HTTPException, status, Query +from fastapi_pagination import Page, Params from fastapi_pagination.ext.sqlalchemy import paginate +from opensearchpy.exceptions import NotFoundError from pymisp import MISPEvent, MISPOrganisation from sqlalchemy.orm import Session, noload from sqlalchemy.sql import select @@ -52,6 +55,68 @@ def get_events(db: Session, info: str = Query(None), deleted: bool = Query(None) return paginate(db, query) +def get_events_from_opensearch( + params: Params, + info: str = None, + deleted: bool = None, + uuid: str = None, +) -> Page[event_schemas.Event]: + client = get_opensearch_client() + + must_clauses = [] + if info is not None: + must_clauses.append({"match": {"info": info}}) + if deleted is not None: + must_clauses.append({"term": {"deleted": deleted}}) + if uuid is not None: + must_clauses.append({"term": {"uuid.keyword": uuid}}) + + query_body = { + "query": {"bool": {"must": must_clauses}}, + "from": (params.page - 1) * params.size, + "size": params.size, + "sort": [{"timestamp": {"order": "desc"}}], + } + + response = client.search(index="misp-events", body=query_body) + total = response["hits"]["total"]["value"] + hits = response["hits"]["hits"] + + items = [] + for hit in hits: + source = hit["_source"] + source.setdefault("attributes", []) + source.setdefault("objects", []) + items.append(event_schemas.Event.model_validate(source)) + + pages = math.ceil(total / params.size) if params.size > 0 else 0 + return Page(items=items, total=total, page=params.page, size=params.size, pages=pages) + + +def get_event_from_opensearch(event_id: Union[int, UUID]) -> Optional[event_schemas.Event]: + client = get_opensearch_client() + + if isinstance(event_id, int): + response = client.search( + index="misp-events", + body={"query": {"term": {"id": event_id}}, "size": 1}, + ) + hits = response["hits"]["hits"] + if not hits: + return None + source = hits[0]["_source"] + else: + try: + doc = client.get(index="misp-events", id=str(event_id)) + source = doc["_source"] + except NotFoundError: + return None + + source.setdefault("attributes", []) + source.setdefault("objects", []) + return event_schemas.Event.model_validate(source) + + def search_events( query: str = None, page: int = 0, diff --git a/api/app/repositories/objects.py b/api/app/repositories/objects.py index 8ca878a4..c1aed9ef 100644 --- a/api/app/repositories/objects.py +++ b/api/app/repositories/objects.py @@ -1,6 +1,7 @@ import logging +import math import time -from typing import Union +from typing import Optional, Union from uuid import UUID, uuid4 from app.models import attribute as attribute_models @@ -19,6 +20,7 @@ from app.worker import tasks from app.services.opensearch import get_opensearch_client from fastapi import HTTPException, status +from fastapi_pagination import Page, Params from fastapi_pagination.ext.sqlalchemy import paginate from pymisp import MISPObject from sqlalchemy.orm import Session @@ -93,6 +95,110 @@ def get_objects( return objects_page +def get_objects_from_opensearch( + params: Params, + event_uuid: str = None, + deleted: bool = False, + template_uuid: list = None, +) -> Page[object_schemas.Object]: + client = get_opensearch_client() + + must_clauses = [{"term": {"deleted": bool(deleted)}}] + if event_uuid is not None: + must_clauses.append({"term": {"event_uuid.keyword": event_uuid}}) + if template_uuid is not None: + must_clauses.append( + {"terms": {"template_uuid.keyword": [str(u) for u in template_uuid]}} + ) + + query_body = { + "query": {"bool": {"must": must_clauses}}, + "from": (params.page - 1) * params.size, + "size": params.size, + "sort": [{"timestamp": {"order": "desc"}}], + } + + response = client.search(index="misp-objects", body=query_body) + total = response["hits"]["total"]["value"] + hits = response["hits"]["hits"] + + if not hits: + return Page(items=[], total=total, page=params.page, size=params.size, pages=0) + + # Batch-fetch attributes for all returned objects + object_uuids = [str(hit["_source"]["uuid"]) for hit in hits] + attr_response = client.search( + index="misp-attributes", + body={ + "query": {"terms": {"object_uuid": object_uuids}}, + "size": 10000, + }, + ) + + raw_by_uuid: dict = {} + for attr_hit in attr_response["hits"]["hits"]: + src = attr_hit["_source"] + key = src.get("object_uuid") + if key: + raw_by_uuid.setdefault(key, []).append(src) + + # Build and correlate per-object attribute lists + attrs_by_uuid: dict = {} + for obj_uuid, raw_attrs in raw_by_uuid.items(): + built = [attribute_schemas.Attribute.model_validate(s) for s in raw_attrs] + attrs_by_uuid[obj_uuid] = enrich_object_attributes_with_correlations(built) + + items = [] + for hit in hits: + source = hit["_source"] + obj_uuid = str(source.get("uuid", "")) + source["attributes"] = attrs_by_uuid.get(obj_uuid, []) + source.setdefault("object_references", []) + items.append(object_schemas.Object.model_validate(source)) + + pages = math.ceil(total / params.size) if params.size > 0 else 0 + return Page(items=items, total=total, page=params.page, size=params.size, pages=pages) + + +def get_object_from_opensearch( + object_id: Union[int, UUID], +) -> Optional[object_schemas.Object]: + client = get_opensearch_client() + + if isinstance(object_id, int): + response = client.search( + index="misp-objects", + body={"query": {"term": {"id": object_id}}, "size": 1}, + ) + hits = response["hits"]["hits"] + if not hits: + return None + source = hits[0]["_source"] + else: + try: + doc = client.get(index="misp-objects", id=str(object_id)) + source = doc["_source"] + except NotFoundError: + return None + + obj_uuid = str(source.get("uuid", "")) + attr_response = client.search( + index="misp-attributes", + body={"query": {"term": {"object_uuid": obj_uuid}}, "size": 10000}, + ) + + attributes = [] + attributes = [ + attribute_schemas.Attribute.model_validate(h["_source"]) + for h in attr_response["hits"]["hits"] + ] + attributes = enrich_object_attributes_with_correlations(attributes) + source["attributes"] = attributes + source.setdefault("object_references", []) + + return object_schemas.Object.model_validate(source) + + def get_object_by_id(db: Session, object_id: int): return ( db.query(object_models.Object) diff --git a/api/app/routers/attributes.py b/api/app/routers/attributes.py index 61e16ca5..60260a17 100644 --- a/api/app/routers/attributes.py +++ b/api/app/routers/attributes.py @@ -11,7 +11,7 @@ from app.schemas import user as user_schemas from app.worker import tasks from fastapi import APIRouter, Depends, HTTPException, Response, Security, status, Query -from fastapi_pagination import Page +from fastapi_pagination import Page, Params from fastapi.responses import JSONResponse from sqlalchemy.orm import Session @@ -35,13 +35,17 @@ async def get_attributes_parameters( @router.get("/attributes/", response_model=Page[attribute_schemas.Attribute]) def get_attributes( params: dict = Depends(get_attributes_parameters), - db: Session = Depends(get_db), + page_params: Params = Depends(), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:read"] ), ) -> Page[attribute_schemas.Attribute]: - return attributes_repository.get_attributes( - db, params["event_uuid"], params["deleted"], params["object_id"], params["type"] + return attributes_repository.get_attributes_from_opensearch( + page_params, + params["event_uuid"], + params["deleted"], + params["object_id"], + params["type"], ) @@ -79,30 +83,16 @@ async def export_attributes( @router.get("/attributes/{attribute_id}", response_model=attribute_schemas.Attribute) def get_attribute_by_id( attribute_id: Union[int, UUID], - db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:read"] ), ) -> attribute_schemas.Attribute: - - if isinstance(attribute_id, UUID): - db_attribute = attributes_repository.get_attribute_by_uuid( - db, attribute_uuid=attribute_id - ) - else: - db_attribute = attributes_repository.get_attribute_by_id( - db, attribute_id=attribute_id - ) - - if db_attribute is None: + os_attribute = attributes_repository.get_attribute_from_opensearch(attribute_id) + if os_attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" ) - - attribute = attribute_schemas.Attribute.from_orm(db_attribute) - attribute.event_uuid = str(db_attribute.event.uuid) - - return attribute + return os_attribute @router.post( diff --git a/api/app/routers/events.py b/api/app/routers/events.py index fd26917a..451083d2 100644 --- a/api/app/routers/events.py +++ b/api/app/routers/events.py @@ -25,7 +25,7 @@ Form, Query, ) -from fastapi_pagination import Page +from fastapi_pagination import Page, Params from sqlalchemy.orm import Session from starlette import status from fastapi.responses import JSONResponse @@ -50,15 +50,14 @@ async def get_events_parameters( @router.get("/events/", response_model=Page[event_schemas.Event]) async def get_events( params: dict = Depends(get_events_parameters), - db: Session = Depends(get_db), + page_params: Params = Depends(), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ) -> Page[event_schemas.Event]: - return events_repository.get_events( - db, + return events_repository.get_events_from_opensearch( + page_params, params["info"], params["deleted"], params["uuid"], - params["include_attributes"], ) @@ -96,20 +95,14 @@ async def export_events( @router.get("/events/{event_id}", response_model=event_schemas.Event) def get_event_by_id( event_id: Union[int, UUID], - db: Session = Depends(get_db), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ) -> event_schemas.Event: - - if isinstance(event_id, int): - db_event = events_repository.get_event_by_id(db, event_id=event_id) - else: - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_id) - - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_id) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - return db_event + return os_event @router.post( diff --git a/api/app/routers/objects.py b/api/app/routers/objects.py index 796f0059..bb3115b5 100644 --- a/api/app/routers/objects.py +++ b/api/app/routers/objects.py @@ -10,7 +10,7 @@ from app.worker import tasks from fastapi import APIRouter, Depends, HTTPException, Security, status, Response from sqlalchemy.orm import Session -from fastapi_pagination import Page +from fastapi_pagination import Page, Params router = APIRouter() @@ -26,39 +26,32 @@ async def get_objects_parameters( @router.get("/objects/", response_model=Page[object_schemas.Object]) def get_objects( params: dict = Depends(get_objects_parameters), - db: Session = Depends(get_db), + page_params: Params = Depends(), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:read"] ), ) -> Page[object_schemas.Object]: - return objects_repository.get_objects( - db, params["event_uuid"], params["deleted"], params["template_uuid"] + return objects_repository.get_objects_from_opensearch( + page_params, + params["event_uuid"], + params["deleted"], + params["template_uuid"], ) @router.get("/objects/{object_id}", response_model=object_schemas.Object) def get_object_by_id( object_id: Union[int, UUID], - db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:read"] ), ): - - if isinstance(object_id, int): - db_object = objects_repository.get_object_by_id(db, object_id=object_id) - else: - db_object = objects_repository.get_object_by_uuid(db, object_uuid=object_id) - - if db_object is None: + os_object = objects_repository.get_object_from_opensearch(object_id) + if os_object is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" ) - - object = object_schemas.Object.from_orm(db_object) - object.event_uuid = str(db_object.event.uuid) - - return object + return os_object @router.post( "/objects/", diff --git a/api/app/schemas/attribute.py b/api/app/schemas/attribute.py index 4aa59f71..e9834e87 100644 --- a/api/app/schemas/attribute.py +++ b/api/app/schemas/attribute.py @@ -31,7 +31,8 @@ class AttributeBase(BaseModel): class Attribute(AttributeBase): id: int tags: list[Tag] = [] - correlations: list[dict] = None + correlations: Optional[list[dict]] = None + expanded: Optional[dict] = None model_config = ConfigDict(from_attributes=True) diff --git a/api/app/tests/api/test_events.py b/api/app/tests/api/test_events.py index 37d4e445..f097e9aa 100644 --- a/api/app/tests/api/test_events.py +++ b/api/app/tests/api/test_events.py @@ -18,12 +18,10 @@ def test_get_events( client: TestClient, user_1: user_models.User, event_1: event_models.Event, - attribute_1: attribute_models.Attribute, auth_token: auth.Token, ): response = client.get( - "/events/", headers={"Authorization": "Bearer " + auth_token}, - params={"include_attributes": True} + "/events/", headers={"Authorization": "Bearer " + auth_token} ) data = response.json()["items"] @@ -35,10 +33,6 @@ def test_get_events( assert data[0]["org_id"] == event_1.org_id assert data[0]["orgc_id"] == event_1.orgc_id assert data[0]["user_id"] == user_1.id - assert data[0]["attributes"][0]["event_id"] == attribute_1.event_id - assert data[0]["attributes"][0]["value"] == attribute_1.value - assert data[0]["attributes"][0]["category"] == attribute_1.category - assert data[0]["attributes"][0]["type"] == attribute_1.type @pytest.mark.parametrize("scopes", [[]]) def test_get_events_unauthorized( @@ -245,6 +239,8 @@ def event_for_filter( user_1: user_models.User, ): """A stable event used for filter/search tests that won't be mutated.""" + from app.worker.tasks import index_event as _index_event + ev = event_models.Event( info="stable filter test event", user_id=user_1.id, @@ -257,6 +253,7 @@ def event_for_filter( db.add(ev) db.commit() db.refresh(ev) + _index_event(str(ev.uuid), full_reindex=False) yield ev # ---- GET /events/{event_id} ---- diff --git a/api/app/tests/api_tester.py b/api/app/tests/api_tester.py index c74efc5c..a19e0be1 100644 --- a/api/app/tests/api_tester.py +++ b/api/app/tests/api_tester.py @@ -96,7 +96,23 @@ def cleanup(self, db: Session): try: pass finally: - # teardown + # clean OpenSearch docs left over from previous test classes + try: + from app.services.opensearch import get_opensearch_client + + os_client = get_opensearch_client() + for index in ("misp-events", "misp-attributes", "misp-objects"): + try: + os_client.delete_by_query( + index=index, + body={"query": {"match_all": {}}}, + refresh=True, + ignore=[404], + ) + except Exception: + pass + except Exception: + pass self.teardown_db(db) # MISP data model fixtures @@ -157,6 +173,8 @@ def event_1( organisation_1: organisation_models.Organisation, user_1: user_models.User, ): + from app.worker.tasks import index_event as _index_event + event_1 = event_models.Event( info="test event", user_id=user_1.id, @@ -169,11 +187,14 @@ def event_1( db.add(event_1) db.commit() db.refresh(event_1) + _index_event(str(event_1.uuid), full_reindex=False) yield event_1 @pytest.fixture(scope="class") def attribute_1(self, db: Session, event_1: event_models.Event): + from app.worker.tasks import index_attribute as _index_attribute + attribute_1 = attribute_models.Attribute( event_id=event_1.id, category="Network activity", @@ -185,11 +206,14 @@ def attribute_1(self, db: Session, event_1: event_models.Event): db.add(attribute_1) db.commit() db.refresh(attribute_1) + _index_attribute(str(attribute_1.uuid)) yield attribute_1 @pytest.fixture(scope="class") def object_1(self, db: Session, event_1: event_models.Event): + from app.worker.tasks import index_object as _index_object + object_1 = object_models.Object( event_id=event_1.id, uuid="90e06ef6-26f8-40dd-9fb7-75897445e2a0", @@ -201,6 +225,7 @@ def object_1(self, db: Session, event_1: event_models.Event): db.add(object_1) db.commit() db.refresh(object_1) + _index_object(str(object_1.uuid)) yield object_1 From 853c6b77c252f1d5fee59d01c164665d5733f685 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 26 Mar 2026 10:32:39 +0100 Subject: [PATCH 05/38] [refactor] do not store events in postgresql, use only opensearch, use uuid references not integer ids --- ...d7e6f5a4b3_detach_object_references_fks.py | 45 ++ ...9_detach_attributes_objects_from_events.py | 30 + .../e1a2b3c4d5f6_drop_events_table.py | 75 ++ ...ach_event_attribute_fks_from_tag_tables.py | 62 ++ api/app/models/attribute.py | 11 +- api/app/models/event.py | 90 --- api/app/models/feed.py | 2 +- api/app/models/object.py | 7 - api/app/models/object_reference.py | 4 +- api/app/models/tag.py | 9 +- api/app/repositories/attachments.py | 16 +- api/app/repositories/attributes.py | 187 +++-- api/app/repositories/events.py | 710 ++++++++---------- api/app/repositories/feeds.py | 34 +- api/app/repositories/notifications.py | 31 +- api/app/repositories/object_references.py | 11 +- api/app/repositories/objects.py | 248 +++--- api/app/repositories/reports.py | 4 +- api/app/repositories/servers.py | 43 +- api/app/repositories/sync.py | 26 +- api/app/repositories/tags.py | 177 +++-- api/app/routers/attributes.py | 68 +- api/app/routers/events.py | 130 +--- api/app/routers/objects.py | 14 +- api/app/schemas/attribute.py | 2 - api/app/schemas/event.py | 1 - api/app/schemas/feed.py | 4 +- api/app/schemas/object.py | 2 - api/app/schemas/object_reference.py | 2 +- api/app/tests/api/test_attributes.py | 61 +- api/app/tests/api/test_events.py | 175 ++--- api/app/tests/api/test_feeds.py | 2 +- api/app/tests/api/test_indexing_parity.py | 299 -------- api/app/tests/api/test_objects.py | 17 +- api/app/tests/api_tester.py | 108 ++- api/app/worker/tasks.py | 416 +++------- docker-compose.dev.yml | 1 + docker-compose.test.yml | 3 +- 38 files changed, 1321 insertions(+), 1806 deletions(-) create mode 100644 api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py create mode 100644 api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py create mode 100644 api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py create mode 100644 api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py delete mode 100644 api/app/tests/api/test_indexing_parity.py diff --git a/api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py b/api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py new file mode 100644 index 00000000..0850c14d --- /dev/null +++ b/api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py @@ -0,0 +1,45 @@ +"""detach object_references FK constraints to events and objects + +These are the last DB-level FK constraints that reference the events and objects +tables from an external table, blocking their eventual removal. + +Revision ID: c8d7e6f5a4b3 +Revises: f3e2d1c0b9a8 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c8d7e6f5a4b3" +down_revision = "f3e2d1c0b9a8" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint( + "object_references_event_id_fkey", "object_references", type_="foreignkey" + ) + op.drop_constraint( + "object_references_object_id_fkey", "object_references", type_="foreignkey" + ) + + +def downgrade(): + op.create_foreign_key( + "object_references_event_id_fkey", + "object_references", + "events", + ["event_id"], + ["id"], + ) + op.create_foreign_key( + "object_references_object_id_fkey", + "object_references", + "objects", + ["object_id"], + ["id"], + ) diff --git a/api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py b/api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py new file mode 100644 index 00000000..acaece1b --- /dev/null +++ b/api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py @@ -0,0 +1,30 @@ +"""remove event_id columns from attributes and objects tables + +Revision ID: d4e5f6a7b8c9 +Revises: c8d7e6f5a4b3 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c8d7e6f5a4b3" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_index("ix_attributes_event_id", table_name="attributes", if_exists=True) + op.drop_column("attributes", "event_id") + op.drop_index("ix_objects_event_id", table_name="objects", if_exists=True) + op.drop_column("objects", "event_id") + + +def downgrade(): + import sqlalchemy as sa + + op.add_column("objects", sa.Column("event_id", sa.Integer(), nullable=True)) + op.create_index("ix_objects_event_id", "objects", ["event_id"]) + op.add_column("attributes", sa.Column("event_id", sa.Integer(), nullable=True)) + op.create_index("ix_attributes_event_id", "attributes", ["event_id"]) diff --git a/api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py b/api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py new file mode 100644 index 00000000..7b6edd69 --- /dev/null +++ b/api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py @@ -0,0 +1,75 @@ +"""drop events table and migrate feeds.event_id to uuid string + +Revision ID: e1a2b3c4d5f6 +Revises: d4e5f6a7b8c9 +Create Date: 2026-03-26 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "e1a2b3c4d5f6" +down_revision = "d4e5f6a7b8c9" +branch_labels = None +depends_on = None + + +def upgrade(): + # Replace event_id (INTEGER) with event_uuid (VARCHAR) in object_references + op.drop_column("object_references", "event_id") + op.add_column("object_references", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + + # Replace event_id (INTEGER) with event_uuid (VARCHAR) in event_tags and attribute_tags + op.drop_column("event_tags", "event_id") + op.add_column("event_tags", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + op.drop_column("attribute_tags", "event_id") + op.add_column("attribute_tags", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + + # Rename feeds.event_id (INTEGER) to feeds.event_uuid (VARCHAR for UUID string) + op.drop_column("feeds", "event_id") + op.add_column("feeds", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + + # Drop the events table (all FK constraints were removed in prior migrations) + op.drop_table("events") + + +def downgrade(): + # Recreate events table (minimal schema for rollback) + op.create_table( + "events", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("org_id", sa.Integer(), nullable=False), + sa.Column("date", sa.Date(), nullable=False), + sa.Column("info", sa.String(), nullable=True), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("uuid", sa.dialects.postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("published", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("attribute_count", sa.Integer(), nullable=True), + sa.Column("object_count", sa.Integer(), nullable=True), + sa.Column("orgc_id", sa.Integer(), nullable=False), + sa.Column("timestamp", sa.Integer(), nullable=False, server_default="0"), + sa.Column("sharing_group_id", sa.Integer(), nullable=True), + sa.Column("proposal_email_lock", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("locked", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("publish_timestamp", sa.Integer(), nullable=False, server_default="0"), + sa.Column("sighting_timestamp", sa.Integer(), nullable=True), + sa.Column("disable_correlation", sa.Boolean(), nullable=True), + sa.Column("extends_uuid", sa.dialects.postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("protected", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("deleted", sa.Boolean(), nullable=False, server_default="false"), + sa.PrimaryKeyConstraint("id"), + ) + + # Revert feeds.event_uuid back to event_id INTEGER + op.drop_column("feeds", "event_uuid") + op.add_column("feeds", sa.Column("event_id", sa.Integer(), nullable=True)) + + # Revert object_references, event_tags, attribute_tags + op.drop_column("object_references", "event_uuid") + op.add_column("object_references", sa.Column("event_id", sa.Integer(), nullable=False)) + op.drop_column("event_tags", "event_uuid") + op.add_column("event_tags", sa.Column("event_id", sa.Integer(), nullable=True)) + op.drop_column("attribute_tags", "event_uuid") + op.add_column("attribute_tags", sa.Column("event_id", sa.Integer(), nullable=True)) diff --git a/api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py b/api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py new file mode 100644 index 00000000..0c966a77 --- /dev/null +++ b/api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py @@ -0,0 +1,62 @@ +"""detach event/attribute FK constraints from event_tags and attribute_tags + +These FK constraints block dropping the events and attributes tables. +The tag junction tables will remain, referencing only the tags table. + +Revision ID: f3e2d1c0b9a8 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f3e2d1c0b9a8" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint( + "event_tags_event_id_fkey", "event_tags", type_="foreignkey" + ) + op.drop_constraint( + "attribute_tags_attribute_id_fkey", "attribute_tags", type_="foreignkey" + ) + op.drop_constraint( + "attribute_tags_event_id_fkey", "attribute_tags", type_="foreignkey" + ) + # Allow NULLs now that FK constraints are removed (events/attributes may not exist in SQL) + op.alter_column("event_tags", "event_id", nullable=True) + op.alter_column("attribute_tags", "event_id", nullable=True) + op.alter_column("attribute_tags", "attribute_id", nullable=True) + + +def downgrade(): + op.alter_column("attribute_tags", "attribute_id", nullable=False) + op.alter_column("attribute_tags", "event_id", nullable=False) + op.alter_column("event_tags", "event_id", nullable=False) + op.create_foreign_key( + "event_tags_event_id_fkey", + "event_tags", + "events", + ["event_id"], + ["id"], + ) + op.create_foreign_key( + "attribute_tags_attribute_id_fkey", + "attribute_tags", + "attributes", + ["attribute_id"], + ["id"], + ) + op.create_foreign_key( + "attribute_tags_event_id_fkey", + "attribute_tags", + "events", + ["event_id"], + ["id"], + ) diff --git a/api/app/models/attribute.py b/api/app/models/attribute.py index 9a54cb94..21572997 100644 --- a/api/app/models/attribute.py +++ b/api/app/models/attribute.py @@ -19,7 +19,6 @@ class Attribute(Base): id = Column( Integer, primary_key=True, index=True, autoincrement=True, nullable=False ) - event_id = Column(Integer, ForeignKey("events.id"), index=True, nullable=False) object_id = Column(Integer, ForeignKey("objects.id")) object_relation = Column(String(255), index=True) category = Column(String(255), index=True) @@ -41,13 +40,6 @@ class Attribute(Base): disable_correlation = Column(Boolean, default=False) first_seen = Column(BigInteger(), index=True) last_seen = Column(BigInteger(), index=True) - event = relationship( - "Event", - lazy="joined", - viewonly=True - ) - tags = relationship("Tag", secondary="attribute_tags", lazy="subquery") - def to_misp_format( self, settings: Settings = get_settings(), @@ -56,7 +48,6 @@ def to_misp_format( attr_json = { "id": self.id, - "event_id": self.event_id, "object_id": self.object_id, "object_relation": self.object_relation, "category": self.category, @@ -72,7 +63,7 @@ def to_misp_format( "disable_correlation": self.disable_correlation, "first_seen": self.first_seen, "last_seen": self.last_seen, - "Tag": [tag.to_misp_format() for tag in self.tags], + "Tag": [], } # if its a file attribute, we need to handle it differently diff --git a/api/app/models/event.py b/api/app/models/event.py index 926e4366..117a250f 100644 --- a/api/app/models/event.py +++ b/api/app/models/event.py @@ -1,10 +1,4 @@ import enum -import uuid - -from app.database import Base -from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship class DistributionLevel(enum.Enum): @@ -41,87 +35,3 @@ class AnalysisLevel(enum.Enum): COMPLETE = 2 -class Event(Base): - __tablename__ = "events" - - id = Column( - Integer, primary_key=True, index=True, autoincrement=True, nullable=False - ) - org_id = Column(Integer, ForeignKey("organisations.id"), index=True, nullable=False) - date = Column(Date, nullable=False) - info = Column(String, index=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - published = Column(Boolean, default=False, nullable=False) - analysis: Mapped[AnalysisLevel] = mapped_column( - Enum(AnalysisLevel, name="analysis_level"), - nullable=False, - default=AnalysisLevel.INITIAL, - ) - attribute_count = Column(Integer, default=0) - object_count = Column(Integer, default=0) - orgc_id = Column( - Integer, ForeignKey("organisations.id"), index=True, nullable=False - ) - timestamp = Column(Integer, nullable=False, default=0) - distribution: Mapped[DistributionLevel] = mapped_column( - Enum(DistributionLevel, name="distribution_level"), - nullable=False, - default=DistributionLevel.ORGANISATION_ONLY, - ) - sharing_group_id = Column( - Integer, ForeignKey("sharing_groups.id"), index=True, nullable=True - ) - proposal_email_lock = Column(Boolean, nullable=False, default=False) - locked = Column(Boolean, nullable=False, default=False) - threat_level: Mapped[ThreatLevel] = mapped_column( - Enum(ThreatLevel, name="threat_level"), - nullable=False, - default=ThreatLevel.UNDEFINED, - ) - publish_timestamp = Column(Integer, nullable=False, default=0) - sighting_timestamp = Column(Integer, nullable=True) - disable_correlation = Column(Boolean, default=False) - extends_uuid = Column(UUID(as_uuid=True), index=True, nullable=True) - protected = Column(Boolean, nullable=False, default=False) - deleted = Column(Boolean, nullable=False, default=False) - - attributes = relationship("Attribute", lazy="subquery", cascade="all, delete-orphan") - objects = relationship("Object", lazy="subquery", cascade="all, delete-orphan") - sharing_group = relationship("SharingGroup", lazy="joined") - tags = relationship("Tag", secondary="event_tags", lazy="joined") - organisation = relationship("Organisation", lazy="joined", uselist=False, foreign_keys=[org_id]) - - def to_misp_format(self): - """Convert the Event to a MISP-compatible dictionary representation.""" - - return { - "id": self.id, - "org_id": self.org_id, - "date": self.date.isoformat(), - "info": self.info, - "user_id": self.user_id, - "uuid": str(self.uuid), - "published": self.published, - "analysis": self.analysis.value, - "orgc_id": self.orgc_id, - "timestamp": self.timestamp, - "distribution": self.distribution.value, - "sharing_group_id": self.sharing_group_id, - "proposal_email_lock": self.proposal_email_lock, - "locked": self.locked, - "threat_level": self.threat_level.value, - "publish_timestamp": self.publish_timestamp, - "disable_correlation": self.disable_correlation, - "extends_uuid": str(self.extends_uuid) if self.extends_uuid else None, - "protected": self.protected, - "deleted": self.deleted, - "Attribute": [attribute.to_misp_format() for attribute in self.attributes if attribute.object_id == None and not attribute.deleted], - "Object": [obj.to_misp_format() for obj in self.objects if not obj.deleted], - "Tag": [tag.to_misp_format() for tag in self.tags], - "Organisation": { - "id": self.organisation.id, - "name": self.organisation.name, - "uuid": str(self.organisation.uuid), - } if self.organisation else None - } diff --git a/api/app/models/feed.py b/api/app/models/feed.py index 84482d41..73efff30 100644 --- a/api/app/models/feed.py +++ b/api/app/models/feed.py @@ -24,7 +24,7 @@ class Feed(Base): source_format = Column(String, nullable=False) fixed_event = Column(Boolean, nullable=False, default=False) delta_merge = Column(Boolean, nullable=False, default=False) - event_id = Column(Integer, ForeignKey("events.id"), nullable=True) + event_uuid = Column(String, nullable=True) publish = Column(Boolean, nullable=False, default=False) override_ids = Column(Boolean, nullable=False, default=False) settings = Column(JSON, nullable=False, default={}) diff --git a/api/app/models/object.py b/api/app/models/object.py index 1ea4996d..4ee15d2a 100644 --- a/api/app/models/object.py +++ b/api/app/models/object.py @@ -16,7 +16,6 @@ class Object(Base): description = Column(String) template_uuid = Column(String) template_version = Column(Integer, nullable=False) - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) timestamp = Column(Integer, nullable=False) distribution: Mapped[DistributionLevel] = mapped_column( @@ -29,11 +28,6 @@ class Object(Base): deleted = Column(Boolean, nullable=False, default=False) first_seen = Column(Integer) last_seen = Column(Integer) - event = relationship( - "Event", - lazy="joined", - viewonly=True - ) attributes = relationship("Attribute", lazy="subquery", cascade="all, delete-orphan") object_references = relationship("ObjectReference", lazy="subquery", cascade="all, delete-orphan") @@ -46,7 +40,6 @@ def to_misp_format(self): "description": self.description if self.description else self.name, "template_uuid": self.template_uuid, "template_version": self.template_version, - "event_id": self.event_id, "uuid": str(self.uuid), "timestamp": self.timestamp, "distribution": self.distribution.value if self.distribution else None, diff --git a/api/app/models/object_reference.py b/api/app/models/object_reference.py index 0b5a186c..9e417238 100644 --- a/api/app/models/object_reference.py +++ b/api/app/models/object_reference.py @@ -23,7 +23,7 @@ class ObjectReference(Base): uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) timestamp = Column(Integer, nullable=False) object_id = Column(Integer, ForeignKey("objects.id"), nullable=False) - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) + event_uuid = Column(UUID(as_uuid=True), nullable=True) source_uuid = Column(UUID(as_uuid=True)) referenced_uuid = Column(UUID(as_uuid=True)) referenced_id = Column(Integer, nullable=True) @@ -41,7 +41,7 @@ def to_misp_format(self): "uuid": str(self.uuid), "timestamp": self.timestamp, "object_id": self.object_id, - "event_id": self.event_id, + "event_uuid": str(self.event_uuid) if self.event_uuid else None, "source_uuid": str(self.source_uuid) if self.source_uuid else None, "referenced_uuid": str(self.referenced_uuid) if self.referenced_uuid else None, "referenced_id": self.referenced_id, diff --git a/api/app/models/tag.py b/api/app/models/tag.py index 54876f1b..70378ca4 100644 --- a/api/app/models/tag.py +++ b/api/app/models/tag.py @@ -1,5 +1,6 @@ from app.database import Base from sqlalchemy import Boolean, Column, ForeignKey, Integer, String +from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship @@ -39,8 +40,7 @@ class EventTag(Base): __tablename__ = "event_tags" id = Column(Integer, primary_key=True, index=True) - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) - event = relationship("Event", lazy="subquery", overlaps="tags") + event_uuid = Column(UUID(as_uuid=True), nullable=True) tag_id = Column(Integer, ForeignKey("tags.id"), nullable=False) tag = relationship("Tag", lazy="subquery", overlaps="tags") local = Column(Boolean, nullable=False, default=False) @@ -50,9 +50,8 @@ class AttributeTag(Base): __tablename__ = "attribute_tags" id = Column(Integer, primary_key=True, index=True) - attribute_id = Column(Integer, ForeignKey("attributes.id"), nullable=False) - attribute = relationship("Attribute", lazy="subquery", overlaps="tags") - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) + attribute_id = Column(Integer, nullable=True) + event_uuid = Column(UUID(as_uuid=True), nullable=True) tag_id = Column(Integer, ForeignKey("tags.id"), nullable=False) tag = relationship("Tag", lazy="subquery", overlaps="tags") local = Column(Boolean, nullable=False, default=False) diff --git a/api/app/repositories/attachments.py b/api/app/repositories/attachments.py index 5d3cade6..9d0d2c78 100644 --- a/api/app/repositories/attachments.py +++ b/api/app/repositories/attachments.py @@ -77,12 +77,12 @@ def upload_attachments_to_event( meta_category=template["meta_category"], template_version=template["version"], comment=attachment.filename, - event_id=event.id, + event_uuid=event.uuid, timestamp=int(time.time()), ) filename_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="filename", category=attachment_meta.get("category", "External analysis"), type="filename", @@ -100,7 +100,7 @@ def upload_attachments_to_event( sha1.update(file_content) sha1 = sha1.hexdigest() sha1_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="sha1", category="External analysis", type="sha1", @@ -115,7 +115,7 @@ def upload_attachments_to_event( sha256.update(file_content) sha256 = sha256.hexdigest() sha256_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="sha256", category="External analysis", type="sha256", @@ -128,7 +128,7 @@ def upload_attachments_to_event( # get file md5 md5sum = hashlib.md5(file_content).hexdigest() md5_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="md5", category="External analysis", type="md5", @@ -141,7 +141,7 @@ def upload_attachments_to_event( # get file size size = len(file_content) size_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="size-in-bytes", disable_correlation=True, category="Other", @@ -154,7 +154,7 @@ def upload_attachments_to_event( # attachment attribute attachment_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="attachment", category="Payload delivery", type="attachment", @@ -167,7 +167,7 @@ def upload_attachments_to_event( # malware analysis if attachment_meta.get("is_malware", False): malware_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="malware", category="External analysis", type="malware", diff --git a/api/app/repositories/attributes.py b/api/app/repositories/attributes.py index d6ea0f0a..25fd63ba 100644 --- a/api/app/repositories/attributes.py +++ b/api/app/repositories/attributes.py @@ -1,14 +1,14 @@ import math import time +from datetime import datetime from typing import Iterable, Optional, Union -from uuid import UUID +from uuid import UUID, uuid4 from app.models.event import DistributionLevel from app.services.opensearch import get_opensearch_client from app.models import attribute as attribute_models from app.models import tag as tag_models from app.models import user as user_models from app.repositories import attachments as attachments_repository -from app.repositories import events as events_repository from app.schemas import attribute as attribute_schemas from app.schemas import event as event_schemas from app.worker import tasks @@ -66,14 +66,6 @@ def get_attributes( ) -> Page[attribute_schemas.Attribute]: query = select(attribute_models.Attribute) - if event_uuid is not None: - db_event = events_repository.get_event_by_uuid(event_uuid=event_uuid, db=db) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - query = query.where(attribute_models.Attribute.event_id == db_event.id) - if deleted is not None: query = query.where(attribute_models.Attribute.deleted == deleted) @@ -178,56 +170,56 @@ def get_attribute_by_uuid( def create_attribute( db: Session, attribute: attribute_schemas.AttributeCreate -) -> attribute_models.Attribute: - db_attribute = attribute_models.Attribute( - event_id=attribute.event_id, - object_id=( - attribute.object_id - if attribute.object_id is not None and attribute.object_id > 0 - else None - ), - object_relation=attribute.object_relation, - category=attribute.category, - type=attribute.type, - value=attribute.value, - to_ids=attribute.to_ids, - uuid=attribute.uuid, - timestamp=attribute.timestamp or time.time(), - distribution=( - event_schemas.DistributionLevel(attribute.distribution) - if attribute.distribution is not None - else event_schemas.DistributionLevel.INHERIT_EVENT - ), - sharing_group_id=attribute.sharing_group_id, - comment=attribute.comment, - deleted=attribute.deleted, - disable_correlation=attribute.disable_correlation, - first_seen=attribute.first_seen, - last_seen=attribute.last_seen, - ) - db.add(db_attribute) - db.commit() - db.refresh(db_attribute) +) -> attribute_schemas.Attribute: + client = get_opensearch_client() + attribute_uuid = str(attribute.uuid or uuid4()) + now = int(time.time()) + + event_uuid = str(attribute.event_uuid) if attribute.event_uuid else None + + dist = attribute.distribution + dist_val = dist.value if hasattr(dist, "value") else (dist if dist is not None else 5) + + attr_doc = { + "uuid": attribute_uuid, + "event_uuid": event_uuid, + "object_id": attribute.object_id if attribute.object_id and attribute.object_id > 0 else None, + "object_uuid": None, + "object_relation": attribute.object_relation, + "category": attribute.category, + "type": attribute.type, + "value": attribute.value, + "to_ids": attribute.to_ids if attribute.to_ids is not None else True, + "timestamp": attribute.timestamp or now, + "distribution": dist_val, + "sharing_group_id": attribute.sharing_group_id, + "comment": attribute.comment or "", + "deleted": attribute.deleted or False, + "disable_correlation": attribute.disable_correlation or False, + "first_seen": attribute.first_seen, + "last_seen": attribute.last_seen, + "data": "", + "tags": [], + "@timestamp": datetime.fromtimestamp(attribute.timestamp or now).isoformat(), + } - if db_attribute is not None: - tasks.handle_created_attribute( - db_attribute.id, db_attribute.object_id, db_attribute.event_id - ) + client.index(index="misp-attributes", id=attribute_uuid, body=attr_doc, refresh=True) - return db_attribute + tasks.handle_created_attribute(attribute_uuid, attr_doc["object_id"], event_uuid) + + return attribute_schemas.Attribute.model_validate(attr_doc) def create_attribute_from_pulled_attribute( db: Session, pulled_attribute: MISPAttribute, - local_event_id: int, + event_uuid: str, user: user_models.User, ) -> attribute_models.Attribute: # TODO: process sharing group // captureSG # TODO: enforce warninglist local_attribute = attribute_models.Attribute( - event_id=local_event_id, category=pulled_attribute.category, type=pulled_attribute.type, value=( @@ -270,7 +262,7 @@ def create_attribute_from_pulled_attribute( # TODO: process galaxies capture_attribute_tags( - db, local_attribute, pulled_attribute.tags, local_event_id, user + db, local_attribute, pulled_attribute.tags, event_uuid, user ) return local_attribute @@ -280,16 +272,14 @@ def update_attribute_from_pulled_attribute( db: Session, local_attribute: attribute_models.Attribute, pulled_attribute: MISPAttribute, - local_event_id: int, + event_uuid: str, user: user_models.User, ) -> attribute_models.Attribute: pulled_attribute.id = local_attribute.id - pulled_attribute.event_id = local_event_id if local_attribute.timestamp < pulled_attribute.timestamp.timestamp(): attribute_patch = attribute_schemas.AttributeUpdate( - event_id=local_event_id, category=pulled_attribute.category, type=pulled_attribute.type, value=( @@ -329,73 +319,68 @@ def update_attribute_from_pulled_attribute( ) capture_attribute_tags( - db, local_attribute, pulled_attribute.tags, local_event_id, user + db, local_attribute, pulled_attribute.tags, event_uuid, user ) # TODO: process sigthings # TODO: process galaxies tasks.handle_updated_attribute.delay( - local_attribute.id, local_attribute.object_id, local_attribute.event_id + str(local_attribute.uuid), local_attribute.object_id, None ) return local_attribute def update_attribute( - db: Session, attribute_id: int, attribute: attribute_schemas.AttributeUpdate -) -> attribute_models.Attribute: - # TODO: Attribute::beforeValidate() && Attribute::$validate - db_attribute = get_attribute_by_id(db, attribute_id=attribute_id) - - if db_attribute is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" - ) + db: Session, attribute_id: Union[int, UUID], attribute: attribute_schemas.AttributeUpdate +) -> attribute_schemas.Attribute: + client = get_opensearch_client() + os_attr = get_attribute_from_opensearch(attribute_id) + if os_attr is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") - attribute_patch = attribute.model_dump(exclude_unset=True) - for key, value in attribute_patch.items(): - setattr(db_attribute, key, value) + patch = attribute.model_dump(exclude_unset=True) + for k, v in list(patch.items()): + if hasattr(v, "value"): + patch[k] = v.value - db.add(db_attribute) - db.commit() - db.refresh(db_attribute) + client.update(index="misp-attributes", id=str(os_attr.uuid), body={"doc": patch}, refresh=True) - tasks.handle_updated_attribute( - db_attribute.id, db_attribute.object_id, db_attribute.event_id - ) + tasks.handle_updated_attribute(str(os_attr.uuid), os_attr.object_id, str(os_attr.event_uuid) if os_attr.event_uuid else None) - return db_attribute + return get_attribute_from_opensearch(os_attr.uuid) -def delete_attribute(db: Session, attribute_id: int | str) -> None: +def delete_attribute(db: Session, attribute_id: Union[int, str, UUID]) -> None: + client = get_opensearch_client() if isinstance(attribute_id, str): - db_attribute = get_attribute_by_uuid(db, attribute_uuid=UUID(attribute_id)) + try: + os_attr = get_attribute_from_opensearch(UUID(attribute_id)) + except ValueError: + os_attr = None else: - db_attribute = get_attribute_by_id(db, attribute_id=attribute_id) + os_attr = get_attribute_from_opensearch(attribute_id) - if db_attribute is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" - ) - - db_attribute.deleted = True - - db.add(db_attribute) - db.commit() - db.refresh(db_attribute) + if os_attr is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") - tasks.handle_deleted_attribute( - db_attribute.id, db_attribute.object_id, db_attribute.event_id + client.update( + index="misp-attributes", + id=str(os_attr.uuid), + body={"doc": {"deleted": True}}, + refresh=True, ) + tasks.handle_deleted_attribute(str(os_attr.uuid), os_attr.object_id, str(os_attr.event_uuid) if os_attr.event_uuid else None) + def capture_attribute_tags( db: Session, db_attribute: attribute_models.Attribute, tags: list[MISPTag], - local_event_id: int, + event_uuid: str, user: user_models.User, ): tag_name_to_db_tag = {} @@ -437,7 +422,7 @@ def capture_attribute_tags( db_attribute_tag = tag_models.AttributeTag( attribute=db_attribute, - event_id=local_event_id, + event_uuid=event_uuid, tag_id=db_tag.id, local=tag.local, ) @@ -449,21 +434,19 @@ def capture_attribute_tags( def get_vulnerability_attributes( db: Session, event_uuid: str = None ) -> list[attribute_schemas.Attribute]: - query = select(attribute_models.Attribute).where( - attribute_models.Attribute.type == "vulnerability" - ) - + client = get_opensearch_client() + must_clauses = [{"term": {"type.keyword": "vulnerability"}}] if event_uuid is not None: - db_event = events_repository.get_event_by_uuid(event_uuid=event_uuid, db=db) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - query = query.where(attribute_models.Attribute.event_id == db_event.id) - - results = db.execute(query).scalars().unique().all() + must_clauses.append({"term": {"event_uuid.keyword": event_uuid}}) - return results + response = client.search( + index="misp-attributes", + body={"query": {"bool": {"must": must_clauses}}, "size": 10000}, + ) + return [ + attribute_schemas.Attribute.model_validate(h["_source"]) + for h in response["hits"]["hits"] + ] def search_attributes( diff --git a/api/app/repositories/events.py b/api/app/repositories/events.py index 35c9c652..29028298 100644 --- a/api/app/repositories/events.py +++ b/api/app/repositories/events.py @@ -2,59 +2,31 @@ import math import time from datetime import datetime -from uuid import UUID +from uuid import UUID, uuid4 from typing import Optional, Union, Iterable from app.worker import tasks from app.services.opensearch import get_opensearch_client from app.services.vulnerability_lookup import lookup as vulnerability_lookup from app.services.rulezet import lookup as rulezet_lookup -from app.models import event as event_models from app.models import feed as feed_models from app.models import tag as tag_models +from app.models import organisation as org_models from app.repositories import tags as tags_repository from app.repositories import attributes as attributes_repository from app.schemas import event as event_schemas from app.schemas import user as user_schemas +from app.schemas import organisations as org_schemas import app.schemas.attribute as attribute_schemas import app.schemas.vulnerability as vulnerability_schemas from fastapi import HTTPException, status, Query from fastapi_pagination import Page, Params -from fastapi_pagination.ext.sqlalchemy import paginate from opensearchpy.exceptions import NotFoundError from pymisp import MISPEvent, MISPOrganisation -from sqlalchemy.orm import Session, noload -from sqlalchemy.sql import select +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) -def get_events(db: Session, info: str = Query(None), deleted: bool = Query(None), uuid: str = Query(None), include_attributes: bool = Query(False)): - query = select(event_models.Event) - - if include_attributes: - query = select(event_models.Event) - else: - # avoid loading child relationships (attributes/objects) to keep the query lightweight - query = select(event_models.Event).options( - noload(event_models.Event.attributes), noload(event_models.Event.objects) - ) - - if info is not None: - search = f"%{info}%" - query = query.where(event_models.Event.info.like(search)) - - if deleted is not None: - query = query.where(event_models.Event.deleted == deleted) - - if uuid is not None: - query = query.where(event_models.Event.uuid == uuid) - - # Sort the query by timestamp in descending order - query = query.order_by(event_models.Event.timestamp.desc()) - - return paginate(db, query) - - def get_events_from_opensearch( params: Params, info: str = None, @@ -124,16 +96,18 @@ def search_events( size: int = 10, sort_by: str = "@timestamp", sort_order: str = "desc", + searchAttributes: bool = False, ): OpenSearchClient = get_opensearch_client() + index = "misp-attributes" if searchAttributes else "misp-events" search_body = { "query": {"query_string": {"query": query, "default_field": "info"}}, "from": from_value, "size": size, "sort": [{sort_by: {"order": sort_order}}], } - response = OpenSearchClient.search(index="misp-events", body=search_body) + response = OpenSearchClient.search(index=index, body=search_body) return { "page": page, @@ -186,237 +160,164 @@ def export_events( search_after = hits[-1].get("sort") -def get_event_by_id(db: Session, event_id: int): - return ( - db.query(event_models.Event).filter(event_models.Event.id == event_id).first() - ) - - -def get_event_by_uuid(db: Session, event_uuid: str): - return ( - db.query(event_models.Event) - .filter(event_models.Event.uuid == event_uuid) - .first() +def get_event_by_info(info: str) -> Optional[event_schemas.Event]: + client = get_opensearch_client() + response = client.search( + index="misp-events", + body={"query": {"term": {"info.keyword": info}}, "size": 1}, ) + hits = response["hits"]["hits"] + if not hits: + return None + source = hits[0]["_source"] + source.setdefault("attributes", []) + source.setdefault("objects", []) + return event_schemas.Event.model_validate(source) -def get_user_by_info(db: Session, info: str): - return db.query(event_models.Event).filter(event_models.Event.info == info).first() - - -def create_event(db: Session, event: event_schemas.EventCreate) -> event_models.Event: - # TODO: Event::beforeValidate() && Event::$validate - db_event = event_models.Event( - org_id=event.org_id, - date=event.date or datetime.now(), - info=event.info, - user_id=event.user_id, - uuid=event.uuid, - published=event.published, - analysis=event_models.AnalysisLevel(event.analysis), - attribute_count=event.attribute_count, - object_count=event.object_count, - orgc_id=event.orgc_id or event.org_id, - timestamp=event.timestamp or time.time(), - distribution=event_models.DistributionLevel(event.distribution), - sharing_group_id=event.sharing_group_id, - proposal_email_lock=event.proposal_email_lock, - locked=event.locked, - threat_level=event_models.ThreatLevel(event.threat_level), - publish_timestamp=event.publish_timestamp, - sighting_timestamp=event.sighting_timestamp, - disable_correlation=event.disable_correlation, - extends_uuid=event.extends_uuid, - protected=event.protected, - deleted=event.deleted, - ) - db.add(db_event) - db.commit() - db.flush() - db.refresh(db_event) - - tasks.handle_created_event(str(db_event.uuid)) - - return db_event - - -def create_event_from_pulled_event(db: Session, pulled_event: MISPEvent): - event = event_models.Event( - org_id=pulled_event.org_id, - date=pulled_event.date, - info=pulled_event.info, - user_id=pulled_event.user_id, - uuid=pulled_event.uuid, - published=pulled_event.published, - analysis=event_models.AnalysisLevel(pulled_event.analysis), - attribute_count=pulled_event.attribute_count, - object_count=len(pulled_event.objects), - orgc_id=pulled_event.orgc_id, - timestamp=pulled_event.timestamp.timestamp(), - distribution=event_models.DistributionLevel(pulled_event.distribution), - sharing_group_id=( - pulled_event.sharing_group_id - if pulled_event.sharing_group_id is not None - and int(pulled_event.sharing_group_id) > 0 - else None - ), - proposal_email_lock=pulled_event.proposal_email_lock, - locked=pulled_event.locked, - threat_level=event_models.ThreatLevel(pulled_event.threat_level_id), - publish_timestamp=pulled_event.publish_timestamp.timestamp(), - # sighting_timestamp=pulled_event.sighting_timestamp, # TODO: add sighting_timestamp - disable_correlation=pulled_event.disable_correlation, - extends_uuid=pulled_event.extends_uuid or None, - # protected=pulled_event.protected # TODO: add protected [pymisp] +def get_event_uuids_from_opensearch() -> list[str]: + """Return all event UUIDs from OpenSearch (used for server push).""" + client = get_opensearch_client() + response = client.search( + index="misp-events", + body={"query": {"match_all": {}}, "size": 10000, "_source": ["uuid"]}, ) - db.add(event) - db.commit() - db.refresh(event) + return [hit["_source"]["uuid"] for hit in response["hits"]["hits"]] - tasks.handle_created_event.delay(str(event.uuid)) - return event - - -def update_event_from_pulled_event( - db: Session, existing_event: event_models.Event, pulled_event: MISPEvent -): - existing_event.date = pulled_event.date - existing_event.info = pulled_event.info - existing_event.uuid = pulled_event.uuid - existing_event.published = pulled_event.published - existing_event.attribute_count = pulled_event.attribute_count - existing_event.object_count = len(pulled_event.objects) - existing_event.analysis = event_models.AnalysisLevel(pulled_event.analysis) - existing_event.timestamp = pulled_event.timestamp.timestamp() or time.time() - existing_event.distribution = event_models.DistributionLevel( - pulled_event.distribution - ) - existing_event.sharing_group_id = ( - pulled_event.sharing_group_id - if int(pulled_event.sharing_group_id) > 0 - else None +def get_events_by_uuids_from_opensearch(uuids) -> list[event_schemas.Event]: + """Return events matching the given UUIDs from OpenSearch.""" + client = get_opensearch_client() + uuids = [str(u) for u in uuids] + if not uuids: + return [] + response = client.search( + index="misp-events", + body={"query": {"terms": {"uuid.keyword": uuids}}, "size": len(uuids)}, ) - existing_event.threat_level = event_models.ThreatLevel(pulled_event.threat_level_id) - existing_event.disable_correlation = pulled_event.disable_correlation - existing_event.extends_uuid = pulled_event.extends_uuid or None - db.commit() - db.refresh(existing_event) - - tasks.handle_updated_event.delay(str(existing_event.uuid)) - - return existing_event - - -def update_event(db: Session, event_id: int, event: event_schemas.EventUpdate): - # TODO: Event::beforeValidate() && Event::$validate - db_event = get_event_by_id(db, event_id=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - event_patch = event.model_dump(exclude_unset=True) - for key, value in event_patch.items(): - setattr(db_event, key, value) - - db.commit() - db.refresh(db_event) - - tasks.handle_updated_event(str(db_event.uuid)) - - return db_event - - -def delete_event(db: Session, event_id: Union[int, UUID], force: bool = False) -> None: - - if isinstance(event_id, int): - db_event = get_event_by_id(db, event_id=event_id) - else: - db_event = get_event_by_uuid(db, event_uuid=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - db_event.deleted = True - - if force: - event_uuid = str(db_event.uuid) - db.delete(db_event) - db.commit() - tasks.delete_indexed_event(event_uuid) - return - - db.commit() - db.refresh(db_event) - - tasks.handle_deleted_event(str(db_event.uuid)) - - -def increment_attribute_count( - db: Session, event_id: int, attributes_count: int = 1 -) -> None: - db_event = get_event_by_id(db, event_id=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - db_event.attribute_count += attributes_count - - db.commit() - db.refresh(db_event) - - -def decrement_attribute_count( - db: Session, event_id: int, attributes_count: int = 1 -) -> None: - db_event = get_event_by_id(db, event_id=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) + events = [] + for hit in response["hits"]["hits"]: + source = hit["_source"] + source.setdefault("attributes", []) + source.setdefault("objects", []) + events.append(event_schemas.Event.model_validate(source)) + return events - if db_event.attribute_count > 0: - db_event.attribute_count -= attributes_count - db.commit() - db.refresh(db_event) +def create_event(db: Session, event: event_schemas.EventCreate) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(event.uuid or uuid4()) + now = int(time.time()) + ts = event.timestamp or now + + org = db.query(org_models.Organisation).filter_by(id=event.org_id).first() + org_dict = org_schemas.Organisation.model_validate(org).model_dump(mode="json") if org else None + + event_doc = { + "uuid": event_uuid, + "org_id": event.org_id, + "date": (event.date or datetime.now()).strftime("%Y-%m-%d") if event.date else datetime.now().strftime("%Y-%m-%d"), + "info": event.info, + "user_id": event.user_id, + "published": event.published or False, + "analysis": event.analysis.value if hasattr(event.analysis, "value") else (event.analysis or 0), + "attribute_count": 0, + "object_count": 0, + "orgc_id": event.orgc_id or event.org_id, + "timestamp": ts, + "distribution": event.distribution.value if hasattr(event.distribution, "value") else (event.distribution or 0), + "sharing_group_id": event.sharing_group_id, + "sharing_group": None, + "proposal_email_lock": event.proposal_email_lock or False, + "locked": event.locked or False, + "threat_level": event.threat_level.value if hasattr(event.threat_level, "value") else (event.threat_level or 4), + "publish_timestamp": event.publish_timestamp or 0, + "sighting_timestamp": event.sighting_timestamp, + "disable_correlation": event.disable_correlation or False, + "extends_uuid": str(event.extends_uuid) if event.extends_uuid else None, + "protected": event.protected or False, + "deleted": event.deleted or False, + "tags": [], + "attributes": [], + "objects": [], + "organisation": org_dict, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } -def increment_object_count(db: Session, event_id: int, objects_count: int = 1) -> None: - db_event = get_event_by_id(db, event_id=event_id) + client.index(index="misp-events", id=event_uuid, body=event_doc, refresh=True) + tasks.handle_created_event(event_uuid) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) + return event_schemas.Event.model_validate(event_doc) - db_event.object_count += objects_count - db.commit() - db.refresh(db_event) +def create_event_from_pulled_event(pulled_event: MISPEvent) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(pulled_event.uuid) + ts = int(pulled_event.timestamp.timestamp()) + + event_doc = { + "uuid": event_uuid, + "org_id": pulled_event.org_id, + "orgc_id": pulled_event.orgc_id or pulled_event.org_id, + "date": pulled_event.date.isoformat() if hasattr(pulled_event.date, "isoformat") else str(pulled_event.date), + "info": pulled_event.info, + "user_id": pulled_event.user_id, + "published": pulled_event.published or False, + "analysis": int(pulled_event.analysis) if pulled_event.analysis is not None else 0, + "attribute_count": pulled_event.attribute_count or 0, + "object_count": len(pulled_event.objects), + "timestamp": ts, + "distribution": int(pulled_event.distribution) if pulled_event.distribution is not None else 0, + "sharing_group_id": int(pulled_event.sharing_group_id) if pulled_event.sharing_group_id and int(pulled_event.sharing_group_id) > 0 else None, + "proposal_email_lock": pulled_event.proposal_email_lock or False, + "locked": pulled_event.locked or False, + "threat_level": int(pulled_event.threat_level_id) if pulled_event.threat_level_id else 4, + "publish_timestamp": int(pulled_event.publish_timestamp.timestamp()), + "disable_correlation": pulled_event.disable_correlation or False, + "extends_uuid": str(pulled_event.extends_uuid) if pulled_event.extends_uuid else None, + "protected": getattr(pulled_event, "protected", False) or False, + "deleted": pulled_event.deleted or False, + "tags": [], + "attributes": [], + "objects": [], + "organisation": None, + "sharing_group": None, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } + client.index(index="misp-events", id=event_uuid, body=event_doc, refresh=True) + tasks.handle_created_event.delay(event_uuid) -def decrement_object_count(db: Session, event_id: int, objects_count: int = 1) -> None: - db_event = get_event_by_id(db, event_id=event_id) + return event_schemas.Event.model_validate(event_doc) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - db_event.object_count -= objects_count +def update_event_from_pulled_event( + existing_event: event_schemas.Event, pulled_event: MISPEvent +) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(existing_event.uuid) + ts = int(pulled_event.timestamp.timestamp()) + + patch = { + "date": pulled_event.date.isoformat() if hasattr(pulled_event.date, "isoformat") else str(pulled_event.date), + "info": pulled_event.info, + "published": pulled_event.published or False, + "analysis": int(pulled_event.analysis) if pulled_event.analysis is not None else 0, + "attribute_count": pulled_event.attribute_count or 0, + "object_count": len(pulled_event.objects), + "timestamp": ts, + "distribution": int(pulled_event.distribution) if pulled_event.distribution is not None else 0, + "sharing_group_id": int(pulled_event.sharing_group_id) if pulled_event.sharing_group_id and int(pulled_event.sharing_group_id) > 0 else None, + "threat_level": int(pulled_event.threat_level_id) if pulled_event.threat_level_id else 4, + "disable_correlation": pulled_event.disable_correlation or False, + "extends_uuid": str(pulled_event.extends_uuid) if pulled_event.extends_uuid else None, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } - if db_event.object_count < 0: - db_event.object_count = 0 + client.update(index="misp-events", id=event_uuid, body={"doc": patch}, refresh=True) + tasks.handle_updated_event.delay(event_uuid) - db.commit() - db.refresh(db_event) + return get_event_from_opensearch(UUID(event_uuid)) def create_event_from_fetched_event( @@ -425,72 +326,64 @@ def create_event_from_fetched_event( Orgc: MISPOrganisation, feed: feed_models.Feed, user: user_schemas.User, -) -> event_models.Event: - db_event = event_models.Event( - org_id=Orgc.id, - date=fetched_event.date, - info=fetched_event.info, - user_id=user.id, - uuid=fetched_event.uuid, - published=fetched_event.published, - analysis=event_models.AnalysisLevel(fetched_event.analysis), - object_count=len(fetched_event.objects), - orgc_id=Orgc.id, - timestamp=fetched_event.timestamp.timestamp(), - distribution=feed.distribution, - sharing_group_id=feed.sharing_group_id, - locked=(fetched_event.locked if hasattr(fetched_event, "locked") else False), - threat_level=event_models.ThreatLevel(fetched_event.threat_level_id), - publish_timestamp=fetched_event.publish_timestamp.timestamp(), - # sighting_timestamp=fetched_event.sighting_timestamp, # TODO: add sighting_timestamp - disable_correlation=getattr(fetched_event, "disable_correlation", False), - extends_uuid=( - fetched_event.extends_uuid - if hasattr(fetched_event, "extends_uuid") - and fetched_event.extends_uuid != "" +) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(fetched_event.uuid) + ts = int(fetched_event.timestamp.timestamp()) + + event_doc = { + "uuid": event_uuid, + "org_id": Orgc.id, + "orgc_id": Orgc.id, + "date": fetched_event.date.isoformat() if hasattr(fetched_event.date, "isoformat") else str(fetched_event.date), + "info": fetched_event.info, + "user_id": user.id, + "published": fetched_event.published or False, + "analysis": int(fetched_event.analysis) if fetched_event.analysis is not None else 0, + "attribute_count": 0, + "object_count": len(fetched_event.objects), + "timestamp": ts, + "distribution": feed.distribution.value if hasattr(feed.distribution, "value") else int(feed.distribution), + "sharing_group_id": feed.sharing_group_id, + "locked": (fetched_event.locked if hasattr(fetched_event, "locked") else False), + "threat_level": int(fetched_event.threat_level_id) if fetched_event.threat_level_id else 4, + "publish_timestamp": int(fetched_event.publish_timestamp.timestamp()), + "disable_correlation": getattr(fetched_event, "disable_correlation", False), + "extends_uuid": ( + str(fetched_event.extends_uuid) + if hasattr(fetched_event, "extends_uuid") and fetched_event.extends_uuid != "" else None ), - # protected=fetched_event.protected # TODO: add protected [pymisp] - ) + "protected": False, + "deleted": False, + "proposal_email_lock": False, + "tags": [], + "attributes": [], + "objects": [], + "organisation": None, + "sharing_group": None, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } - db.add(db_event) + client.index(index="misp-events", id=event_uuid, body=event_doc, refresh=True) - # process tags + # process tags into OS event doc for tag in fetched_event.tags: db_tag = tags_repository.get_tag_by_name(db, tag.name) - if db_tag is None: - # create tag if not exists db_tag = tag_models.Tag( name=tag.name, colour=tag.colour, org_id=user.org_id, user_id=user.id, local_only=tag.local, - # exportable=tag.exportable, - # hide_tag=tag.hide_tag, - # numerical_value=tag.numerical_value, - # is_galaxy=tag.is_galaxy, - # is_custom_galaxy=tag.is_custom_galaxy, ) db.add(db_tag) + db.commit() + db.refresh(db_tag) + tags_repository.tag_event(db, event_schemas.Event.model_validate(event_doc), db_tag) - db_event_tag = tag_models.EventTag( - event=db_event, - tag=db_tag, - local=tag.local, - ) - db.add(db_event_tag) - - # TODO: process galaxies - # TODO: process reports - # TODO: process analyst notes - - db.commit() - db.flush() - db.refresh(db_event) - - return db_event + return event_schemas.Event.model_validate(event_doc) def update_event_from_fetched_event( @@ -499,146 +392,183 @@ def update_event_from_fetched_event( Orgc: MISPOrganisation, feed: feed_models.Feed, user: user_schemas.User, -) -> event_models.Event: - db_event = get_event_by_uuid(db, fetched_event.uuid) +) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(fetched_event.uuid) + + existing = get_event_from_opensearch(UUID(event_uuid)) + if existing is None: + logger.error(f"Event {event_uuid} not found in OpenSearch") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + ts = int(fetched_event.timestamp.timestamp()) + patch = { + "date": fetched_event.date.isoformat() if hasattr(fetched_event.date, "isoformat") else str(fetched_event.date), + "info": fetched_event.info, + "published": fetched_event.published or False, + "analysis": int(fetched_event.analysis) if fetched_event.analysis is not None else 0, + "object_count": len(fetched_event.objects), + "org_id": Orgc.id, + "orgc_id": Orgc.id, + "timestamp": ts, + "distribution": feed.distribution.value if hasattr(feed.distribution, "value") else int(feed.distribution), + "sharing_group_id": feed.sharing_group_id, + "locked": (fetched_event.locked if hasattr(fetched_event, "locked") else False), + "threat_level": int(fetched_event.threat_level_id) if fetched_event.threat_level_id else 4, + "publish_timestamp": int(fetched_event.publish_timestamp.timestamp()), + "disable_correlation": getattr(fetched_event, "disable_correlation", False), + "extends_uuid": ( + str(fetched_event.extends_uuid) + if hasattr(fetched_event, "extends_uuid") and fetched_event.extends_uuid != "" + else None + ), + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } - if db_event is None: - logger.error(f"Event {fetched_event.uuid} not found") - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - db_event.date = fetched_event.date - db_event.info = fetched_event.info - db_event.published = fetched_event.published - db_event.analysis = event_models.AnalysisLevel(fetched_event.analysis) - db_event.object_count = len(fetched_event.objects) - db_event.org_id = Orgc.id - db_event.orgc_id = Orgc.id - db_event.timestamp = fetched_event.timestamp.timestamp() - db_event.distribution = feed.distribution - db_event.sharing_group_id = feed.sharing_group_id - db_event.locked = ( - fetched_event.locked if hasattr(fetched_event, "locked") else False - ) - db_event.threat_level = event_models.ThreatLevel(fetched_event.threat_level_id) - db_event.publish_timestamp = fetched_event.publish_timestamp.timestamp() - db_event.disable_correlation = getattr(fetched_event, "disable_correlation", False) - db_event.extends_uuid = ( - fetched_event.extends_uuid - if hasattr(fetched_event, "extends_uuid") and fetched_event.extends_uuid != "" - else None - ) + client.update(index="misp-events", id=event_uuid, body={"doc": patch}, refresh=True) # process tags for tag in fetched_event.tags: db_tag = tags_repository.get_tag_by_name(db, tag.name) - if db_tag is None: - # create tag if not exists db_tag = tag_models.Tag( name=tag.name, colour=tag.colour, org_id=user.org_id, user_id=user.id, local_only=tag.local, - # exportable=tag.exportable, - # hide_tag=tag.hide_tag, - # numerical_value=tag.numerical_value, - # is_galaxy=tag.is_galaxy, - # is_custom_galaxy=tag.is_custom_galaxy, ) db.add(db_tag) + db.commit() + db.refresh(db_tag) + tags_repository.tag_event(db, existing, db_tag) - db_event_tag = tag_models.EventTag( - event=db_event, - tag=db_tag, - local=tag.local, - ) - db.add(db_event_tag) + return get_event_from_opensearch(UUID(event_uuid)) - # remove tags that are not in fetched event - event_tags = db.query(tag_models.EventTag).filter( - tag_models.EventTag.event_id == db_event.id - ) - for event_tag in event_tags: - if event_tag.tag.name not in [tag.name in fetched_event.tags]: - db.delete(event_tag) - # TODO: process galaxies - # TODO: process reports - # TODO: process analyst notes +def update_event(db: Session, event_id: Union[int, UUID], event: event_schemas.EventUpdate) -> event_schemas.Event: + client = get_opensearch_client() + os_event = get_event_from_opensearch(event_id) + if os_event is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + patch = event.model_dump(exclude_unset=True) + for k, v in list(patch.items()): + if hasattr(v, "value"): + patch[k] = v.value - db.commit() - db.flush() - db.refresh(db_event) + client.update(index="misp-events", id=str(os_event.uuid), body={"doc": patch}, refresh=True) + tasks.handle_updated_event(str(os_event.uuid)) - return db_event + return get_event_from_opensearch(os_event.uuid) -def get_event_uuids(db: Session) -> list[UUID]: - return db.query(event_models.Event.uuid).all() +def delete_event(db: Session, event_id: Union[int, UUID], force: bool = False) -> None: + client = get_opensearch_client() + os_event = get_event_from_opensearch(event_id) + if os_event is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + event_uuid = str(os_event.uuid) -def get_events_by_uuids(db: Session, uuids: list[UUID]) -> list[event_models.Event]: - return ( - db.query(event_models.Event) - .options(noload("*")) - .filter(event_models.Event.uuid.in_(uuids)) - .all() + if force: + tasks.delete_indexed_event(event_uuid) + return + + # Soft delete: mark deleted=True in OS but keep the document so it remains searchable + client.update(index="misp-events", id=event_uuid, body={"doc": {"deleted": True}}, refresh=True) + + +def increment_attribute_count(db: Session, event_uuid: str, attributes_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": f"ctx._source.attribute_count += {attributes_count}", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, ) -def publish_event(db: Session, db_event: event_models.Event) -> event_models.Event: +def decrement_attribute_count(db: Session, event_uuid: str, attributes_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": f"if (ctx._source.attribute_count > 0) {{ ctx._source.attribute_count -= {attributes_count}; }}", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, + ) - if db_event.published: - return db_event - db_event.published = True - db_event.publish_timestamp = time.time() +def increment_object_count(db: Session, event_uuid: str, objects_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": f"ctx._source.object_count += {objects_count}", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, + ) - db.commit() - db.refresh(db_event) - tasks.handle_published_event(str(db_event.uuid)) - tasks.index_event(str(db_event.uuid), full_reindex=False) +def decrement_object_count(db: Session, event_uuid: str, objects_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": "if (ctx._source.object_count > 0) { ctx._source.object_count -= 1; } else { ctx._source.object_count = 0; }", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, + ) - return db_event +def publish_event(event: event_schemas.Event) -> event_schemas.Event: + client = get_opensearch_client() + if event.published: + return event -def unpublish_event(db: Session, db_event: event_models.Event) -> event_models.Event: + patch = {"published": True, "publish_timestamp": int(time.time())} + client.update(index="misp-events", id=str(event.uuid), body={"doc": patch}, refresh=True) - if not db_event.published: - return db_event + tasks.handle_published_event(str(event.uuid)) - db_event.published = False + return get_event_from_opensearch(event.uuid) - db.commit() - db.refresh(db_event) - tasks.handle_unpublished_event(str(db_event.uuid)) - tasks.index_event(str(db_event.uuid), full_reindex=False) +def unpublish_event(event: event_schemas.Event) -> event_schemas.Event: + client = get_opensearch_client() + if not event.published: + return event - return db_event + client.update(index="misp-events", id=str(event.uuid), body={"doc": {"published": False}}, refresh=True) + tasks.handle_unpublished_event(str(event.uuid)) -def toggle_event_correlation( - db: Session, db_event: event_models.Event -) -> event_models.Event: - db_event.disable_correlation = not db_event.disable_correlation + return get_event_from_opensearch(event.uuid) - db.commit() - db.refresh(db_event) - tasks.handle_toggled_event_correlation( - str(db_event.uuid), db_event.disable_correlation +def toggle_event_correlation(event: event_schemas.Event) -> event_schemas.Event: + client = get_opensearch_client() + new_val = not event.disable_correlation + + client.update( + index="misp-events", + id=str(event.uuid), + body={"doc": {"disable_correlation": new_val}}, + refresh=True, ) - tasks.index_event(str(db_event.uuid), full_reindex=False) - return db_event + tasks.handle_toggled_event_correlation(str(event.uuid), new_val) + + return get_event_from_opensearch(event.uuid) -def import_data(db: Session, event: event_models.Event, data: dict): +def import_data(db: Session, event: event_schemas.Event, data: dict): total_imported_attributes = 0 total_attributes = 0 @@ -649,7 +579,7 @@ def import_data(db: Session, event: event_models.Event, data: dict): for raw_attribute in data["attributes"]: try: attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, category=raw_attribute.get("category", "External analysis"), type=raw_attribute["type"], value=raw_attribute["value"], diff --git a/api/app/repositories/feeds.py b/api/app/repositories/feeds.py index e1025974..0c67ccdd 100644 --- a/api/app/repositories/feeds.py +++ b/api/app/repositories/feeds.py @@ -46,7 +46,7 @@ def create_feed(db: Session, feed: feed_schemas.FeedCreate): source_format=feed.source_format, fixed_event=feed.fixed_event, delta_merge=feed.delta_merge, - event_id=feed.event_id, + event_uuid=feed.event_uuid, publish=feed.publish, override_ids=feed.override_ids, settings=feed.settings, @@ -158,12 +158,12 @@ def process_feed_event( # process objects sync_repository.create_pulled_event_objects( - db, local_event.id, event.objects, user + db, str(local_event.uuid), event.objects, user ) # process attributes sync_repository.create_pulled_event_attributes( - db, local_event.id, event.attributes, user + db, str(local_event.uuid), event.attributes, user ) else: @@ -179,22 +179,16 @@ def process_feed_event( # process objects sync_repository.update_pulled_event_objects( - db, local_event.id, event.objects, user + db, str(local_event.uuid), event.objects, user ) # process attributes sync_repository.update_pulled_event_attributes( - db, local_event.id, event.attributes, user + db, str(local_event.uuid), event.attributes, user ) - local_event.attribute_count = len(local_event.attributes) - local_event.object_count = len(local_event.objects) - db.add(local_event) - db.commit() - tasks.index_event.delay(str(local_event.uuid), full_reindex=True) - return {"result": "success", "message": "Event processed"} @@ -736,21 +730,21 @@ def get_or_create_feed_event( ): label = _feed_import_label(db_feed) if db_feed.fixed_event: - if db_feed.event_id is None: - db_event = _create_feed_event(db, db_feed, user, label) - db_feed.event_id = db_event.id + if db_feed.event_uuid is None: + os_event = _create_feed_event(db, db_feed, user, label) + db_feed.event_uuid = str(os_event.uuid) db.commit() db.refresh(db_feed) else: - db_event = events_repository.get_event_by_id(db, db_feed.event_id) + os_event = events_repository.get_event_from_opensearch(db_feed.event_uuid) - if db_event is None or db_event.deleted: - db_event = _create_feed_event(db, db_feed, user, label) - db_feed.event_id = db_event.id + if os_event is None or os_event.deleted: + os_event = _create_feed_event(db, db_feed, user, label) + db_feed.event_uuid = str(os_event.uuid) db.commit() db.refresh(db_feed) else: - db_event = events_repository.create_event( + os_event = events_repository.create_event( db, event_schemas.EventCreate( info="%s: %s - %s" % (label, db_feed.name, datetime.now().isoformat()), @@ -762,7 +756,7 @@ def get_or_create_feed_event( ), ) - return db_event + return os_event def _create_feed_event(db, db_feed, user, label: str): diff --git a/api/app/repositories/notifications.py b/api/app/repositories/notifications.py index da58089d..eab98953 100644 --- a/api/app/repositories/notifications.py +++ b/api/app/repositories/notifications.py @@ -5,7 +5,6 @@ from app.models import notification as notification_models from app.models import organisation as organisation_models from app.models import object as object_models -from app.models import event as event_models from app.models import attribute as attribute_models from app.repositories import user_settings as user_settings_repository from sqlalchemy import select, update, text @@ -186,7 +185,7 @@ def build_event_notification( ) -def create_event_notifications(db: Session, type: str, event: event_models.Event): +def create_event_notifications(db: Session, type: str, event): """Create notifications for users following the organisation or event.""" # Get event organisation @@ -288,16 +287,16 @@ def build_attribute_notification( def create_attribute_notifications( - db: Session, type: str, attribute: attribute_models.Attribute + db: Session, type: str, attribute ): """Create notifications for users following event of the attribute.""" - # get event - event = ( - db.query(event_models.Event) - .filter(event_models.Event.id == attribute.event_id) - .first() - ) + event = None + event_uuid_val = getattr(attribute, "event_uuid", None) + if event_uuid_val: + from app.repositories import events as events_repository_local + from uuid import UUID as _UUID + event = events_repository_local.get_event_from_opensearch(_UUID(str(event_uuid_val))) if not event: return [] @@ -368,15 +367,15 @@ def build_object_notification( ) -def create_object_notifications(db: Session, type: str, object: object_models.Object): +def create_object_notifications(db: Session, type: str, object): """Create notifications for users following event of the object.""" - # get event - event = ( - db.query(event_models.Event) - .filter(event_models.Event.id == object.event_id) - .first() - ) + event = None + event_uuid_val = getattr(object, "event_uuid", None) + if event_uuid_val: + from app.repositories import events as events_repository_local + from uuid import UUID as _UUID + event = events_repository_local.get_event_from_opensearch(_UUID(str(event_uuid_val))) if not event: return [] diff --git a/api/app/repositories/object_references.py b/api/app/repositories/object_references.py index 917a4b55..6a8703de 100644 --- a/api/app/repositories/object_references.py +++ b/api/app/repositories/object_references.py @@ -1,4 +1,5 @@ import time +from uuid import UUID from app.models import object_reference as object_reference_models from app.schemas import object_reference as object_reference_schemas @@ -13,7 +14,7 @@ def create_object_reference( db_object_reference = object_reference_models.ObjectReference( uuid=object_reference.uuid, object_id=object_reference.object_id, - event_id=object_reference.event_id, + event_uuid=object_reference.event_uuid, source_uuid=object_reference.source_uuid, referenced_uuid=object_reference.referenced_uuid, timestamp=object_reference.timestamp or time.time(), @@ -32,11 +33,11 @@ def create_object_reference( def create_object_reference_from_pulled_object_reference( - db: Session, pulled_object_reference: MISPObjectReference, local_event_id: int + db: Session, pulled_object_reference: MISPObjectReference, event_uuid: UUID ): db_object_refence = object_reference_models.ObjectReference( uuid=pulled_object_reference.uuid, - event_id=local_event_id, + event_uuid=event_uuid, source_uuid=pulled_object_reference.object_uuid, referenced_uuid=pulled_object_reference.referenced_uuid, timestamp=pulled_object_reference.timestamp, @@ -59,9 +60,9 @@ def update_object_reference_from_pulled_object_reference( db: Session, db_object_reference: object_reference_models.ObjectReference, pulled_object_reference: MISPObjectReference, - local_event_id: int, + event_uuid: UUID, ): - db_object_reference.event_id = local_event_id + db_object_reference.event_uuid = event_uuid db_object_reference.source_uuid = pulled_object_reference.object_uuid db_object_reference.referenced_uuid = pulled_object_reference.referenced_uuid db_object_reference.timestamp = pulled_object_reference.timestamp diff --git a/api/app/repositories/objects.py b/api/app/repositories/objects.py index c1aed9ef..6aa3544c 100644 --- a/api/app/repositories/objects.py +++ b/api/app/repositories/objects.py @@ -5,7 +5,6 @@ from uuid import UUID, uuid4 from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import feed as feed_models from app.models import object as object_models from app.models import user as user_models @@ -74,13 +73,20 @@ def get_objects( query = db.query(object_models.Object) if event_uuid is not None: - db_event = events_repository.get_event_by_uuid(event_uuid=event_uuid, db=db) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - query = query.filter(object_models.Object.event_id == db_event.id) + os_client = get_opensearch_client() + _resp = os_client.search( + index="misp-objects", + body={ + "query": {"term": {"event_uuid.keyword": str(event_uuid)}}, + "size": 10000, + "_source": ["uuid"], + }, + ) + object_uuids = [h["_source"]["uuid"] for h in _resp["hits"]["hits"]] + if not object_uuids: + from fastapi_pagination import Page as _Page + return _Page(items=[], total=0, page=1, size=50, pages=0) + query = query.filter(object_models.Object.uuid.in_(object_uuids)) if template_uuid is not None: query = query.filter(object_models.Object.template_uuid.in_(template_uuid)) @@ -217,58 +223,68 @@ def get_object_by_uuid(db: Session, object_uuid: UUID): def create_object( db: Session, object: object_schemas.ObjectCreate -) -> object_models.Object: - # TODO: MispObject::beforeValidate() && MispObject::$validate - db_object = object_models.Object( - event_id=object.event_id, - name=object.name, - meta_category=object.meta_category, - description=object.description, - template_uuid=object.template_uuid, - template_version=object.template_version, - uuid=object.uuid, - timestamp=object.timestamp or time.time(), - distribution=( - event_schemas.DistributionLevel(object.distribution) - if object.distribution is not None - else event_schemas.DistributionLevel.INHERIT_EVENT - ), - sharing_group_id=object.sharing_group_id, - comment=object.comment, - deleted=object.deleted, - first_seen=object.first_seen, - last_seen=object.last_seen, - ) +) -> object_schemas.Object: + from datetime import datetime as _datetime - db.add(db_object) - db.commit() - db.refresh(db_object) + client = get_opensearch_client() + object_uuid = str(object.uuid or uuid4()) + + event_uuid = str(object.event_uuid) if object.event_uuid else None + + dist = object.distribution + dist_val = dist.value if hasattr(dist, "value") else (dist if dist is not None else 5) + + obj_doc = { + "uuid": object_uuid, + "event_uuid": event_uuid, + "name": object.name, + "meta_category": object.meta_category, + "description": object.description, + "template_uuid": object.template_uuid, + "template_version": object.template_version, + "timestamp": object.timestamp, + "distribution": dist_val, + "sharing_group_id": object.sharing_group_id, + "comment": object.comment or "", + "deleted": object.deleted or False, + "first_seen": object.first_seen, + "last_seen": object.last_seen, + "object_references": [], + "@timestamp": _datetime.fromtimestamp(object.timestamp).isoformat(), + } - for attribute in object.attributes: - attribute.object_id = db_object.id - attribute.event_id = object.event_id - attributes_repository.create_attribute(db, attribute) + client.index(index="misp-objects", id=object_uuid, body=obj_doc, refresh=True) + + built_attrs = [] + for attr in (object.attributes or []): + attr.object_id = None + attr.event_uuid = event_uuid + attr_schema = attributes_repository.create_attribute(db, attr) + client.update( + index="misp-attributes", + id=str(attr_schema.uuid), + body={"doc": {"object_uuid": object_uuid}}, + refresh=True, + ) + built_attrs.append(attr_schema) - for object_reference in object.object_references: - object_reference.object_id = db_object.id - object_reference.event_id = db_object.event_id + for object_reference in (object.object_references or []): + object_reference.event_uuid = event_uuid object_references_repository.create_object_reference(db, object_reference) - db.refresh(db_object) + tasks.handle_created_object(object_uuid, event_uuid) - tasks.handle_created_object(db_object.id, db_object.event_id) - - return db_object + obj_doc["attributes"] = built_attrs + return object_schemas.Object.model_validate(obj_doc) def create_object_from_pulled_object( - db: Session, pulled_object: MISPObject, local_event_id: int, user: user_models.User + db: Session, pulled_object: MISPObject, event_uuid: str, user: user_models.User ) -> object_models.Object: # TODO: process sharing group // captureSG # TODO: enforce warninglist db_object = object_models.Object( - event_id=local_event_id, name=pulled_object.name, meta_category=pulled_object["meta-category"], description=pulled_object.description, @@ -295,14 +311,14 @@ def create_object_from_pulled_object( for pulled_attribute in pulled_object.attributes: local_object_attribute = ( attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_attribute, local_event_id, user + db, pulled_attribute, event_uuid, user ) ) db_object.attributes.append(local_object_attribute) for pulled_object_reference in pulled_object.ObjectReference: local_object_reference = object_references_repository.create_object_reference_from_pulled_object_reference( - db, pulled_object_reference, local_event_id + db, pulled_object_reference, event_uuid ) db_object.object_references.append(local_object_reference) @@ -315,7 +331,7 @@ def update_object_from_pulled_object( db: Session, local_object: object_models.Object, pulled_object: MISPObject, - local_event_id: int, + event_uuid: str, user: user_models.User, ): @@ -341,17 +357,16 @@ def update_object_from_pulled_object( if local_attribute is None: local_attribute = ( attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_object_attribute, local_event_id, user + db, pulled_object_attribute, event_uuid, user ) ) else: pulled_object_attribute.id = local_attribute.id attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_object_attribute, local_event_id, user + db, local_attribute, pulled_object_attribute, event_uuid, user ) object_patch = object_schemas.ObjectUpdate( - event_id=local_event_id, name=pulled_object.name, meta_category=pulled_object["meta-category"], description=pulled_object.description, @@ -384,7 +399,7 @@ def update_object_from_pulled_object( if local_object_reference is None: local_object_reference = object_references_repository.create_object_reference_from_pulled_object_reference( - db, pulled_object_reference, local_event_id + db, pulled_object_reference, event_uuid ) local_object.object_references.append(local_object_reference) else: @@ -397,103 +412,101 @@ def update_object_from_pulled_object( db, local_object_reference, pulled_object_reference, - local_event_id, + event_uuid, ) update_object(db, local_object.id, object_patch) def update_object( - db: Session, object_id: int, object: object_schemas.ObjectUpdate -) -> object_models.Object: - db_object = get_object_by_id(db, object_id=object_id) + db: Session, object_id: Union[int, UUID], object: object_schemas.ObjectUpdate +) -> object_schemas.Object: + client = get_opensearch_client() + os_obj = get_object_from_opensearch(object_id) + if os_obj is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Object not found") - if db_object is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" + patch = object.model_dump( + exclude_unset=True, + exclude={"attributes", "new_attributes", "update_attributes", "delete_attributes"}, + ) + for k, v in list(patch.items()): + if hasattr(v, "value"): + patch[k] = v.value + + if patch: + client.update(index="misp-objects", id=str(os_obj.uuid), body={"doc": patch}, refresh=True) + + for attr in (object.new_attributes or []): + attr.object_id = None + attr.event_uuid = os_obj.event_uuid + attr_schema = attributes_repository.create_attribute(db, attr) + client.update( + index="misp-attributes", + id=str(attr_schema.uuid), + body={"doc": {"object_uuid": str(os_obj.uuid)}}, + refresh=True, ) - object_patch = object.model_dump(exclude_unset=True) - for key, value in object_patch.items(): - if key == "attributes": - continue - setattr(db_object, key, value) + for attr in (object.update_attributes or []): + attributes_repository.update_attribute(db, attr.uuid, attr) - # new attribute - for attribute in object.new_attributes: - attribute.object_id = db_object.id - attribute.event_id = db_object.event_id - attribute.uuid = str(uuid4()) - attributes_repository.create_attribute(db, attribute) + for attr_id in (object.delete_attributes or []): + attributes_repository.delete_attribute(db, attr_id) - # existing attribute - for attribute in object.update_attributes: - attributes_repository.update_attribute(db, attribute.id, attribute) + tasks.handle_updated_object(str(os_obj.uuid), str(os_obj.event_uuid) if os_obj.event_uuid else None) - # delete attribute - for attribute_id in object.delete_attributes: - attributes_repository.delete_attribute(db, attribute_id) + return get_object_from_opensearch(os_obj.uuid) - db.add(db_object) - db.commit() - db.refresh(db_object) - - tasks.handle_updated_object(db_object.id, db_object.event_id) - - return db_object - - -def delete_object(db: Session, object_id: Union[int, UUID]) -> object_models.Object: - - if isinstance(object_id, int): - db_object = get_object_by_id(db, object_id=object_id) - else: - db_object = get_object_by_uuid(db, object_uuid=object_id) - if db_object is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" +def delete_object(db: Session, object_id: Union[int, UUID]) -> None: + client = get_opensearch_client() + os_obj = get_object_from_opensearch(object_id) + if os_obj is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Object not found") + + client.update(index="misp-objects", id=str(os_obj.uuid), body={"doc": {"deleted": True}}, refresh=True) + + for attr in os_obj.attributes: + client.update( + index="misp-attributes", + id=str(attr.uuid), + body={"doc": {"deleted": True}}, + refresh=True, ) - db_object.deleted = True - - # delete attributes - for attribute in db_object.attributes: - attributes_repository.delete_attribute(db, attribute.id) - - db.add(db_object) - db.commit() - db.refresh(db_object) - - tasks.handle_deleted_object(db_object.id, db_object.event_id) - - return db_object + tasks.handle_deleted_object(str(os_obj.uuid), str(os_obj.event_uuid) if os_obj.event_uuid else None) def create_objects_from_fetched_event( db: Session, - local_event: event_models.Event, + local_event: event_schemas.Event, objects: list[MISPObject], feed: feed_models.Feed, user: user_schemas.User, ): for object in objects: - create_object_from_pulled_object(db, object, local_event.id, user) + create_object_from_pulled_object(db, object, str(local_event.uuid), user) def update_objects_from_fetched_event( db: Session, - local_event: event_models.Event, + local_event: event_schemas.Event, event: event_schemas.Event, feed: feed_models.Feed, user: user_schemas.User, -) -> event_models.Event: - local_event_objects = ( - db.query(object_models.Object.uuid, object_models.Object.timestamp) - .filter(object_models.Object.event_id == local_event.id) - .all() +) -> event_schemas.Event: + from app.services.opensearch import get_opensearch_client as _get_os_client + _os = _get_os_client() + _resp = _os.search( + index="misp-objects", + body={"query": {"term": {"event_uuid.keyword": str(local_event.uuid)}}, "size": 10000}, ) + local_event_objects = [ + (h["_source"]["uuid"], h["_source"].get("timestamp", 0)) + for h in _resp["hits"]["hits"] + ] local_event_dict = {str(uuid): timestamp for uuid, timestamp in local_event_objects} new_objects = [ @@ -555,7 +568,6 @@ def update_objects_from_fetched_event( for attribute in updated_object.attributes: attribute.object_id = db_object.id - attribute.event_id = local_event.id # process attributes local_event = attributes_repository.update_attributes_from_fetched_event( @@ -588,7 +600,7 @@ def update_objects_from_fetched_event( db_object_reference = object_reference_models.ObjectReference( uuid=reference.uuid, - event_id=local_event.id, + event_uuid=str(local_event.uuid), object_id=db_object.id, referenced_uuid=referenced.uuid, referenced_id=referenced.id if referenced else None, diff --git a/api/app/repositories/reports.py b/api/app/repositories/reports.py index 0b7e5077..81f416dd 100644 --- a/api/app/repositories/reports.py +++ b/api/app/repositories/reports.py @@ -1,5 +1,5 @@ from app.services.opensearch import get_opensearch_client -from app.models import event as event_models +from app.schemas import event as event_schemas import logging import uuid import time @@ -25,7 +25,7 @@ def get_event_reports_by_event_uuid(event_uuid: uuid.UUID): return response["hits"]["hits"] -def create_event_report(event: event_models.Event, report: dict): +def create_event_report(event: event_schemas.Event, report: dict): OpenSearchClient = get_opensearch_client() report_uuid = str(uuid.uuid4()) diff --git a/api/app/repositories/servers.py b/api/app/repositories/servers.py index 1db9efc4..a5d802a7 100644 --- a/api/app/repositories/servers.py +++ b/api/app/repositories/servers.py @@ -5,8 +5,8 @@ from app.models import server as server_models from app.models import user as user_models -from app.models import event as event_models from app.models.event import DistributionLevel +from app.schemas import event as event_schemas from app.repositories import sync as sync_repository from app.repositories import events as events_repository from app.repositories import sharing_groups as sharing_groups_repository @@ -175,7 +175,7 @@ def pull_event_by_uuid( server: server_schemas.Server, user: user_models.User, settings: Settings, -) -> Union[event_models.Event, bool]: +) -> Union[event_schemas.Event, bool]: """ see: app/Model/Server.php::__pullEvent() """ @@ -236,8 +236,6 @@ def pull_event_by_uuid( # TODO: process tag collection, see app/Model/Event.php::_add() - tasks.index_event.delay(str(db_event.uuid), full_reindex=True) # TODO: optimize to index only changes - return db_event @@ -394,10 +392,10 @@ def create_or_update_pulled_event( db, created.uuid, event.event_reports, user ) sync_repository.create_pulled_event_objects( - db, created.id, event.objects, user + db, str(created.uuid), event.objects, user ) sync_repository.create_pulled_event_attributes( - db, created.id, event.attributes, user + db, str(created.uuid), event.attributes, user ) logger.info(f"Event {event.uuid} created") @@ -441,10 +439,10 @@ def create_or_update_pulled_event( db, updated.uuid, event.event_reports, user ) sync_repository.update_pulled_event_objects( - db, updated.id, event.objects, user + db, str(updated.uuid), event.objects, user ) sync_repository.update_pulled_event_attributes( - db, updated.id, event.attributes, user + db, str(updated.uuid), event.attributes, user ) # TODO: publish event update to ZMQ @@ -853,13 +851,28 @@ def push_server_by_id_full( user: user_models.User, ): - # get a list of the event_ids eligible to be pushed to the server - event_uuids = db.query(event_models.Event.uuid).filter( - event_models.Event.published == True, - event_models.Event.distribution == DistributionLevel.CONNECTED_COMMUNITIES - or event_models.Event.distribution == DistributionLevel.ALL_COMMUNITIES, - ).all() - event_uuids = [event_uuid for (event_uuid,) in event_uuids] + # get a list of the event UUIDs eligible to be pushed to the server + from app.services.opensearch import get_opensearch_client as _get_os_client + _os = _get_os_client() + _resp = _os.search( + index="misp-events", + body={ + "query": { + "bool": { + "must": [ + {"term": {"published": True}}, + {"terms": {"distribution": [ + DistributionLevel.CONNECTED_COMMUNITIES.value, + DistributionLevel.ALL_COMMUNITIES.value, + ]}}, + ] + } + }, + "_source": ["uuid"], + "size": 10000, + }, + ) + event_uuids = [h["_source"]["uuid"] for h in _resp["hits"]["hits"]] # push each of the events in different tasks for event_uuid in event_uuids: diff --git a/api/app/repositories/sync.py b/api/app/repositories/sync.py index 711838bf..b14094e4 100644 --- a/api/app/repositories/sync.py +++ b/api/app/repositories/sync.py @@ -3,8 +3,8 @@ from uuid import UUID from datetime import datetime -from app.models import event as event_models from app.models import tag as tag_models +from app.schemas import event as event_schemas from app.models import user as user_models from app.models import attribute as attribute_models from app.models import object as object_models @@ -26,7 +26,7 @@ def create_pulled_tags( db: Session, - event: event_models.Event, + event: event_schemas.Event, pulled_tags: list[MISPTag], user: user_models.User, ) -> list[tag_models.Tag]: @@ -42,7 +42,7 @@ def create_pulled_tags( def create_pulled_event_tags( db: Session, - event: event_models.Event, + event: event_schemas.Event, pulled_tags: list[MISPTag], user: user_models.User, ) -> None: @@ -94,7 +94,7 @@ def create_pulled_event_reports( def create_pulled_event_attributes( db: Session, - local_event_id: int, + event_uuid: str, attributes: list[attribute_models.Attribute], user: user_models.User, ): @@ -106,7 +106,7 @@ def create_pulled_event_attributes( if hash not in hashes_dict: local_attribute = ( attributes_repository.create_attribute_from_pulled_attribute( - db, attribute, local_event_id, user + db, attribute, event_uuid, user ) ) hashes_dict[hash] = True @@ -117,13 +117,13 @@ def create_pulled_event_attributes( def create_pulled_event_objects( db: Session, - local_event_id: int, + event_uuid: str, objects: list[object_models.Object], user: user_models.User, ): for object in objects: objects_repository.create_object_from_pulled_object( - db, object, local_event_id, user + db, object, event_uuid, user ) db.commit() @@ -131,7 +131,7 @@ def create_pulled_event_objects( def update_pulled_event_objects( db: Session, - local_event_id: int, + event_uuid: str, objects: list[MISPObject], user: user_models.User, ) -> None: @@ -140,17 +140,17 @@ def update_pulled_event_objects( if local_object is None: objects_repository.create_object_from_pulled_object( - db, object, local_event_id, user + db, object, event_uuid, user ) else: objects_repository.update_object_from_pulled_object( - db, local_object, object, local_event_id, user + db, local_object, object, event_uuid, user ) def update_pulled_event_attributes( db: Session, - local_event_id: int, + event_uuid: str, attributes: list[MISPAttribute], user: user_models.User, ) -> None: @@ -162,11 +162,11 @@ def update_pulled_event_attributes( if local_attribute is None: local_attribute = ( attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_attribute, local_event_id, user + db, pulled_attribute, event_uuid, user ) ) db.add(local_attribute) else: attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_attribute, local_event_id, user + db, local_attribute, pulled_attribute, event_uuid, user ) diff --git a/api/app/repositories/tags.py b/api/app/repositories/tags.py index fded9c7e..369d9644 100644 --- a/api/app/repositories/tags.py +++ b/api/app/repositories/tags.py @@ -1,10 +1,12 @@ -from app.models import attribute as attribute_models -from app.models import event as event_models +from uuid import UUID + from app.models import tag as tag_models from app.models import user as user_models from app.schemas import tag as tag_schemas +from app.services.opensearch import get_opensearch_client from fastapi import HTTPException, Query, status from fastapi_pagination.ext.sqlalchemy import paginate +from opensearchpy.exceptions import NotFoundError from pymisp import MISPTag from sqlalchemy import func from sqlalchemy.orm import Session @@ -87,112 +89,125 @@ def delete_tag(db: Session, tag_id: int) -> None: db.commit() +def _tag_dict(tag: tag_models.Tag) -> dict: + return { + "id": tag.id, + "name": tag.name, + "colour": tag.colour, + "exportable": tag.exportable, + "hide_tag": tag.hide_tag, + "numerical_value": tag.numerical_value, + "is_galaxy": tag.is_galaxy, + "is_custom_galaxy": tag.is_custom_galaxy, + "local_only": tag.local_only, + } + + def tag_attribute( db: Session, - attribute: attribute_models.Attribute, + attribute, tag: tag_models.Tag, ): - - db_attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute.id, - tag_models.AttributeTag.tag_id == tag.id, - ) - .first() - ) - - if db_attribute_tag is not None: - return db_attribute_tag - - db_attribute_tag = tag_models.AttributeTag( - event_id=attribute.event_id, - attribute_id=attribute.id, - tag_id=tag.id, + client = get_opensearch_client() + attr_uuid = str(getattr(attribute, "uuid", None)) + + try: + doc = client.get(index="misp-attributes", id=attr_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") + + current_tags = doc["_source"].get("tags", []) + if any(t.get("name") == tag.name for t in current_tags): + return current_tags + + current_tags.append(_tag_dict(tag)) + client.update( + index="misp-attributes", + id=attr_uuid, + body={"doc": {"tags": current_tags}}, + refresh=True, ) - - db.add(db_attribute_tag) - db.commit() - db.refresh(db_attribute_tag) - - return db_attribute_tag + return current_tags def untag_attribute( db: Session, - attribute: attribute_models.Attribute, + attribute, tag: tag_models.Tag, ): - db_attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute.id, - tag_models.AttributeTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + attr_uuid = str(getattr(attribute, "uuid", None)) + + try: + doc = client.get(index="misp-attributes", id=attr_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="AttributeTag not found") + + current_tags = doc["_source"].get("tags", []) + new_tags = [t for t in current_tags if t.get("name") != tag.name] + if len(new_tags) == len(current_tags): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="AttributeTag not found") + + client.update( + index="misp-attributes", + id=attr_uuid, + body={"doc": {"tags": new_tags}}, + refresh=True, ) - if db_attribute_tag is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="AttributeTag not found" - ) - - db.delete(db_attribute_tag) - db.commit() - def tag_event( db: Session, - event: event_models.Event, + event, tag: tag_models.Tag, ): - - db_event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event.id, - tag_models.EventTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + event_uuid = str(getattr(event, "uuid", None)) + + try: + doc = client.get(index="misp-events", id=event_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + current_tags = doc["_source"].get("tags", []) + if any(t.get("name") == tag.name for t in current_tags): + return current_tags + + current_tags.append(_tag_dict(tag)) + client.update( + index="misp-events", + id=event_uuid, + body={"doc": {"tags": current_tags}}, + refresh=True, ) - - if db_event_tag is not None: - return db_event_tag - - db_event_tag = tag_models.EventTag( - event_id=event.id, - tag_id=tag.id, - ) - - db.add(db_event_tag) - db.commit() - db.refresh(db_event_tag) - - return db_event_tag + return current_tags def untag_event( db: Session, - event: event_models.Event, + event, tag: tag_models.Tag, ): - db_event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event.id, - tag_models.EventTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + event_uuid = str(getattr(event, "uuid", None)) + + try: + doc = client.get(index="misp-events", id=event_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="EventTag not found") + + current_tags = doc["_source"].get("tags", []) + new_tags = [t for t in current_tags if t.get("name") != tag.name] + if len(new_tags) == len(current_tags): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="EventTag not found") + + client.update( + index="misp-events", + id=event_uuid, + body={"doc": {"tags": new_tags}}, + refresh=True, ) - if db_event_tag is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="EventTag not found" - ) - - db.delete(db_event_tag) - db.commit() - def capture_tag(db: Session, tag: MISPTag, user: user_models.User) -> tag_models.Tag: # see: app/Model/Tag.php::captureTag diff --git a/api/app/routers/attributes.py b/api/app/routers/attributes.py index 60260a17..527521e7 100644 --- a/api/app/routers/attributes.py +++ b/api/app/routers/attributes.py @@ -1,6 +1,4 @@ -from typing import Optional - -from typing import Union +from typing import Optional, Union from uuid import UUID from app.auth.security import get_current_active_user from app.db.session import get_db @@ -80,14 +78,14 @@ async def export_attributes( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid format specified" ) -@router.get("/attributes/{attribute_id}", response_model=attribute_schemas.Attribute) -def get_attribute_by_id( - attribute_id: Union[int, UUID], +@router.get("/attributes/{attribute_uuid}", response_model=attribute_schemas.Attribute) +def get_attribute_by_uuid( + attribute_uuid: UUID, user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:read"] ), ) -> attribute_schemas.Attribute: - os_attribute = attributes_repository.get_attribute_from_opensearch(attribute_id) + os_attribute = attributes_repository.get_attribute_from_opensearch(attribute_uuid) if os_attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" @@ -107,76 +105,66 @@ def create_attribute( get_current_active_user, scopes=["attributes:create"] ), ) -> attribute_schemas.Attribute: - if attribute.event_id is None: - if attribute.event_uuid is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Event ID or UUID must be provided", - ) - event = events_repository.get_event_by_uuid( - db, event_uuid=str(attribute.event_uuid) + if attribute.event_uuid is None and attribute.event_id is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Event ID or UUID must be provided", ) + + if attribute.event_uuid is None: + event = events_repository.get_event_from_opensearch(attribute.event_id) else: - event = events_repository.get_event_by_id(db, event_id=attribute.event_id) + event = events_repository.get_event_from_opensearch(attribute.event_uuid) if event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - attribute.event_id = event.id - db_attribute = attributes_repository.create_attribute(db=db, attribute=attribute) - - return db_attribute + attribute.event_uuid = event.uuid + return attributes_repository.create_attribute(db=db, attribute=attribute) -@router.patch("/attributes/{attribute_id}", response_model=attribute_schemas.Attribute) +@router.patch("/attributes/{attribute_uuid}", response_model=attribute_schemas.Attribute) def update_attribute( - attribute_id: int, + attribute_uuid: UUID, attribute: attribute_schemas.AttributeUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:update"] ), ) -> attribute_schemas.Attribute: - attribute_db = attributes_repository.get_attribute_by_id( - db, attribute_id=attribute_id + return attributes_repository.update_attribute( + db=db, attribute_id=attribute_uuid, attribute=attribute ) - attribute_db = attributes_repository.update_attribute( - db=db, attribute_id=attribute_id, attribute=attribute - ) - event = events_repository.get_event_by_id(db, event_id=attribute_db.event_id) - - return attribute_db - -@router.delete("/attributes/{attribute_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/attributes/{attribute_uuid}", status_code=status.HTTP_204_NO_CONTENT) def delete_attribute( - attribute_id: int, + attribute_uuid: UUID, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:delete"] ), ): - attributes_repository.delete_attribute(db=db, attribute_id=attribute_id) + attributes_repository.delete_attribute(db=db, attribute_id=attribute_uuid) return Response(status_code=status.HTTP_204_NO_CONTENT) @router.post( - "/attributes/{attribute_id}/tag/{tag}", + "/attributes/{attribute_uuid}/tag/{tag}", status_code=status.HTTP_201_CREATED, ) def tag_attribute( - attribute_id: int, + attribute_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:update"] ), ): - attribute = attributes_repository.get_attribute_by_id(db, attribute_id=attribute_id) + attribute = attributes_repository.get_attribute_from_opensearch(attribute_uuid) if attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" @@ -195,18 +183,18 @@ def tag_attribute( @router.delete( - "/attributes/{attribute_id}/tag/{tag}", + "/attributes/{attribute_uuid}/tag/{tag}", status_code=status.HTTP_204_NO_CONTENT, ) def untag_attribute( - attribute_id: int, + attribute_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:update"] ), ): - attribute = attributes_repository.get_attribute_by_id(db, attribute_id=attribute_id) + attribute = attributes_repository.get_attribute_from_opensearch(attribute_uuid) if attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" diff --git a/api/app/routers/events.py b/api/app/routers/events.py index 451083d2..15192bcb 100644 --- a/api/app/routers/events.py +++ b/api/app/routers/events.py @@ -1,8 +1,6 @@ import json -from typing import Optional, Annotated - -from typing import Union +from typing import Annotated, Optional from uuid import UUID from app.auth.security import get_current_active_user from app.db.session import get_db @@ -14,7 +12,6 @@ from app.schemas import user as user_schemas from app.schemas import object as object_schemas from app.schemas import vulnerability as vulnerability_schemas -from app.worker import tasks from fastapi import ( APIRouter, Depends, @@ -92,12 +89,12 @@ async def export_events( ) -@router.get("/events/{event_id}", response_model=event_schemas.Event) -def get_event_by_id( - event_id: Union[int, UUID], +@router.get("/events/{event_uuid}", response_model=event_schemas.Event) +def get_event_by_uuid( + event_uuid: UUID, user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ) -> event_schemas.Event: - os_event = events_repository.get_event_from_opensearch(event_id) + os_event = events_repository.get_event_from_opensearch(event_uuid) if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -115,8 +112,8 @@ def create_event( get_current_active_user, scopes=["events:create"] ), ) -> event_schemas.Event: - db_event = events_repository.get_user_by_info(db, info=event_create_request.info) - if db_event: + existing = events_repository.get_event_by_info(event_create_request.info) + if existing: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="An event with this info already exists", @@ -124,49 +121,46 @@ def create_event( event_create_request.user_id = user.id event_create_request.org_id = user.org_id - db_event = events_repository.create_event(db=db, event=event_create_request) - tasks.index_event(str(db_event.uuid), full_reindex=True) - - return db_event + return events_repository.create_event(db=db, event=event_create_request) -@router.patch("/events/{event_id}", response_model=event_schemas.Event) +@router.patch("/events/{event_uuid}", response_model=event_schemas.Event) def update_event( - event_id: int, + event_uuid: UUID, event: event_schemas.EventUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:update"] ), ) -> event_schemas.Event: - return events_repository.update_event(db=db, event_id=event_id, event=event) + return events_repository.update_event(db=db, event_id=event_uuid, event=event) -@router.delete("/events/{event_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/events/{event_uuid}", status_code=status.HTTP_204_NO_CONTENT) def delete_event( - event_id: Union[int, UUID], + event_uuid: UUID, force: Optional[bool] = Query(False), db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:delete"] ), ): - return events_repository.delete_event(db=db, event_id=event_id, force=force) + return events_repository.delete_event(db=db, event_id=event_uuid, force=force) @router.post( - "/events/{event_id}/tag/{tag}", + "/events/{event_uuid}/tag/{tag}", status_code=status.HTTP_201_CREATED, ) def tag_event( - event_id: int, + event_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:update"] ), ): - event = events_repository.get_event_by_id(db, event_id=event_id) + event = events_repository.get_event_from_opensearch(event_uuid) if event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -179,24 +173,23 @@ def tag_event( ) tags_repository.tag_event(db=db, event=event, tag=tag) - tasks.index_event(str(event.uuid), full_reindex=False) return Response(status_code=status.HTTP_201_CREATED) @router.delete( - "/events/{event_id}/tag/{tag}", + "/events/{event_uuid}/tag/{tag}", status_code=status.HTTP_204_NO_CONTENT, ) def untag_event( - event_id: int, + event_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:update"] ), ): - event = events_repository.get_event_by_id(db, event_id=event_id) + event = events_repository.get_event_from_opensearch(event_uuid) if event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -209,18 +202,17 @@ def untag_event( ) tags_repository.untag_event(db=db, event=event, tag=tag) - tasks.index_event(str(event.uuid), full_reindex=False) return Response(status_code=status.HTTP_204_NO_CONTENT) @router.post( - "/events/{event_id}/upload_attachments/", + "/events/{event_uuid}/upload_attachments/", status_code=status.HTTP_200_OK, response_model=list[object_schemas.Object], ) async def upload_attachments( - event_id: Union[int, UUID], + event_uuid: UUID, attachments: list[UploadFile], attachments_meta: Annotated[str, Form()], db: Session = Depends(get_db), @@ -228,10 +220,7 @@ async def upload_attachments( get_current_active_user, scopes=["events:update"] ), ): - if isinstance(event_id, int): - db_event = events_repository.get_event_by_id(db, event_id=event_id) - else: - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_id) + db_event = events_repository.get_event_from_opensearch(event_uuid) if db_event is None: raise HTTPException( @@ -259,7 +248,7 @@ def get_event_attachments( db: Session = Depends(get_db), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) + db_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) if db_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -277,47 +266,6 @@ def get_event_attachments( return objects -@router.post( - "/events/force-index", - status_code=status.HTTP_201_CREATED, -) -async def force_index( - event_uuid: Optional[UUID] = Query(None, alias="uuid"), - event_id: Optional[int] = Query(None, alias="id"), - db: Session = Depends(get_db), - user: user_schemas.User = Security( - get_current_active_user, scopes=["events:update"] - ), -): - - if event_uuid: - tasks.index_event.delay(str(event_uuid), full_reindex=True) - return JSONResponse( - content={"message": f"Indexing started for event {event_uuid}."}, - status_code=status.HTTP_202_ACCEPTED, - ) - - if event_id: - db_event = events_repository.get_event_by_id(db, event_id=event_id) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - tasks.index_event.delay(str(db_event.uuid), full_reindex=True) - return JSONResponse( - content={"message": f"Indexing started for event {db_event.uuid}."}, - status_code=status.HTTP_202_ACCEPTED, - ) - - uuids = events_repository.get_event_uuids(db) - for uuid in uuids: - tasks.index_event.delay(str(uuid[0]), full_reindex=True) - - return JSONResponse( - content={"message": "Indexing started for all events."}, - status_code=status.HTTP_202_ACCEPTED, - ) - @router.post("/events/{event_uuid}/publish") def publish( @@ -327,13 +275,13 @@ def publish( get_current_active_user, scopes=["events:publish"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - events_repository.publish_event(db, db_event) + events_repository.publish_event(os_event) return JSONResponse( content={"message": f"Event {event_uuid} has been published."}, @@ -349,13 +297,13 @@ def unpublish( get_current_active_user, scopes=["events:publish"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - events_repository.unpublish_event(db, db_event) + events_repository.unpublish_event(os_event) return JSONResponse( content={"message": f"Event {event_uuid} has been unpublished."}, @@ -371,13 +319,13 @@ def toggle_correlation( get_current_active_user, scopes=["events:update"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - events_repository.toggle_event_correlation(db, db_event) + events_repository.toggle_event_correlation(os_event) return JSONResponse( content={ @@ -396,14 +344,14 @@ def import_data( get_current_active_user, scopes=["events:import"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) try: - result = events_repository.import_data(db, event=db_event, data=data) + result = events_repository.import_data(db, event=os_event, data=data) return JSONResponse( content=result, status_code=status.HTTP_202_ACCEPTED, @@ -424,13 +372,13 @@ def get_event_vulnerabilities( db: Session = Depends(get_db), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ) -> list[vulnerability_schemas.Vulnerability]: - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) return events_repository.get_event_vulnerabilities( db, - event_uuid=db_event.uuid, + event_uuid=str(os_event.uuid), ) diff --git a/api/app/routers/objects.py b/api/app/routers/objects.py index bb3115b5..5b04c39a 100644 --- a/api/app/routers/objects.py +++ b/api/app/routers/objects.py @@ -66,10 +66,8 @@ def create_object( ), ): - if object.event_id: - event = events_repository.get_event_by_id(db, event_id=object.event_id) - elif object.event_uuid: - event = events_repository.get_event_by_uuid(db, event_uuid=object.event_uuid) + if object.event_uuid: + event = events_repository.get_event_from_opensearch(object.event_uuid) else: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Event UUID is required" @@ -80,15 +78,13 @@ def create_object( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - object.event_id = event.id - object_db = objects_repository.create_object(db=db, object=object) - - return object_db + object.event_uuid = event.uuid + return objects_repository.create_object(db=db, object=object) @router.patch("/objects/{object_id}", response_model=object_schemas.Object) def update_object( - object_id: int, + object_id: Union[int, UUID], object: object_schemas.ObjectUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( diff --git a/api/app/schemas/attribute.py b/api/app/schemas/attribute.py index e9834e87..e3209161 100644 --- a/api/app/schemas/attribute.py +++ b/api/app/schemas/attribute.py @@ -8,7 +8,6 @@ class AttributeBase(BaseModel): - event_id: Optional[int] = None object_id: Optional[int] = None event_uuid: Optional[UUID] = None object_relation: Optional[str] = None @@ -29,7 +28,6 @@ class AttributeBase(BaseModel): class Attribute(AttributeBase): - id: int tags: list[Tag] = [] correlations: Optional[list[dict]] = None expanded: Optional[dict] = None diff --git a/api/app/schemas/event.py b/api/app/schemas/event.py index b08a21e1..4f28b6c2 100644 --- a/api/app/schemas/event.py +++ b/api/app/schemas/event.py @@ -38,7 +38,6 @@ class EventBase(BaseModel): class Event(EventBase): - id: int attributes: list[Attribute] = [] objects: list[Object] = [] sharing_group: Optional[SharingGroup] = None diff --git a/api/app/schemas/feed.py b/api/app/schemas/feed.py index 883e59c3..52d31ec0 100644 --- a/api/app/schemas/feed.py +++ b/api/app/schemas/feed.py @@ -17,7 +17,7 @@ class FeedBase(BaseModel): source_format: str fixed_event: Optional[bool] = False delta_merge: Optional[bool] = False - event_id: Optional[int] = None + event_uuid: Optional[str] = None publish: Optional[bool] = False override_ids: Optional[bool] = False settings: Optional[dict] = None @@ -55,7 +55,7 @@ class FeedUpdate(FeedBase): source_format: Optional[str] = None fixed_event: Optional[bool] = None delta_merge: Optional[bool] = None - event_id: Optional[int] = None + event_uuid: Optional[str] = None publish: Optional[bool] = None override_ids: Optional[bool] = None settings: Optional[dict] = None diff --git a/api/app/schemas/object.py b/api/app/schemas/object.py index 59e1d173..1dd0bc69 100644 --- a/api/app/schemas/object.py +++ b/api/app/schemas/object.py @@ -13,7 +13,6 @@ class ObjectBase(BaseModel): description: Optional[str] = None template_uuid: Optional[str] = None template_version: int - event_id: Optional[int] = None event_uuid: Optional[UUID] = None uuid: Optional[UUID] = None timestamp: int @@ -27,7 +26,6 @@ class ObjectBase(BaseModel): class Object(ObjectBase): - id: int attributes: list[Attribute] = [] object_references: list[ObjectReference] = [] model_config = ConfigDict(from_attributes=True) diff --git a/api/app/schemas/object_reference.py b/api/app/schemas/object_reference.py index dc704319..feb3f355 100644 --- a/api/app/schemas/object_reference.py +++ b/api/app/schemas/object_reference.py @@ -8,7 +8,7 @@ class ObjectReferenceBase(BaseModel): uuid: UUID object_id: int - event_id: int + event_uuid: Optional[UUID] = None source_uuid: Optional[UUID] = None referenced_uuid: Optional[UUID] = None timestamp: int diff --git a/api/app/tests/api/test_attributes.py b/api/app/tests/api/test_attributes.py index 8f79b251..13d454b4 100644 --- a/api/app/tests/api/test_attributes.py +++ b/api/app/tests/api/test_attributes.py @@ -1,7 +1,6 @@ import pytest from app.auth import auth from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import tag as tag_models from app.tests.api_tester import ApiTester from fastapi import status @@ -25,7 +24,7 @@ def test_get_attributes( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == attribute_1.id + assert data[0]["uuid"] == str(attribute_1.uuid) assert data[0]["category"] == attribute_1.category assert data[0]["type"] == attribute_1.type assert data[0]["value"] == attribute_1.value @@ -42,12 +41,12 @@ def test_get_attributes_unauthorized( @pytest.mark.parametrize("scopes", [["attributes:create"]]) def test_create_attribute( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/attributes/", json={ - "event_id": event_1.id, + "event_uuid": str(event_1.uuid), "category": "Network activity", "type": "ip-dst", "value": "127.0.0.1", @@ -57,20 +56,20 @@ def test_create_attribute( data = response.json() assert response.status_code == status.HTTP_201_CREATED - assert data["id"] is not None - assert data["event_id"] == event_1.id + assert data["uuid"] is not None + assert data["event_uuid"] == str(event_1.uuid) assert data["category"] == "Network activity" assert data["type"] == "ip-dst" assert data["value"] == "127.0.0.1" @pytest.mark.parametrize("scopes", [["attributes:read"]]) def test_create_attribute_unauthorized( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/attributes/", json={ - "event_id": event_1.id, + "event_uuid": str(event_1.uuid), "category": "Network activity", "type": "ip-dst", "value": "127.0.0.1", @@ -81,13 +80,13 @@ def test_create_attribute_unauthorized( @pytest.mark.parametrize("scopes", [["attributes:create"]]) def test_create_attribute_incomplete( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): # missing value response = client.post( "/attributes/", json={ - "event_id": event_1.id, + "event_uuid": str(event_1.uuid), "category": "Network activity", "type": "ip-dst", }, @@ -103,7 +102,7 @@ def test_update_attribute( auth_token: auth.Token, ): response = client.patch( - f"/attributes/{attribute_1.id}", + f"/attributes/{attribute_1.uuid}", json={ "type": "ip-src", "value": "8.8.8.8", @@ -124,7 +123,7 @@ def test_delete_attribute( auth_token: auth.Token, ): response = client.delete( - f"/attributes/{attribute_1.id}", + f"/attributes/{attribute_1.uuid}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -134,54 +133,44 @@ def test_delete_attribute( def test_tag_attribute( self, client: TestClient, - event_1: event_models.Event, + event_1: object, attribute_1: attribute_models.Attribute, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.post( - f"/attributes/{attribute_1.id}/tag/{tlp_white_tag.name}", + f"/attributes/{attribute_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_201_CREATED - attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute_1.id, - tag_models.AttributeTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - - assert attribute_tag is not None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_attr = os_client.get(index="misp-attributes", id=str(attribute_1.uuid)) + tag_names = [t.get("name") for t in os_attr["_source"].get("tags", [])] + assert tlp_white_tag.name in tag_names @pytest.mark.parametrize("scopes", [["attributes:update"]]) def test_untag_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, attribute_1: attribute_models.Attribute, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.delete( - f"/attributes/{attribute_1.id}/tag/{tlp_white_tag.name}", + f"/attributes/{attribute_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_204_NO_CONTENT - attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute_1.id, - tag_models.AttributeTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - - assert attribute_tag is None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_attr = os_client.get(index="misp-attributes", id=str(attribute_1.uuid)) + tag_names = [t.get("name") for t in os_attr["_source"].get("tags", [])] + assert tlp_white_tag.name not in tag_names diff --git a/api/app/tests/api/test_events.py b/api/app/tests/api/test_events.py index f097e9aa..d95ee756 100644 --- a/api/app/tests/api/test_events.py +++ b/api/app/tests/api/test_events.py @@ -1,7 +1,6 @@ import pytest from app.auth import auth from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import organisation as organisation_models from app.models import tag as tag_models from app.models import user as user_models @@ -17,7 +16,7 @@ def test_get_events( self, client: TestClient, user_1: user_models.User, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.get( @@ -28,7 +27,6 @@ def test_get_events( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == event_1.id assert data[0]["info"] == event_1.info assert data[0]["org_id"] == event_1.org_id assert data[0]["orgc_id"] == event_1.orgc_id @@ -65,7 +63,7 @@ def test_create_event( data = response.json() assert response.status_code == status.HTTP_201_CREATED - assert data["id"] is not None + assert data["uuid"] is not None assert data["info"] == "test create event" assert data["user_id"] == api_tester_user.id assert data["org_id"] == api_tester_user.org_id @@ -103,7 +101,7 @@ def test_create_event_incomplete(self, client: TestClient, auth_token: auth.Toke @pytest.mark.parametrize("scopes", [["events:create"]]) def test_create_event_invalid_exists( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): # event with duplicated info response = client.post( @@ -126,11 +124,11 @@ def test_create_event_invalid_exists( def test_update_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.patch( - f"/events/{event_1.id}", + f"/events/{event_1.uuid}", json={ "info": "updated via API", "published": False, @@ -147,11 +145,11 @@ def test_update_event( def test_delete_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.delete( - f"/events/{event_1.id}", + f"/events/{event_1.uuid}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -161,53 +159,45 @@ def test_delete_event( def test_tag_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.post( - f"/events/{event_1.id}/tag/{tlp_white_tag.name}", + f"/events/{event_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_201_CREATED - event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event_1.id, - tag_models.EventTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - assert event_tag is not None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_event = os_client.get(index="misp-events", id=str(event_1.uuid)) + tag_names = [t.get("name") for t in os_event["_source"].get("tags", [])] + assert tlp_white_tag.name in tag_names @pytest.mark.parametrize("scopes", [["events:update"]]) def test_untag_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.delete( - f"/events/{event_1.id}/tag/{tlp_white_tag.name}", + f"/events/{event_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_204_NO_CONTENT - event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event_1.id, - tag_models.EventTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - assert event_tag is None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_event = os_client.get(index="misp-events", id=str(event_1.uuid)) + tag_names = [t.get("name") for t in os_event["_source"].get("tags", [])] + assert tlp_white_tag.name not in tag_names @pytest.fixture(scope="class") def event_2( @@ -217,19 +207,21 @@ def event_2( user_1: user_models.User, ): """A second event used for force-delete and other destructive tests.""" - event_2 = event_models.Event( + from datetime import datetime + from uuid import UUID + from app.repositories import events as events_repository + from app.schemas import event as event_schemas + + event_create = event_schemas.EventCreate( info="test event 2 for force delete", user_id=user_1.id, orgc_id=1, org_id=organisation_1.id, - date="2020-01-01", - uuid="d8a2b0c1-aaaa-bbbb-cccc-ef1234567890", + date=datetime(2020, 1, 1), + uuid=UUID("d8a2b0c1-aaaa-bbbb-cccc-ef1234567890"), timestamp=1577836800, ) - db.add(event_2) - db.commit() - db.refresh(event_2) - yield event_2 + yield events_repository.create_event(db=db, event=event_create) @pytest.fixture(scope="class") def event_for_filter( @@ -239,46 +231,29 @@ def event_for_filter( user_1: user_models.User, ): """A stable event used for filter/search tests that won't be mutated.""" - from app.worker.tasks import index_event as _index_event + from datetime import datetime + from uuid import UUID + from app.repositories import events as events_repository + from app.schemas import event as event_schemas - ev = event_models.Event( + event_create = event_schemas.EventCreate( info="stable filter test event", user_id=user_1.id, orgc_id=1, org_id=organisation_1.id, - date="2020-01-01", - uuid="f1f7e2a3-0000-0000-0000-000000000001", + date=datetime(2020, 1, 1), + uuid=UUID("f1f7e2a3-0000-0000-0000-000000000001"), timestamp=1577836800, ) - db.add(ev) - db.commit() - db.refresh(ev) - _index_event(str(ev.uuid), full_reindex=False) - yield ev + yield events_repository.create_event(db=db, event=event_create) - # ---- GET /events/{event_id} ---- - - @pytest.mark.parametrize("scopes", [["events:read"]]) - def test_get_event_by_id( - self, - client: TestClient, - event_1: event_models.Event, - auth_token: auth.Token, - ): - response = client.get( - f"/events/{event_1.id}", - headers={"Authorization": "Bearer " + auth_token}, - ) - data = response.json() - - assert response.status_code == status.HTTP_200_OK - assert data["id"] == event_1.id + # ---- GET /events/{event_uuid} ---- @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_event_by_uuid( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.get( @@ -288,7 +263,7 @@ def test_get_event_by_uuid( data = response.json() assert response.status_code == status.HTTP_200_OK - assert data["id"] == event_1.id + assert data["uuid"] == str(event_1.uuid) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_event_not_found( @@ -297,7 +272,7 @@ def test_get_event_not_found( auth_token: auth.Token, ): response = client.get( - "/events/999999", + "/events/00000000-0000-0000-0000-000000000000", headers={"Authorization": "Bearer " + auth_token}, ) @@ -310,7 +285,7 @@ def test_get_event_not_found( def test_get_events_filter_by_info( self, client: TestClient, - event_for_filter: event_models.Event, + event_for_filter: object, auth_token: auth.Token, ): response = client.get( @@ -322,7 +297,7 @@ def test_get_events_filter_by_info( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == event_for_filter.id + assert data[0]["uuid"] == str(event_for_filter.uuid) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_events_filter_by_info_no_results( @@ -344,7 +319,7 @@ def test_get_events_filter_by_info_no_results( def test_get_events_filter_by_uuid( self, client: TestClient, - event_for_filter: event_models.Event, + event_for_filter: object, auth_token: auth.Token, ): response = client.get( @@ -356,13 +331,13 @@ def test_get_events_filter_by_uuid( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == event_for_filter.id + assert data[0]["uuid"] == str(event_for_filter.uuid) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_events_filter_by_deleted_true( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): # event_1 was soft-deleted by test_delete_event @@ -374,14 +349,14 @@ def test_get_events_filter_by_deleted_true( data = response.json()["items"] assert response.status_code == status.HTTP_200_OK - assert any(item["id"] == event_1.id for item in data) + assert any(item["uuid"] == str(event_1.uuid) for item in data) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_events_filter_by_deleted_false( self, client: TestClient, - event_1: event_models.Event, - event_for_filter: event_models.Event, + event_1: object, + event_for_filter: object, auth_token: auth.Token, ): # event_1 was soft-deleted; event_for_filter was not @@ -393,9 +368,9 @@ def test_get_events_filter_by_deleted_false( data = response.json()["items"] assert response.status_code == status.HTTP_200_OK - ids = [item["id"] for item in data] - assert event_1.id not in ids - assert event_for_filter.id in ids + uuids = [item["uuid"] for item in data] + assert str(event_1.uuid) not in uuids + assert str(event_for_filter.uuid) in uuids # ---- DELETE /events/{event_id} force ---- @@ -403,25 +378,25 @@ def test_get_events_filter_by_deleted_false( def test_delete_event_force( self, client: TestClient, - event_2: event_models.Event, + event_2: object, db: Session, auth_token: auth.Token, ): response = client.delete( - f"/events/{event_2.id}", + f"/events/{event_2.uuid}", params={"force": True}, headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_204_NO_CONTENT - db.expire_all() - deleted = ( - db.query(event_models.Event) - .filter(event_models.Event.id == event_2.id) - .first() + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + response_os = os_client.search( + index="misp-events", + body={"query": {"term": {"uuid.keyword": str(event_2.uuid)}}}, ) - assert deleted is None + assert response_os["hits"]["total"]["value"] == 0 # ---- Tag/untag 404 cases ---- @@ -433,7 +408,7 @@ def test_tag_event_event_not_found( auth_token: auth.Token, ): response = client.post( - f"/events/999999/tag/{tlp_white_tag.name}", + f"/events/00000000-0000-0000-0000-000000000000/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -444,11 +419,11 @@ def test_tag_event_event_not_found( def test_tag_event_tag_not_found( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( - f"/events/{event_1.id}/tag/nonexistent:tag", + f"/events/{event_1.uuid}/tag/nonexistent:tag", headers={"Authorization": "Bearer " + auth_token}, ) @@ -463,7 +438,7 @@ def test_untag_event_event_not_found( auth_token: auth.Token, ): response = client.delete( - f"/events/999999/tag/{tlp_white_tag.name}", + f"/events/00000000-0000-0000-0000-000000000000/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -474,11 +449,11 @@ def test_untag_event_event_not_found( def test_untag_event_tag_not_found( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.delete( - f"/events/{event_1.id}/tag/nonexistent:tag", + f"/events/{event_1.uuid}/tag/nonexistent:tag", headers={"Authorization": "Bearer " + auth_token}, ) @@ -491,7 +466,7 @@ def test_untag_event_tag_not_found( def test_publish_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): # event_1 has published=False after test_update_event @@ -524,7 +499,7 @@ def test_publish_event_not_found( def test_unpublish_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): # event_1 was published by test_publish_event @@ -557,7 +532,7 @@ def test_unpublish_event_not_found( def test_toggle_correlation( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -589,7 +564,7 @@ def test_toggle_correlation_not_found( def test_import_data( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -639,7 +614,7 @@ def test_import_data_not_found( def test_force_index_by_uuid( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -656,7 +631,7 @@ def test_force_index_by_uuid( def test_force_index_by_id( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -688,7 +663,7 @@ def test_force_index_by_id_not_found( def test_force_index_all_events( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -706,7 +681,7 @@ def test_force_index_all_events( def test_get_event_attachments( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.get( diff --git a/api/app/tests/api/test_feeds.py b/api/app/tests/api/test_feeds.py index f2c08023..9f267c2f 100644 --- a/api/app/tests/api/test_feeds.py +++ b/api/app/tests/api/test_feeds.py @@ -40,7 +40,7 @@ def test_get_feeds( assert data[0]["default"] == feed_1.default assert data[0]["fixed_event"] == feed_1.fixed_event assert data[0]["delta_merge"] == feed_1.delta_merge - assert data[0]["event_id"] == feed_1.event_id + assert data[0]["event_uuid"] == feed_1.event_uuid assert data[0]["publish"] == feed_1.publish assert data[0]["override_ids"] == feed_1.override_ids assert data[0]["settings"] == feed_1.settings diff --git a/api/app/tests/api/test_indexing_parity.py b/api/app/tests/api/test_indexing_parity.py deleted file mode 100644 index e343fcda..00000000 --- a/api/app/tests/api/test_indexing_parity.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Phase 0 regression harness: verifies that index_event produces OpenSearch -documents that are fully consistent with the PostgreSQL source of truth. - -Each test creates SQL fixtures, calls index_event synchronously (full_reindex), -then fetches the indexed documents and asserts field-level parity. - -Run with: - docker compose exec api poetry run pytest tests/api/test_indexing_parity.py -v -""" -import pytest -from app.models import attribute as attribute_models -from app.models import event as event_models -from app.models import object as object_models -from app.models import object_reference as object_reference_models -from app.models import tag as tag_models -from app.tests.api_tester import ApiTester -from app.worker.tasks import index_attribute, index_event, index_object -from sqlalchemy.orm import Session - - -def _os_client(): - from app.services.opensearch import get_opensearch_client - - return get_opensearch_client() - - -class TestIndexingParity(ApiTester): - """Assert SQL ↔ OpenSearch parity for events, attributes, and objects.""" - - # ── cleanup ──────────────────────────────────────────────────────────────── - - @pytest.fixture(scope="class", autouse=True) - def cleanup(self, db: Session): - # Runs at class setup time (same pattern as ApiTester.cleanup) so that - # leftover data from a previous test class is wiped before our fixtures - # try to insert rows with the same hardcoded UUIDs / info strings. - try: - pass - finally: - os = _os_client() - for index in ("misp-events", "misp-attributes", "misp-objects"): - try: - os.delete_by_query( - index=index, - body={"query": {"match_all": {}}}, - refresh=True, - ignore=[404], - ) - except Exception: - pass - self.teardown_db(db) - - # ── event parity ────────────────────────────────────────────────────────── - - def test_event_fields_in_opensearch( - self, - db: Session, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, - object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, - tlp_white_tag: tag_models.Tag, - ): - """All scalar event fields must appear in the OS document with correct values.""" - # tag the event so we can assert tag parity too - event_tag = tag_models.EventTag( - event_id=event_1.id, tag_id=tlp_white_tag.id, local=False - ) - db.add(event_tag) - db.commit() - - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] - - assert doc["id"] == event_1.id - assert doc["uuid"] == str(event_1.uuid) - assert doc["info"] == event_1.info - assert doc["org_id"] == event_1.org_id - assert doc["orgc_id"] == event_1.orgc_id - assert doc["user_id"] == event_1.user_id - assert doc["published"] == event_1.published - assert doc["analysis"] == event_1.analysis.value - assert doc["distribution"] == event_1.distribution.value - assert doc["threat_level"] == event_1.threat_level.value - assert doc["timestamp"] == event_1.timestamp - assert doc["publish_timestamp"] == event_1.publish_timestamp - assert doc["deleted"] == event_1.deleted - assert doc["locked"] == event_1.locked - assert doc["protected"] == event_1.protected - assert doc["disable_correlation"] == event_1.disable_correlation - assert doc["proposal_email_lock"] == event_1.proposal_email_lock - assert doc.get("sighting_timestamp") == event_1.sighting_timestamp - assert doc.get("sharing_group_id") == event_1.sharing_group_id - expected_extends = ( - str(event_1.extends_uuid) if event_1.extends_uuid else None - ) - assert doc.get("extends_uuid") == expected_extends - - def test_event_tags_in_opensearch( - self, - db: Session, - event_1: event_models.Event, - tlp_white_tag: tag_models.Tag, - ): - """Tags attached to an event must appear in the misp-events document.""" - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] - - tag_names = [t["name"] for t in doc.get("tags", [])] - assert tlp_white_tag.name in tag_names - - def test_event_organisation_in_opensearch( - self, - db: Session, - event_1: event_models.Event, - ): - """The organisation nested object must be present in the misp-events document.""" - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] - - assert "organisation" in doc - assert doc["organisation"]["id"] == event_1.org_id - - # ── attribute parity ────────────────────────────────────────────────────── - - def test_standalone_attribute_fields_in_opensearch( - self, - db: Session, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, - ): - """Standalone (non-object) attribute fields must be consistent in misp-attributes.""" - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-attributes", id=str(attribute_1.uuid))["_source"] - - assert doc["id"] == attribute_1.id - assert doc["uuid"] == str(attribute_1.uuid) - assert doc["event_id"] == attribute_1.event_id - assert doc["event_uuid"] == str(event_1.uuid) - assert doc["type"] == attribute_1.type - assert doc["value"] == attribute_1.value - assert doc["category"] == attribute_1.category - assert doc["to_ids"] == attribute_1.to_ids - assert doc["deleted"] == attribute_1.deleted - assert doc["disable_correlation"] == attribute_1.disable_correlation - assert doc["timestamp"] == attribute_1.timestamp - assert doc["distribution"] == attribute_1.distribution.value - assert doc.get("sharing_group_id") == attribute_1.sharing_group_id - assert doc.get("comment") == attribute_1.comment - assert doc.get("first_seen") == attribute_1.first_seen - assert doc.get("last_seen") == attribute_1.last_seen - # standalone attributes must not carry an object_uuid - assert doc.get("object_uuid") is None - - def test_object_attribute_fields_in_opensearch( - self, - db: Session, - event_1: event_models.Event, - object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, - ): - """Object attributes must be in misp-attributes (not embedded in the object doc) - and must carry both event_uuid and object_uuid.""" - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - - # must exist as a separate document in misp-attributes - attr_doc = os.get( - index="misp-attributes", id=str(object_attribute_1.uuid) - )["_source"] - - assert attr_doc["event_uuid"] == str(event_1.uuid) - assert attr_doc["object_uuid"] == str(object_1.uuid) - assert attr_doc["value"] == object_attribute_1.value - - def test_standalone_attribute_object_uuid_via_index_attribute( - self, - db: Session, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, - ): - """index_attribute must not set object_uuid for standalone attributes.""" - index_attribute(str(attribute_1.uuid)) - - os = _os_client() - doc = os.get(index="misp-attributes", id=str(attribute_1.uuid))["_source"] - - assert doc.get("object_uuid") is None - assert doc["event_uuid"] == str(event_1.uuid) - - def test_object_attribute_object_uuid_via_index_attribute( - self, - db: Session, - event_1: event_models.Event, - object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, - ): - """index_attribute must populate object_uuid for attributes that belong to an object.""" - index_attribute(str(object_attribute_1.uuid)) - - os = _os_client() - doc = os.get( - index="misp-attributes", id=str(object_attribute_1.uuid) - )["_source"] - - assert doc["object_uuid"] == str(object_1.uuid) - assert doc["event_uuid"] == str(event_1.uuid) - - # ── object parity ───────────────────────────────────────────────────────── - - def test_object_fields_in_opensearch( - self, - db: Session, - event_1: event_models.Event, - object_1: object_models.Object, - ): - """Object scalar fields must be consistent between SQL and misp-objects.""" - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-objects", id=str(object_1.uuid))["_source"] - - assert doc["id"] == object_1.id - assert doc["uuid"] == str(object_1.uuid) - assert doc["event_uuid"] == str(event_1.uuid) - assert doc["name"] == object_1.name - assert doc["deleted"] == object_1.deleted - assert doc["timestamp"] == object_1.timestamp - assert doc["distribution"] == object_1.distribution.value - assert doc.get("sharing_group_id") == object_1.sharing_group_id - assert doc.get("meta_category") == object_1.meta_category - assert doc.get("template_uuid") == object_1.template_uuid - assert doc.get("template_version") == object_1.template_version - assert doc.get("first_seen") == object_1.first_seen - assert doc.get("last_seen") == object_1.last_seen - - def test_object_doc_does_not_contain_attributes( - self, - db: Session, - event_1: event_models.Event, - object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, - ): - """Attributes must NOT be embedded inside the misp-objects document; they - live as top-level documents in misp-attributes.""" - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-objects", id=str(object_1.uuid))["_source"] - - assert "attributes" not in doc or doc["attributes"] == [] - - def test_index_object_task_does_not_embed_attributes( - self, - db: Session, - event_1: event_models.Event, - object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, - ): - """index_object (the standalone task) must also not embed attributes.""" - index_object(str(object_1.uuid)) - - os = _os_client() - doc = os.get(index="misp-objects", id=str(object_1.uuid))["_source"] - - assert "attributes" not in doc or doc["attributes"] == [] - - # and the attribute must exist separately - attr_doc = os.get( - index="misp-attributes", id=str(object_attribute_1.uuid) - )["_source"] - assert attr_doc["object_uuid"] == str(object_1.uuid) - - # ── attribute count parity ──────────────────────────────────────────────── - - def test_attribute_count_in_event_doc( - self, - db: Session, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, - object_attribute_1: attribute_models.Attribute, - ): - """attribute_count in the misp-events document must match the SQL value.""" - db.refresh(event_1) - index_event(str(event_1.uuid), full_reindex=True) - - os = _os_client() - doc = os.get(index="misp-events", id=str(event_1.uuid))["_source"] - - assert doc["attribute_count"] == event_1.attribute_count diff --git a/api/app/tests/api/test_objects.py b/api/app/tests/api/test_objects.py index 765d671d..7eeaf9d0 100644 --- a/api/app/tests/api/test_objects.py +++ b/api/app/tests/api/test_objects.py @@ -1,6 +1,5 @@ import pytest from app.auth import auth -from app.models import event as event_models from app.models import object as object_models from app.tests.api_tester import ApiTester from fastapi import status @@ -23,7 +22,7 @@ def test_get_objects( assert response.status_code == status.HTTP_200_OK assert len(data['items']) == 1 - assert data['items'][0]["id"] == object_1.id + assert data['items'][0]["uuid"] == str(object_1.uuid) assert data['items'][0]["name"] == object_1.name assert data['items'][0]["template_version"] == object_1.template_version assert data['items'][0]["timestamp"] == object_1.timestamp @@ -39,7 +38,7 @@ def test_get_objects_unauthorized(self, client: TestClient, auth_token: auth.Tok @pytest.mark.parametrize("scopes", [["objects:create"]]) def test_create_object( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/objects/", @@ -55,15 +54,15 @@ def test_create_object( data = response.json() assert response.status_code == status.HTTP_201_CREATED - assert data["id"] is not None - assert data["event_id"] == event_1.id + assert data["uuid"] is not None + assert data["event_uuid"] == str(event_1.uuid) assert data["template_version"] == 0 assert data["timestamp"] == 1655283899 assert data["deleted"] is False @pytest.mark.parametrize("scopes", [["objects:read"]]) def test_create_object_unauthorized( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/objects/", @@ -80,7 +79,7 @@ def test_create_object_unauthorized( @pytest.mark.parametrize("scopes", [["objects:create"]]) def test_create_object_incomplete( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): # missing value response = client.post( @@ -100,7 +99,7 @@ def test_update_object( auth_token: auth.Token, ): response = client.patch( - f"/objects/{object_1.id}", + f"/objects/{object_1.uuid}", json={ "name": "updated via API", "comment": "test comment", @@ -121,7 +120,7 @@ def test_delete_object( auth_token: auth.Token, ): response = client.delete( - f"/objects/{object_1.id}", + f"/objects/{object_1.uuid}", headers={"Authorization": "Bearer " + auth_token}, ) diff --git a/api/app/tests/api_tester.py b/api/app/tests/api_tester.py index a19e0be1..85a25fc7 100644 --- a/api/app/tests/api_tester.py +++ b/api/app/tests/api_tester.py @@ -75,7 +75,6 @@ def teardown_db(self, db: Session): db.query(attribute_models.Attribute).delete(synchronize_session=False) db.query(object_reference_models.ObjectReference).delete(synchronize_session=False) db.query(object_models.Object).delete(synchronize_session=False) - db.query(event_models.Event).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupOrganisation).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupServer).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroup).delete(synchronize_session=False) @@ -173,30 +172,30 @@ def event_1( organisation_1: organisation_models.Organisation, user_1: user_models.User, ): - from app.worker.tasks import index_event as _index_event + from datetime import datetime + from uuid import UUID + from app.repositories import events as events_repository + from app.schemas import event as event_schemas - event_1 = event_models.Event( + event_create = event_schemas.EventCreate( info="test event", user_id=user_1.id, orgc_id=1, org_id=organisation_1.id, - date="2020-01-01", - uuid="ba4b11b6-dcce-4315-8fd0-67b69160ea76", + date=datetime(2020, 1, 1), + uuid=UUID("ba4b11b6-dcce-4315-8fd0-67b69160ea76"), timestamp=1577836800, ) - db.add(event_1) - db.commit() - db.refresh(event_1) - _index_event(str(event_1.uuid), full_reindex=False) + event_1 = events_repository.create_event(db=db, event=event_create) yield event_1 @pytest.fixture(scope="class") - def attribute_1(self, db: Session, event_1: event_models.Event): - from app.worker.tasks import index_attribute as _index_attribute + def attribute_1(self, db: Session, event_1): + from datetime import datetime + from app.services.opensearch import get_opensearch_client attribute_1 = attribute_models.Attribute( - event_id=event_1.id, category="Network activity", type="ip-src", value="127.0.0.1", @@ -206,16 +205,41 @@ def attribute_1(self, db: Session, event_1: event_models.Event): db.add(attribute_1) db.commit() db.refresh(attribute_1) - _index_attribute(str(attribute_1.uuid)) + + client = get_opensearch_client() + client.index( + index="misp-attributes", + id=str(attribute_1.uuid), + body={ + "uuid": str(attribute_1.uuid), + "event_uuid": str(event_1.uuid), + "category": attribute_1.category, + "type": attribute_1.type, + "value": attribute_1.value, + "timestamp": attribute_1.timestamp, + "@timestamp": datetime.fromtimestamp(attribute_1.timestamp).isoformat(), + "deleted": attribute_1.deleted or False, + "to_ids": attribute_1.to_ids or False, + "disable_correlation": attribute_1.disable_correlation or False, + "distribution": attribute_1.distribution.value if attribute_1.distribution else 0, + "sharing_group_id": attribute_1.sharing_group_id, + "comment": attribute_1.comment, + "first_seen": attribute_1.first_seen, + "last_seen": attribute_1.last_seen, + "data": "", + "tags": [], + }, + refresh=True, + ) yield attribute_1 @pytest.fixture(scope="class") - def object_1(self, db: Session, event_1: event_models.Event): - from app.worker.tasks import index_object as _index_object + def object_1(self, db: Session, event_1): + from datetime import datetime + from app.services.opensearch import get_opensearch_client object_1 = object_models.Object( - event_id=event_1.id, uuid="90e06ef6-26f8-40dd-9fb7-75897445e2a0", name="test object", template_version=0, @@ -225,16 +249,40 @@ def object_1(self, db: Session, event_1: event_models.Event): db.add(object_1) db.commit() db.refresh(object_1) - _index_object(str(object_1.uuid)) + + client = get_opensearch_client() + client.index( + index="misp-objects", + id=str(object_1.uuid), + body={ + "uuid": str(object_1.uuid), + "event_uuid": str(event_1.uuid), + "name": object_1.name, + "meta_category": object_1.meta_category, + "template_uuid": object_1.template_uuid, + "template_version": object_1.template_version, + "timestamp": object_1.timestamp, + "@timestamp": datetime.fromtimestamp(object_1.timestamp).isoformat(), + "deleted": object_1.deleted, + "distribution": object_1.distribution.value if object_1.distribution else 0, + "sharing_group_id": object_1.sharing_group_id, + "first_seen": object_1.first_seen, + "last_seen": object_1.last_seen, + "object_references": [], + }, + refresh=True, + ) yield object_1 @pytest.fixture(scope="class") def object_attribute_1( - self, db: Session, event_1: event_models.Event, object_1: object_models.Object + self, db: Session, event_1, object_1: object_models.Object ): + from datetime import datetime + from app.services.opensearch import get_opensearch_client + object_attribute_1 = attribute_models.Attribute( - event_id=event_1.id, object_id=object_1.id, category="Network activity", type="ip-src", @@ -246,6 +294,26 @@ def object_attribute_1( db.commit() db.refresh(object_attribute_1) + client = get_opensearch_client() + client.index( + index="misp-attributes", + id=str(object_attribute_1.uuid), + body={ + "uuid": str(object_attribute_1.uuid), + "event_uuid": str(event_1.uuid), + "object_uuid": str(object_1.uuid), + "category": object_attribute_1.category, + "type": object_attribute_1.type, + "value": object_attribute_1.value, + "timestamp": object_attribute_1.timestamp, + "@timestamp": datetime.fromtimestamp(object_attribute_1.timestamp).isoformat(), + "deleted": False, + "data": "", + "tags": [], + }, + refresh=True, + ) + yield object_attribute_1 @pytest.fixture(scope="class") @@ -390,7 +458,7 @@ def feed_1(self, db: Session, organisation_1: organisation_models.Organisation): source_format="misp", fixed_event=False, delta_merge=False, - event_id=None, + event_uuid=None, publish=False, override_ids=False, settings=None, diff --git a/api/app/worker/tasks.py b/api/app/worker/tasks.py index aa204deb..1f5dc600 100644 --- a/api/app/worker/tasks.py +++ b/api/app/worker/tasks.py @@ -3,6 +3,7 @@ import smtplib from email.message import EmailMessage from datetime import datetime +from uuid import UUID from app.database import SQLALCHEMY_DATABASE_URL from app.services.opensearch import get_opensearch_client @@ -23,7 +24,6 @@ from app.repositories import taxonomies as taxonomies_repository from app.schemas import event as event_schemas from app.schemas import attribute as attribute_schemas -from app.models import event as event_models from celery import Celery from sqlalchemy import create_engine from sqlalchemy.orm import Session @@ -142,14 +142,10 @@ def push_event_by_uuid(event_uuid: str, server_id: int, user_id: int): def handle_created_event(event_uuid: str): logger.info("handling created event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - notifications_repository.create_event_notifications( - db, "created", event=db_event - ) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "created", event=os_event) return True @@ -158,21 +154,16 @@ def handle_created_event(event_uuid: str): def handle_updated_event(event_uuid: str): logger.info("handling updated event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - db_event.timestamp = datetime.now().timestamp() - db.commit() - db.refresh(db_event) - - notifications_repository.create_event_notifications( - db, "updated", event=db_event + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + get_opensearch_client().update( + index="misp-events", + id=event_uuid, + body={"doc": {"timestamp": int(datetime.now().timestamp())}}, + refresh=True, ) - - index_event(str(db_event.uuid), full_reindex=False) + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "updated", event=os_event) return True @@ -181,113 +172,95 @@ def handle_updated_event(event_uuid: str): def handle_deleted_event(event_uuid: str): logger.info("handling deleted event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "deleted", event=os_event) - notifications_repository.create_event_notifications( - db, "deleted", event=db_event - ) - - delete_indexed_event(str(event_uuid)) + delete_indexed_event(event_uuid) return True @celery_app.task -def handle_created_attribute(attribute_id: int, object_id: int | None, event_id: int): - logger.info("handling created attribute id=%s job started", attribute_id) +def handle_created_attribute(attribute_uuid: str, object_id, event_uuid: str | None): + logger.info("handling created attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - if object_id is None: - events_repository.increment_attribute_count(db, event_id) - - db_attribute = attributes_repository.get_attribute_by_id(db, attribute_id) - notifications_repository.create_attribute_notifications( - db, "created", attribute=db_attribute - ) + if object_id is None and event_uuid: + events_repository.increment_attribute_count(db, event_uuid) - index_attribute(str(db_attribute.uuid)) + os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) + if os_attr is not None: + notifications_repository.create_attribute_notifications(db, "created", attribute=os_attr) return True @celery_app.task -def handle_updated_attribute(attribute_id: int, object_id: int | None, event_id: int): - logger.info("handling updated attribute id=%s job started", attribute_id) +def handle_updated_attribute(attribute_uuid: str, object_id, event_uuid: str | None): + logger.info("handling updated attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - db_attribute = attributes_repository.get_attribute_by_id(db, attribute_id) - notifications_repository.create_attribute_notifications( - db, "updated", attribute=db_attribute - ) - - index_attribute(str(db_attribute.uuid)) + os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) + if os_attr is not None: + notifications_repository.create_attribute_notifications(db, "updated", attribute=os_attr) return True @celery_app.task -def handle_deleted_attribute(attribute_id: int, object_id: int | None, event_id: int): - logger.info("handling deleted attribute id=%s job started", attribute_id) +def handle_deleted_attribute(attribute_uuid: str, object_id, event_uuid: str | None): + logger.info("handling deleted attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - db_attribute = attributes_repository.get_attribute_by_id(db, attribute_id) - if object_id is None: - events_repository.decrement_attribute_count(db, event_id) - - notifications_repository.create_attribute_notifications( - db, "deleted", attribute=db_attribute - ) + if object_id is None and event_uuid: + events_repository.decrement_attribute_count(db, event_uuid) - delete_indexed_attribute(str(db_attribute.uuid)) + os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) + if os_attr is not None: + notifications_repository.create_attribute_notifications(db, "deleted", attribute=os_attr) return True @celery_app.task -def handle_created_object(object_id: int, event_id: int): - logger.info("handling created object id=%s job started", object_id) +def handle_created_object(object_uuid: str, event_uuid: str | None): + logger.info("handling created object uuid=%s job started", object_uuid) with Session(engine) as db: - events_repository.increment_object_count(db, event_id) - - db_object = objects_repository.get_object_by_id(db, object_id) - notifications_repository.create_object_notifications( - db, "created", object=db_object - ) + if event_uuid: + events_repository.increment_object_count(db, event_uuid) - index_object(str(db_object.uuid)) + os_obj = objects_repository.get_object_from_opensearch(UUID(object_uuid)) + if os_obj is not None: + notifications_repository.create_object_notifications(db, "created", object=os_obj) return True @celery_app.task -def handle_updated_object(object_id: int, event_id: int): - logger.info("handling updated object id=%s job started", object_id) +def handle_updated_object(object_uuid: str, event_uuid: str | None): + logger.info("handling updated object uuid=%s job started", object_uuid) with Session(engine) as db: - db_object = objects_repository.get_object_by_id(db, object_id) - notifications_repository.create_object_notifications( - db, "updated", object=db_object - ) - - index_object(str(db_object.uuid)) + os_obj = objects_repository.get_object_from_opensearch(UUID(object_uuid)) + if os_obj is not None: + notifications_repository.create_object_notifications(db, "updated", object=os_obj) return True @celery_app.task -def handle_deleted_object(object_id: int, event_id: int): - logger.info("handling deleted object id=%s job started", object_id) +def handle_deleted_object(object_uuid: str, event_uuid: str | None): + logger.info("handling deleted object uuid=%s job started", object_uuid) with Session(engine) as db: - events_repository.decrement_object_count(db, event_id) + if event_uuid: + events_repository.decrement_object_count(db, event_uuid) - db_object = objects_repository.get_object_by_id(db, object_id) - notifications_repository.create_object_notifications( - db, "deleted", object=db_object - ) + os_obj = objects_repository.get_object_from_opensearch(UUID(object_uuid)) + if os_obj is not None: + notifications_repository.create_object_notifications(db, "deleted", object=os_obj) - delete_indexed_object(str(db_object.uuid)) + delete_indexed_object(object_uuid) return True @@ -322,163 +295,6 @@ def send_email(email: dict): return True -@celery_app.task -def index_event(event_uuid: str, full_reindex: bool = False): - logger.info("index event uuid=%s job started", event_uuid) - - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - event = event_schemas.Event.model_validate(db_event) - - OpenSearchClient = get_opensearch_client() - - event_raw = event.model_dump() - attributes = event_raw.pop("attributes") - objects = event_raw.pop("objects") - - # convert timestamp to datetime so it can be indexed - event_raw["@timestamp"] = datetime.fromtimestamp(event_raw["timestamp"]).isoformat() - - if event_raw["publish_timestamp"]: - event_raw["@publish_timestamp"] = datetime.fromtimestamp( - event_raw["publish_timestamp"] - ).isoformat() - - response = OpenSearchClient.index( - index="misp-events", id=event.uuid, body=event_raw, refresh=True - ) - - if response["result"] not in ["created", "updated"]: - logger.error( - "Failed to index event uuid=%s. Response: %s", event_uuid, response - ) - raise Exception("Failed to index event.") - - logger.info("indexed event uuid=%s", event_uuid) - - if not full_reindex: - return True - - # delete existing indexed attributes and objects - query = {"query": {"bool": {"must": [{"term": {"event_uuid": str(event.uuid)}}]}}} - - response = OpenSearchClient.delete_by_query( - index="misp-attributes", body=query, refresh=True - ) - logger.info( - "deleted %s previously indexed attributes for event uuid=%s", - response["deleted"], - event_uuid, - ) - - response = OpenSearchClient.delete_by_query( - index="misp-objects", body=query, refresh=True - ) - logger.info( - "deleted %s previously indexed objects for event uuid=%s", - response["deleted"], - event_uuid, - ) - - # index attributes - attributes_docs = [] - for attribute in attributes: - attribute["@timestamp"] = datetime.fromtimestamp( - attribute["timestamp"] - ).isoformat() - attribute["event_uuid"] = event.uuid - attribute["data"] = "" # do not index file contents - - attributes_docs.append( - { - "_id": attribute["uuid"], - "_index": "misp-attributes", - "_source": attribute, - } - ) - - success, failed = opensearch_helpers.bulk( - OpenSearchClient, attributes_docs, refresh=True - ) - if failed: - logger.error("Failed to index attributes. Failed: %s", failed) - raise Exception("Failed to index attributes.") - logger.info( - "indexed attributes of event uuid=%s, indexed %s attributes", - event_uuid, - len(attributes_docs), - ) - - # index objects - objects_docs = [] - for object in objects: - object["@timestamp"] = datetime.fromtimestamp(object["timestamp"]).isoformat() - - object_attributes = object.pop("attributes") - - object_attributes_docs = [] - for attribute in object_attributes: - attribute["@timestamp"] = datetime.fromtimestamp( - attribute["timestamp"] - ).isoformat() - attribute["event_uuid"] = event.uuid - attribute["object_uuid"] = object["uuid"] - attribute["data"] = "" # do not index file contents - object_attributes_docs.append( - { - "_id": attribute["uuid"], - "_index": "misp-attributes", - "_source": attribute, - } - ) - - success, failed = opensearch_helpers.bulk( - OpenSearchClient, object_attributes_docs, refresh=True - ) - if failed: - logger.error("Failed to index object attributes. Failed: %s", failed) - raise Exception("Failed to index object attributes.") - logger.info( - "indexed attributes of event uuid=%s, object uuid=%s, indexed %s attributes", - event_uuid, - object["uuid"], - len(object_attributes_docs), - ) - - for reference in object["object_references"]: - reference["@timestamp"] = datetime.fromtimestamp( - reference["timestamp"] - ).isoformat() - - object["event_uuid"] = event.uuid - - objects_docs.append( - { - "_id": object["uuid"], - "_index": "misp-objects", - "_source": object, - } - ) - - success, failed = opensearch_helpers.bulk( - OpenSearchClient, objects_docs, refresh=True - ) - if failed: - logger.error("Failed to index objects. Failed: %s", failed) - raise Exception("Failed to index objects.") - logger.info( - "indexed objects of event uuid=%s, indexed %s objects", - event_uuid, - len(objects_docs), - ) - - logger.info("index event uuid=%s job finished", event_uuid) - - return True - @celery_app.task def fetch_feed(feed_id: int, user_id: int): logger.info("fetch feed id=%s job started", feed_id) @@ -537,7 +353,7 @@ def fetch_csv_feed(feed_id: int, user_id: int): attribute = feeds_repository.process_csv_feed_row(row, db_feed.settings) db_attribute = attribute_schemas.AttributeCreate( - event_id=db_event.id, + event_uuid=db_event.uuid, type=attribute["type"], value=attribute["value"], category=attribute.get("category", "External analysis"), @@ -561,8 +377,6 @@ def fetch_csv_feed(feed_id: int, user_id: int): index += 1 - index_event.delay(str(db_event.uuid), full_reindex=True) - logger.info("fetch csv feed id=%s job finished", feed_id) return { @@ -608,7 +422,7 @@ def fetch_freetext_feed(feed_id: int, user_id: int): attr_type = freetext_repository.detect_type(value) db_attribute = attribute_schemas.AttributeCreate( - event_id=db_event.id, + event_uuid=db_event.uuid, type=attr_type, value=value, category="External analysis", @@ -620,8 +434,6 @@ def fetch_freetext_feed(feed_id: int, user_id: int): failed_rows += 1 logger.error("Error processing freetext feed line: %s", e) - index_event.delay(str(db_event.uuid), full_reindex=True) - logger.info("fetch freetext feed id=%s job finished", feed_id) return { @@ -663,7 +475,7 @@ def fetch_json_feed(feed_id: int, user_id: int): continue db_attribute = attribute_schemas.AttributeCreate( - event_id=db_event.id, + event_uuid=db_event.uuid, type=attribute["type"], value=attribute["value"], category=attribute.get("category", "External analysis"), @@ -682,8 +494,6 @@ def fetch_json_feed(feed_id: int, user_id: int): failed_items += 1 logger.error("Error processing JSON feed item: %s", e) - index_event.delay(str(db_event.uuid), full_reindex=True) - logger.info("fetch json feed id=%s job finished", feed_id) return { @@ -809,17 +619,12 @@ def handle_created_correlation( def handle_published_event(event_uuid: str): logger.info("handling published event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - notifications_repository.create_event_notifications( - db, "published", event=db_event - ) - - logger.info("handling published event uuid=%s job finished", event_uuid) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "published", event=os_event) + logger.info("handling published event uuid=%s job finished", event_uuid) return True @@ -827,17 +632,12 @@ def handle_published_event(event_uuid: str): def handle_unpublished_event(event_uuid: str): logger.info("handling unpublished event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - notifications_repository.create_event_notifications( - db, "unpublished", event=db_event - ) - - logger.info("handling unpublished event uuid=%s job finished", event_uuid) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "unpublished", event=os_event) + logger.info("handling unpublished event uuid=%s job finished", event_uuid) return True @@ -845,27 +645,20 @@ def handle_unpublished_event(event_uuid: str): def handle_toggled_event_correlation(event_uuid: str, disable_correlation: bool): logger.info("handling toggled event correlation uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - if disable_correlation: - correlations_repository.delete_event_correlations(event_uuid) - else: - with Session(engine) as db: - runtimeSettings = get_runtime_settings(db) - - correlations_repository.correlate_event( - runtimeSettings, str(event_uuid) - ) - - run_correlation_hunts.delay() + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is None: + logger.warning("handle_toggled_event_correlation: event %s not found", event_uuid) + return True - logger.info( - "handling toggled event correlation uuid=%s job finished", event_uuid - ) + if disable_correlation: + correlations_repository.delete_event_correlations(event_uuid) + else: + with Session(engine) as db: + runtimeSettings = get_runtime_settings(db) + correlations_repository.correlate_event(runtimeSettings, str(event_uuid)) + run_correlation_hunts.delay() + logger.info("handling toggled event correlation uuid=%s job finished", event_uuid) return True @@ -915,29 +708,33 @@ def delete_indexed_event(event_uuid: str): def index_attribute(attribute_uuid: str): logger.info("indexing attribute uuid=%s job started", attribute_uuid) + OpenSearchClient = get_opensearch_client() + + # Preserve event_uuid and tags from existing indexed doc (event_id FK removed from SQL) + existing_event_uuid = None + existing_tags = [] + try: + existing_doc = OpenSearchClient.get(index="misp-attributes", id=attribute_uuid) + existing_event_uuid = existing_doc["_source"].get("event_uuid") + existing_tags = existing_doc["_source"].get("tags", []) + except Exception: + pass + with Session(engine) as db: db_attribute = attributes_repository.get_attribute_by_uuid(db, attribute_uuid) if db_attribute is None: raise Exception("Attribute with uuid=%s not found", attribute_uuid) - if db_attribute.event and db_attribute.event.deleted: - logger.info( - "skipping indexing attribute uuid=%s, event is deleted", attribute_uuid - ) - return True - attribute = event_schemas.Attribute.model_validate(db_attribute) - event_uuid = db_attribute.event.uuid if db_attribute.event else None object_uuid = None if db_attribute.object_id: db_object = objects_repository.get_object_by_id(db, db_attribute.object_id) object_uuid = db_object.uuid if db_object else None - OpenSearchClient = get_opensearch_client() - attribute_raw = attribute.model_dump() - attribute_raw["event_uuid"] = str(event_uuid) if event_uuid else None + attribute_raw["event_uuid"] = existing_event_uuid attribute_raw["object_uuid"] = str(object_uuid) if object_uuid else None + attribute_raw["tags"] = existing_tags # convert timestamp to datetime so it can be indexed attribute_raw["@timestamp"] = datetime.fromtimestamp( @@ -1036,25 +833,32 @@ def load_taxonomies(): def index_object(object_uuid: str): logger.info("indexing object uuid=%s job started", object_uuid) + OpenSearchClient = get_opensearch_client() + + # Preserve event_uuid from existing indexed doc (event_id FK was removed from SQL) + existing_event_uuid = None + try: + existing_doc = OpenSearchClient.get(index="misp-objects", id=object_uuid) + existing_event_uuid = existing_doc["_source"].get("event_uuid") + except Exception: + pass + with Session(engine) as db: db_object = objects_repository.get_object_by_uuid(db, object_uuid) if db_object is None: raise Exception("Object with uuid=%s not found", object_uuid) - event_uuid = str(db_object.event.uuid) object = event_schemas.Object.model_validate(db_object) - OpenSearchClient = get_opensearch_client() - object_raw = object.model_dump() # convert timestamp to datetime so it can be indexed object_raw["@timestamp"] = datetime.fromtimestamp( object_raw["timestamp"] ).isoformat() - object_raw["event_uuid"] = event_uuid + object_raw["event_uuid"] = existing_event_uuid - # pop attributes and index them separately in misp-attributes (consistent with index_event) + # pop attributes and index them separately in misp-attributes object_attributes = object_raw.pop("attributes", []) response = OpenSearchClient.index( @@ -1078,7 +882,7 @@ def index_object(object_uuid: str): attribute["@timestamp"] = datetime.fromtimestamp( attribute["timestamp"] ).isoformat() - attribute["event_uuid"] = event_uuid + attribute["event_uuid"] = existing_event_uuid attribute["object_uuid"] = str(object_raw["uuid"]) attribute["data"] = "" attribute_docs.append( diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d90799ec..6807ca0e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -32,6 +32,7 @@ services: retries: 5 api: + mem_limit: 4g build: context: ./api dockerfile: Dockerfile diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 91c8b7b8..592ffaad 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,8 +1,7 @@ services: api: - mem_limit: 2g - memswap_limit: 2g + mem_limit: 4g volumes: - ./api:/code ports: From 72cd74e36173bb80484073a0c1d4eb8febe61a87 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 26 Mar 2026 10:48:13 +0100 Subject: [PATCH 06/38] fix: remove usages of event model --- api/app/models/__init__.py | 1 - api/app/tests/repositories/test_feeds.py | 31 +++++++++------------- api/app/tests/repositories/test_servers.py | 20 +++++++------- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/api/app/models/__init__.py b/api/app/models/__init__.py index 517da763..4e611fa7 100644 --- a/api/app/models/__init__.py +++ b/api/app/models/__init__.py @@ -1,5 +1,4 @@ from app.models.attribute import Attribute # noqa -from app.models.event import Event # noqa from app.models.hunt import Hunt # noqa from app.models.module import ModuleSettings # noqa from app.models.object import Object # noqa diff --git a/api/app/tests/repositories/test_feeds.py b/api/app/tests/repositories/test_feeds.py index 65c79655..fa6540ff 100644 --- a/api/app/tests/repositories/test_feeds.py +++ b/api/app/tests/repositories/test_feeds.py @@ -1,12 +1,14 @@ from unittest.mock import MagicMock, patch +from uuid import UUID + from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import feed as feed_models from app.models import object as object_models from app.models import object_reference as object_reference_models from app.models import tag as tag_models from app.models import user as user_models +from app.repositories import events as events_repository from app.repositories import feeds as feeds_repository from app.tests.api_tester import ApiTester from app.tests.scenarios import feed_fetch_scenarios @@ -43,15 +45,10 @@ def test_fetch_feed_by_id_new_event( ) # check that the events were created - events = ( - db.query(event_models.Event) - .filter( - event_models.Event.uuid - == feed_fetch_scenarios.feed_new_event["Event"]["uuid"] - ) - .all() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_new_event["Event"]["uuid"]) ) - assert len(events) == 1 + assert os_event is not None # check that the attributes were created attributes = ( @@ -119,7 +116,7 @@ def test_fetch_feed_by_id_existing_event( self, db: Session, feed_1: feed_models.Feed, - event_1: event_models.Event, + event_1, attribute_1: attribute_models.Attribute, object_1: object_models.Object, object_attribute_1: attribute_models.Attribute, @@ -149,16 +146,12 @@ def test_fetch_feed_by_id_existing_event( ) # check that the events was updated - event = ( - db.query(event_models.Event) - .filter( - event_models.Event.uuid - == feed_fetch_scenarios.feed_update_event["Event"]["uuid"] - ) - .first() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_update_event["Event"]["uuid"]) ) - assert event.info == "Updated by Feed fetch" - assert event.timestamp == 1577836801 + assert os_event is not None + assert os_event.info == "Updated by Feed fetch" + assert os_event.timestamp == 1577836801 # check that the attribute was updated attribute = ( diff --git a/api/app/tests/repositories/test_servers.py b/api/app/tests/repositories/test_servers.py index ae198da7..87ced78b 100644 --- a/api/app/tests/repositories/test_servers.py +++ b/api/app/tests/repositories/test_servers.py @@ -1,8 +1,9 @@ from unittest.mock import MagicMock, patch import pytest +from uuid import UUID + from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import object as object_models from app.models import object_reference as object_reference_models from app.models import organisation as organisations_models @@ -10,6 +11,7 @@ from app.models import sharing_groups as sharing_groups_models from app.models import tag as tag_models from app.models import user as user_models +from app.repositories import events as events_repository from app.repositories import servers as servers_repository from app.settings import Settings from app.tests.api_tester import ApiTester @@ -69,16 +71,12 @@ def test_pull_server_by_id( ) # check that the events were created - events = ( - db.query(event_models.Event) - .filter( - event_models.Event.uuid.in_( - scenario["expected_result"]["event_uuids"] - ) - ) - .all() - ) - assert len(events) == len(scenario["expected_result"]["event_uuids"]) + os_events = [ + events_repository.get_event_from_opensearch(UUID(uuid)) + for uuid in scenario["expected_result"]["event_uuids"] + ] + os_events = [e for e in os_events if e is not None] + assert len(os_events) == len(scenario["expected_result"]["event_uuids"]) # check that the attributes were created attributes = ( From c59232b2a6642e1cf525da32bd0ba6e58570b136 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 26 Mar 2026 14:05:38 +0100 Subject: [PATCH 07/38] chg: [refactor] remove attribute, object db uses --- api/app/models/__init__.py | 1 - api/app/models/attribute.py | 77 ---- api/app/models/object.py | 3 +- api/app/repositories/attributes.py | 101 ++---- api/app/repositories/notifications.py | 1 - api/app/repositories/object_references.py | 115 +++--- api/app/repositories/objects.py | 387 ++++++++------------- api/app/repositories/sync.py | 17 +- api/app/schemas/attribute.py | 35 ++ api/app/schemas/object.py | 20 ++ api/app/schemas/object_reference.py | 42 ++- api/app/tests/api/test_attributes.py | 11 +- api/app/tests/api/test_events.py | 1 - api/app/tests/api/test_objects.py | 7 +- api/app/tests/api_tester.py | 139 +++----- api/app/tests/repositories/test_feeds.py | 72 ++-- api/app/tests/repositories/test_servers.py | 27 +- 17 files changed, 418 insertions(+), 638 deletions(-) delete mode 100644 api/app/models/attribute.py diff --git a/api/app/models/__init__.py b/api/app/models/__init__.py index 4e611fa7..c03974b4 100644 --- a/api/app/models/__init__.py +++ b/api/app/models/__init__.py @@ -1,4 +1,3 @@ -from app.models.attribute import Attribute # noqa from app.models.hunt import Hunt # noqa from app.models.module import ModuleSettings # noqa from app.models.object import Object # noqa diff --git a/api/app/models/attribute.py b/api/app/models/attribute.py deleted file mode 100644 index 21572997..00000000 --- a/api/app/models/attribute.py +++ /dev/null @@ -1,77 +0,0 @@ -import uuid - -import logging -from app.database import Base -from app.models.event import DistributionLevel -from app.services.attachments import get_b64_attachment -from sqlalchemy import BigInteger, Boolean, Column, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship -from app.settings import Settings, get_settings - - -logger = logging.getLogger(__name__) - - -class Attribute(Base): - __tablename__ = "attributes" - - id = Column( - Integer, primary_key=True, index=True, autoincrement=True, nullable=False - ) - object_id = Column(Integer, ForeignKey("objects.id")) - object_relation = Column(String(255), index=True) - category = Column(String(255), index=True) - type = Column(String(100), index=True) - value = Column(String()) - to_ids = Column(Boolean, default=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - timestamp = Column(Integer, nullable=False, default=0) - distribution: Mapped[DistributionLevel] = mapped_column( - Enum(DistributionLevel, name="distribution_level"), - nullable=False, - default=DistributionLevel.INHERIT_EVENT, - ) - sharing_group_id = Column( - Integer, ForeignKey("sharing_groups.id"), index=True, nullable=True - ) - comment = Column(String()) - deleted = Column(Boolean, default=False) - disable_correlation = Column(Boolean, default=False) - first_seen = Column(BigInteger(), index=True) - last_seen = Column(BigInteger(), index=True) - def to_misp_format( - self, - settings: Settings = get_settings(), - ): - """Convert the Attribute to a MISP-compatible dictionary representation.""" - - attr_json = { - "id": self.id, - "object_id": self.object_id, - "object_relation": self.object_relation, - "category": self.category, - "type": self.type, - "value": self.value, - "to_ids": self.to_ids, - "uuid": str(self.uuid), - "timestamp": self.timestamp, - "distribution": self.distribution.value if self.distribution else DistributionLevel.INHERIT_EVENT, - "sharing_group_id": self.sharing_group_id if self.sharing_group_id else None, - "comment": self.comment, - "deleted": self.deleted, - "disable_correlation": self.disable_correlation, - "first_seen": self.first_seen, - "last_seen": self.last_seen, - "Tag": [], - } - - # if its a file attribute, we need to handle it differently - if self.type in ["malware-sample", "attachment"]: - try: - attr_json["data"] = get_b64_attachment(self.uuid, settings) - except Exception as e: - logger.error(f"Error storing attachment: {str(e)}") - print(f"Error fetching file from storage: {str(e)}") - - return attr_json diff --git a/api/app/models/object.py b/api/app/models/object.py index 4ee15d2a..d53ff65b 100644 --- a/api/app/models/object.py +++ b/api/app/models/object.py @@ -28,7 +28,6 @@ class Object(Base): deleted = Column(Boolean, nullable=False, default=False) first_seen = Column(Integer) last_seen = Column(Integer) - attributes = relationship("Attribute", lazy="subquery", cascade="all, delete-orphan") object_references = relationship("ObjectReference", lazy="subquery", cascade="all, delete-orphan") def to_misp_format(self): @@ -48,6 +47,6 @@ def to_misp_format(self): "deleted": self.deleted, "first_seen": self.first_seen, "last_seen": self.last_seen, - "Attribute": [attribute.to_misp_format() for attribute in self.attributes], + "Attribute": [], "ObjectReference": [ref.to_misp_format() for ref in self.object_references], } \ No newline at end of file diff --git a/api/app/repositories/attributes.py b/api/app/repositories/attributes.py index 25fd63ba..e20164f5 100644 --- a/api/app/repositories/attributes.py +++ b/api/app/repositories/attributes.py @@ -5,7 +5,6 @@ from uuid import UUID, uuid4 from app.models.event import DistributionLevel from app.services.opensearch import get_opensearch_client -from app.models import attribute as attribute_models from app.models import tag as tag_models from app.models import user as user_models from app.repositories import attachments as attachments_repository @@ -14,10 +13,8 @@ from app.worker import tasks from fastapi import HTTPException, status from fastapi_pagination import Page, Params -from fastapi_pagination.ext.sqlalchemy import paginate from pymisp import MISPAttribute, MISPTag from sqlalchemy.orm import Session -from sqlalchemy.sql import select from collections import defaultdict from opensearchpy.exceptions import NotFoundError @@ -57,28 +54,6 @@ def enrich_attributes_page_with_correlations( return attributes_page -def get_attributes( - db: Session, - event_uuid: str = None, - deleted: bool = None, - object_id: int = None, - type: str = None, -) -> Page[attribute_schemas.Attribute]: - query = select(attribute_models.Attribute) - - if deleted is not None: - query = query.where(attribute_models.Attribute.deleted == deleted) - - if type is not None: - query = query.where(attribute_models.Attribute.type == type) - - query = query.where(attribute_models.Attribute.object_id == object_id) - - page_results = paginate(db, query) - - return enrich_attributes_page_with_correlations(page_results) - - def get_attributes_from_opensearch( params: Params, event_uuid: str = None, @@ -150,22 +125,14 @@ def get_attribute_from_opensearch( def get_attribute_by_id( db: Session, attribute_id: int -) -> Union[attribute_models.Attribute, None]: - return ( - db.query(attribute_models.Attribute) - .filter(attribute_models.Attribute.id == attribute_id) - .first() - ) +) -> Optional[attribute_schemas.Attribute]: + return get_attribute_from_opensearch(attribute_id) def get_attribute_by_uuid( db: Session, attribute_uuid: UUID -) -> Union[attribute_models.Attribute, None]: - return ( - db.query(attribute_models.Attribute) - .filter(attribute_models.Attribute.uuid == attribute_uuid) - .first() - ) +) -> Optional[attribute_schemas.Attribute]: + return get_attribute_from_opensearch(attribute_uuid) def create_attribute( @@ -215,11 +182,18 @@ def create_attribute_from_pulled_attribute( pulled_attribute: MISPAttribute, event_uuid: str, user: user_models.User, -) -> attribute_models.Attribute: +) -> attribute_schemas.Attribute: # TODO: process sharing group // captureSG # TODO: enforce warninglist - local_attribute = attribute_models.Attribute( + dist = pulled_attribute.distribution + dist_val = ( + event_schemas.DistributionLevel(dist) + if dist is not None + else event_schemas.DistributionLevel.INHERIT_EVENT + ) + + attr_create = attribute_schemas.AttributeCreate( category=pulled_attribute.category, type=pulled_attribute.type, value=( @@ -230,11 +204,7 @@ def create_attribute_from_pulled_attribute( to_ids=pulled_attribute.to_ids, uuid=pulled_attribute.uuid, timestamp=pulled_attribute.timestamp.timestamp(), - distribution=event_schemas.DistributionLevel( - event_schemas.DistributionLevel(pulled_attribute.distribution) - if pulled_attribute.distribution is not None - else event_schemas.DistributionLevel.INHERIT_EVENT - ), + distribution=dist_val, comment=pulled_attribute.comment, sharing_group_id=None, deleted=pulled_attribute.deleted, @@ -242,41 +212,39 @@ def create_attribute_from_pulled_attribute( object_relation=getattr(pulled_attribute, "object_relation", None), first_seen=( pulled_attribute.first_seen.timestamp() - if hasattr(pulled_attribute, "first_seen") + if hasattr(pulled_attribute, "first_seen") and pulled_attribute.first_seen else None ), last_seen=( pulled_attribute.last_seen.timestamp() - if hasattr(pulled_attribute, "last_seen") + if hasattr(pulled_attribute, "last_seen") and pulled_attribute.last_seen else None ), + event_uuid=event_uuid, ) + local_attribute = create_attribute(db, attr_create) + if pulled_attribute.data is not None: - # store file attachments_repository.store_attachment( str(pulled_attribute.uuid), pulled_attribute.data.getvalue() ) - # TODO: process sigthings + # TODO: process sightings # TODO: process galaxies - capture_attribute_tags( - db, local_attribute, pulled_attribute.tags, event_uuid, user - ) + capture_attribute_tags(db, pulled_attribute.tags, event_uuid, user) return local_attribute def update_attribute_from_pulled_attribute( db: Session, - local_attribute: attribute_models.Attribute, + local_attribute: attribute_schemas.Attribute, pulled_attribute: MISPAttribute, event_uuid: str, user: user_models.User, -) -> attribute_models.Attribute: - - pulled_attribute.id = local_attribute.id +) -> attribute_schemas.Attribute: if local_attribute.timestamp < pulled_attribute.timestamp.timestamp(): attribute_patch = attribute_schemas.AttributeUpdate( @@ -301,35 +269,34 @@ def update_attribute_from_pulled_attribute( ), first_seen=( pulled_attribute.first_seen.timestamp() - if hasattr(pulled_attribute, "first_seen") + if hasattr(pulled_attribute, "first_seen") and pulled_attribute.first_seen else local_attribute.first_seen ), last_seen=( pulled_attribute.last_seen.timestamp() - if hasattr(pulled_attribute, "last_seen") + if hasattr(pulled_attribute, "last_seen") and pulled_attribute.last_seen else local_attribute.last_seen ), ) - update_attribute(db, local_attribute.id, attribute_patch) + update_attribute(db, local_attribute.uuid, attribute_patch) if pulled_attribute.data is not None: - # store file attachments_repository.store_attachment( str(pulled_attribute.uuid), pulled_attribute.data.getvalue() ) - capture_attribute_tags( - db, local_attribute, pulled_attribute.tags, event_uuid, user - ) + capture_attribute_tags(db, pulled_attribute.tags, event_uuid, user) - # TODO: process sigthings + # TODO: process sightings # TODO: process galaxies tasks.handle_updated_attribute.delay( - str(local_attribute.uuid), local_attribute.object_id, None + str(local_attribute.uuid), + local_attribute.object_id, + str(local_attribute.event_uuid) if local_attribute.event_uuid else None, ) - return local_attribute + return get_attribute_from_opensearch(local_attribute.uuid) def update_attribute( @@ -378,7 +345,6 @@ def delete_attribute(db: Session, attribute_id: Union[int, str, UUID]) -> None: def capture_attribute_tags( db: Session, - db_attribute: attribute_models.Attribute, tags: list[MISPTag], event_uuid: str, user: user_models.User, @@ -421,7 +387,6 @@ def capture_attribute_tags( db_tag = tag_name_to_db_tag[tag.name] db_attribute_tag = tag_models.AttributeTag( - attribute=db_attribute, event_uuid=event_uuid, tag_id=db_tag.id, local=tag.local, @@ -514,4 +479,4 @@ def export_attributes( if format == "json": yield hit - search_after = hits[-1].get("sort") \ No newline at end of file + search_after = hits[-1].get("sort") diff --git a/api/app/repositories/notifications.py b/api/app/repositories/notifications.py index eab98953..838cf156 100644 --- a/api/app/repositories/notifications.py +++ b/api/app/repositories/notifications.py @@ -5,7 +5,6 @@ from app.models import notification as notification_models from app.models import organisation as organisation_models from app.models import object as object_models -from app.models import attribute as attribute_models from app.repositories import user_settings as user_settings_repository from sqlalchemy import select, update, text from sqlalchemy.orm import Session diff --git a/api/app/repositories/object_references.py b/api/app/repositories/object_references.py index 6a8703de..3be9ac1e 100644 --- a/api/app/repositories/object_references.py +++ b/api/app/repositories/object_references.py @@ -1,72 +1,91 @@ import time +from typing import Optional from uuid import UUID -from app.models import object_reference as object_reference_models from app.schemas import object_reference as object_reference_schemas +from app.services.opensearch import get_opensearch_client +from opensearchpy.exceptions import NotFoundError from pymisp import MISPObjectReference from sqlalchemy.orm import Session def create_object_reference( db: Session, object_reference: object_reference_schemas.ObjectReferenceCreate -): - # TODO: ObjectReference::beforeValidate() && ObjectReference::$validate - db_object_reference = object_reference_models.ObjectReference( - uuid=object_reference.uuid, - object_id=object_reference.object_id, - event_uuid=object_reference.event_uuid, - source_uuid=object_reference.source_uuid, - referenced_uuid=object_reference.referenced_uuid, - timestamp=object_reference.timestamp or time.time(), - referenced_id=object_reference.referenced_id, - referenced_type=object_reference.referenced_type, - relationship_type=object_reference.relationship_type, - comment=object_reference.comment, - deleted=object_reference.deleted, - ) +) -> object_reference_schemas.ObjectReference: + client = get_opensearch_client() + ref_uuid = str(object_reference.uuid) + + ref_doc = { + "uuid": ref_uuid, + "object_id": object_reference.object_id, + "event_uuid": str(object_reference.event_uuid) if object_reference.event_uuid else None, + "source_uuid": str(object_reference.source_uuid) if object_reference.source_uuid else None, + "referenced_uuid": str(object_reference.referenced_uuid) if object_reference.referenced_uuid else None, + "timestamp": object_reference.timestamp or int(time.time()), + "referenced_id": object_reference.referenced_id, + "referenced_type": object_reference.referenced_type, + "relationship_type": object_reference.relationship_type, + "comment": object_reference.comment or "", + "deleted": object_reference.deleted or False, + } - db.add(db_object_reference) - db.commit() - db.refresh(db_object_reference) + client.index(index="misp-object-references", id=ref_uuid, body=ref_doc, refresh=True) - return db_object_reference + return object_reference_schemas.ObjectReference.model_validate(ref_doc) def create_object_reference_from_pulled_object_reference( db: Session, pulled_object_reference: MISPObjectReference, event_uuid: UUID -): - db_object_refence = object_reference_models.ObjectReference( - uuid=pulled_object_reference.uuid, - event_uuid=event_uuid, - source_uuid=pulled_object_reference.object_uuid, - referenced_uuid=pulled_object_reference.referenced_uuid, - timestamp=pulled_object_reference.timestamp, - relationship_type=pulled_object_reference.relationship_type, - comment=pulled_object_reference.comment, - ) +) -> object_reference_schemas.ObjectReference: + client = get_opensearch_client() + ref_uuid = str(pulled_object_reference.uuid) + + ref_doc = { + "uuid": ref_uuid, + "event_uuid": str(event_uuid), + "source_uuid": str(pulled_object_reference.object_uuid), + "referenced_uuid": str(pulled_object_reference.referenced_uuid), + "timestamp": pulled_object_reference.timestamp, + "relationship_type": pulled_object_reference.relationship_type, + "comment": pulled_object_reference.comment or "", + "deleted": False, + } + + client.index(index="misp-object-references", id=ref_uuid, body=ref_doc, refresh=True) + + return object_reference_schemas.ObjectReference.model_validate(ref_doc) - return db_object_refence def get_object_reference_by_uuid( - db: Session, object_reference_uuid: int -) -> object_reference_models.ObjectReference: - return ( - db.query(object_reference_models.ObjectReference) - .filter(object_reference_models.ObjectReference.uuid == object_reference_uuid) - .first() - ) + db: Session, object_reference_uuid +) -> Optional[object_reference_schemas.ObjectReference]: + client = get_opensearch_client() + try: + doc = client.get(index="misp-object-references", id=str(object_reference_uuid)) + return object_reference_schemas.ObjectReference.model_validate(doc["_source"]) + except NotFoundError: + return None + def update_object_reference_from_pulled_object_reference( db: Session, - db_object_reference: object_reference_models.ObjectReference, + db_object_reference: object_reference_schemas.ObjectReference, pulled_object_reference: MISPObjectReference, event_uuid: UUID, -): - db_object_reference.event_uuid = event_uuid - db_object_reference.source_uuid = pulled_object_reference.object_uuid - db_object_reference.referenced_uuid = pulled_object_reference.referenced_uuid - db_object_reference.timestamp = pulled_object_reference.timestamp - db_object_reference.relationship_type = pulled_object_reference.relationship_type - db_object_reference.comment = pulled_object_reference.comment - - return db_object_reference \ No newline at end of file +) -> object_reference_schemas.ObjectReference: + client = get_opensearch_client() + patch = { + "event_uuid": str(event_uuid), + "source_uuid": str(pulled_object_reference.object_uuid), + "referenced_uuid": str(pulled_object_reference.referenced_uuid), + "timestamp": pulled_object_reference.timestamp, + "relationship_type": pulled_object_reference.relationship_type, + "comment": pulled_object_reference.comment or "", + } + client.update( + index="misp-object-references", + id=str(db_object_reference.uuid), + body={"doc": patch}, + refresh=True, + ) + return get_object_reference_by_uuid(db, db_object_reference.uuid) diff --git a/api/app/repositories/objects.py b/api/app/repositories/objects.py index 6aa3544c..1862d776 100644 --- a/api/app/repositories/objects.py +++ b/api/app/repositories/objects.py @@ -4,23 +4,20 @@ from typing import Optional, Union from uuid import UUID, uuid4 -from app.models import attribute as attribute_models from app.models import feed as feed_models -from app.models import object as object_models from app.models import user as user_models -from app.models import object_reference as object_reference_models from app.repositories import attributes as attributes_repository from app.repositories import object_references as object_references_repository from app.repositories import events as events_repository from app.schemas import event as event_schemas from app.schemas import object as object_schemas from app.schemas import attribute as attribute_schemas +from app.schemas import object_reference as object_reference_schemas from app.schemas import user as user_schemas from app.worker import tasks from app.services.opensearch import get_opensearch_client from fastapi import HTTPException, status from fastapi_pagination import Page, Params -from fastapi_pagination.ext.sqlalchemy import paginate from pymisp import MISPObject from sqlalchemy.orm import Session from collections import defaultdict @@ -64,42 +61,6 @@ def enrich_object_attributes_with_correlations( return attributes -def get_objects( - db: Session, - event_uuid: str = None, - deleted: bool = False, - template_uuid: list[UUID] = None, -) -> list[object_models.Object]: - query = db.query(object_models.Object) - - if event_uuid is not None: - os_client = get_opensearch_client() - _resp = os_client.search( - index="misp-objects", - body={ - "query": {"term": {"event_uuid.keyword": str(event_uuid)}}, - "size": 10000, - "_source": ["uuid"], - }, - ) - object_uuids = [h["_source"]["uuid"] for h in _resp["hits"]["hits"]] - if not object_uuids: - from fastapi_pagination import Page as _Page - return _Page(items=[], total=0, page=1, size=50, pages=0) - query = query.filter(object_models.Object.uuid.in_(object_uuids)) - - if template_uuid is not None: - query = query.filter(object_models.Object.template_uuid.in_(template_uuid)) - - query = query.filter(object_models.Object.deleted.is_(bool(deleted))) - - objects_page = paginate(db, query) - - for obj in objects_page.items: - obj.attributes = enrich_object_attributes_with_correlations(obj.attributes) - - return objects_page - def get_objects_from_opensearch( params: Params, @@ -206,19 +167,11 @@ def get_object_from_opensearch( def get_object_by_id(db: Session, object_id: int): - return ( - db.query(object_models.Object) - .filter(object_models.Object.id == object_id) - .first() - ) + return get_object_from_opensearch(object_id) def get_object_by_uuid(db: Session, object_uuid: UUID): - return ( - db.query(object_models.Object) - .filter(object_models.Object.uuid == object_uuid) - .first() - ) + return get_object_from_opensearch(object_uuid) def create_object( @@ -280,142 +233,141 @@ def create_object( def create_object_from_pulled_object( db: Session, pulled_object: MISPObject, event_uuid: str, user: user_models.User -) -> object_models.Object: - # TODO: process sharing group // captureSG - # TODO: enforce warninglist - - db_object = object_models.Object( - name=pulled_object.name, - meta_category=pulled_object["meta-category"], - description=pulled_object.description, - template_uuid=pulled_object.template_uuid, - template_version=pulled_object.template_version, - uuid=pulled_object.uuid, - timestamp=pulled_object.timestamp.timestamp(), - distribution=event_schemas.DistributionLevel(pulled_object.distribution), - sharing_group_id=None, - comment=pulled_object.comment, - deleted=pulled_object.deleted, - first_seen=( - pulled_object.first_seen.timestamp() - if hasattr(pulled_object, "first_seen") +) -> object_schemas.Object: + from datetime import datetime as _datetime + + client = get_opensearch_client() + object_uuid = str(pulled_object.uuid) + + dist = pulled_object.distribution + dist_val = event_schemas.DistributionLevel(dist).value if dist is not None else 5 + ts_raw = pulled_object.timestamp + ts = int(ts_raw.timestamp()) if hasattr(ts_raw, "timestamp") else int(ts_raw or 0) + + obj_doc = { + "uuid": object_uuid, + "event_uuid": event_uuid, + "name": pulled_object.name, + "meta_category": pulled_object["meta-category"], + "description": pulled_object.description, + "template_uuid": str(pulled_object.template_uuid) if pulled_object.template_uuid else None, + "template_version": pulled_object.template_version, + "timestamp": ts, + "distribution": dist_val, + "sharing_group_id": None, + "comment": pulled_object.comment or "", + "deleted": pulled_object.deleted or False, + "first_seen": ( + int(pulled_object.first_seen.timestamp()) + if hasattr(pulled_object, "first_seen") and pulled_object.first_seen else None ), - last_seen=( - pulled_object.last_seen.timestamp() - if hasattr(pulled_object, "last_seen") + "last_seen": ( + int(pulled_object.last_seen.timestamp()) + if hasattr(pulled_object, "last_seen") and pulled_object.last_seen else None ), - ) + "object_references": [], + "@timestamp": _datetime.fromtimestamp(ts).isoformat(), + } + + client.index(index="misp-objects", id=object_uuid, body=obj_doc, refresh=True) for pulled_attribute in pulled_object.attributes: - local_object_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_attribute, event_uuid, user - ) + local_attribute = attributes_repository.create_attribute_from_pulled_attribute( + db, pulled_attribute, event_uuid, user ) - db_object.attributes.append(local_object_attribute) + if local_attribute: + client.update( + index="misp-attributes", + id=str(local_attribute.uuid), + body={"doc": {"object_uuid": object_uuid}}, + refresh=True, + ) for pulled_object_reference in pulled_object.ObjectReference: - local_object_reference = object_references_repository.create_object_reference_from_pulled_object_reference( + object_references_repository.create_object_reference_from_pulled_object_reference( db, pulled_object_reference, event_uuid ) - db_object.object_references.append(local_object_reference) - db.add(db_object) + tasks.handle_created_object(object_uuid, event_uuid) - return db_object + return get_object_from_opensearch(UUID(object_uuid)) def update_object_from_pulled_object( db: Session, - local_object: object_models.Object, + local_object: object_schemas.Object, pulled_object: MISPObject, event_uuid: str, user: user_models.User, ): + ts_raw = pulled_object.timestamp + pulled_ts = ts_raw.timestamp() if hasattr(ts_raw, "timestamp") else float(ts_raw or 0) - if local_object.timestamp < pulled_object.timestamp.timestamp(): - # find object attributes to delete - local_object_attribute_uuids = [ - attribute.uuid for attribute in local_object.attributes - ] - pulled_object_attribute_uuids = [ - attribute.uuid for attribute in pulled_object.attributes - ] - delete_attributes = [ - str(uuid) - for uuid in local_object_attribute_uuids - if uuid not in pulled_object_attribute_uuids - ] - - for pulled_object_attribute in pulled_object.attributes: - pulled_object_attribute.object_id = local_object.id - local_attribute = attributes_repository.get_attribute_by_uuid( - db, pulled_object_attribute.uuid - ) + if local_object.timestamp < pulled_ts: + client = get_opensearch_client() + local_attr_uuids = {str(a.uuid) for a in local_object.attributes} + pulled_attr_uuids = {str(a.uuid) for a in pulled_object.attributes} + + for pulled_attr in pulled_object.attributes: + local_attribute = attributes_repository.get_attribute_by_uuid(db, pulled_attr.uuid) if local_attribute is None: - local_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_object_attribute, event_uuid, user - ) + new_attr = attributes_repository.create_attribute_from_pulled_attribute( + db, pulled_attr, event_uuid, user ) + if new_attr: + client.update( + index="misp-attributes", + id=str(new_attr.uuid), + body={"doc": {"object_uuid": str(local_object.uuid)}}, + refresh=True, + ) else: - pulled_object_attribute.id = local_attribute.id attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_object_attribute, event_uuid, user + db, local_attribute, pulled_attr, event_uuid, user ) + for uuid_to_delete in local_attr_uuids - pulled_attr_uuids: + attributes_repository.delete_attribute(db, uuid_to_delete) + object_patch = object_schemas.ObjectUpdate( name=pulled_object.name, meta_category=pulled_object["meta-category"], description=pulled_object.description, template_uuid=pulled_object.template_uuid, template_version=pulled_object.template_version, - timestamp=pulled_object.timestamp.timestamp(), + timestamp=int(pulled_ts), distribution=event_schemas.DistributionLevel(pulled_object.distribution), sharing_group_id=None, comment=pulled_object.comment, deleted=pulled_object.deleted, first_seen=( - pulled_object.first_seen.timestamp() - if hasattr(pulled_object, "first_seen") + int(pulled_object.first_seen.timestamp()) + if hasattr(pulled_object, "first_seen") and pulled_object.first_seen else local_object.first_seen ), last_seen=( - pulled_object.last_seen.timestamp() - if hasattr(pulled_object, "last_seen") + int(pulled_object.last_seen.timestamp()) + if hasattr(pulled_object, "last_seen") and pulled_object.last_seen else local_object.last_seen ), - delete_attributes=delete_attributes, ) - for pulled_object_reference in pulled_object.ObjectReference: - local_object_reference = ( - object_references_repository.get_object_reference_by_uuid( - db, pulled_object_reference.uuid - ) + for pulled_ref in pulled_object.ObjectReference: + local_ref = object_references_repository.get_object_reference_by_uuid( + db, pulled_ref.uuid ) - - if local_object_reference is None: - local_object_reference = object_references_repository.create_object_reference_from_pulled_object_reference( - db, pulled_object_reference, event_uuid + if local_ref is None: + object_references_repository.create_object_reference_from_pulled_object_reference( + db, pulled_ref, event_uuid + ) + elif local_ref.timestamp < int(pulled_ts): + object_references_repository.update_object_reference_from_pulled_object_reference( + db, local_ref, pulled_ref, event_uuid ) - local_object.object_references.append(local_object_reference) - else: - if ( - local_object_reference.timestamp - < pulled_object.timestamp.timestamp() - ): - pulled_object_reference.id = local_object_reference.id - local_object_reference = object_references_repository.update_object_reference_from_pulled_object_reference( - db, - local_object_reference, - pulled_object_reference, - event_uuid, - ) - update_object(db, local_object.id, object_patch) + update_object(db, local_object.uuid, object_patch) def update_object( @@ -497,131 +449,68 @@ def update_objects_from_fetched_event( feed: feed_models.Feed, user: user_schemas.User, ) -> event_schemas.Event: - from app.services.opensearch import get_opensearch_client as _get_os_client - _os = _get_os_client() - _resp = _os.search( + client = get_opensearch_client() + resp = client.search( index="misp-objects", - body={"query": {"term": {"event_uuid.keyword": str(local_event.uuid)}}, "size": 10000}, + body={ + "query": {"term": {"event_uuid.keyword": str(local_event.uuid)}}, + "size": 10000, + }, ) - local_event_objects = [ - (h["_source"]["uuid"], h["_source"].get("timestamp", 0)) - for h in _resp["hits"]["hits"] - ] - local_event_dict = {str(uuid): timestamp for uuid, timestamp in local_event_objects} - - new_objects = [ - object for object in event.objects if object.uuid not in local_event_dict.keys() - ] + local_event_dict = { + h["_source"]["uuid"]: h["_source"].get("timestamp", 0) + for h in resp["hits"]["hits"] + } + new_objects = [obj for obj in event.objects if str(obj.uuid) not in local_event_dict] updated_objects = [ - object - for object in event.objects - if object.uuid in local_event_dict - and object.timestamp.timestamp() > local_event_dict[object.uuid] + obj + for obj in event.objects + if str(obj.uuid) in local_event_dict + and obj.timestamp.timestamp() > local_event_dict[str(obj.uuid)] ] - # add new objects - local_event = create_objects_from_fetched_event( - db, local_event, new_objects, feed, user - ) - - # update existing attributes - batch_size = 100 # TODO: set the batch size via configuration - updated_uuids = [object.uuid for object in updated_objects] - - for batch_start in range(0, len(updated_uuids), batch_size): - batch_uuids = updated_uuids[batch_start : batch_start + batch_size] + create_objects_from_fetched_event(db, local_event, new_objects, feed, user) - db_objects = ( - db.query(object_models.Object) - .filter(object_models.Object.uuid.in_(batch_uuids)) - .enable_eagerloads(False) - .yield_per(batch_size) - ) - - updated_objects_dict = {object.uuid: object for object in updated_objects} - - for db_object in db_objects: - updated_object = updated_objects_dict[str(db_object.uuid)] - db_object.name = updated_object.name - db_object.meta_category = updated_object["meta-category"] - db_object.description = updated_object.description - db_object.template_uuid = updated_object.template_uuid - db_object.template_version = updated_object.template_version - db_object.timestamp = (updated_object.timestamp.timestamp(),) - db_object.comment = (updated_object.comment,) - db_object.deleted = updated_object.deleted - db_object.first_seen = ( - ( - updated_object.first_seen.timestamp() - if hasattr(updated_object, "first_seen") - else None - ), - ) - db_object.last_seen = ( - ( - updated_object.last_seen.timestamp() - if hasattr(updated_object, "last_seen") - else None - ), + for updated_object in updated_objects: + local_obj = get_object_from_opensearch(updated_object.uuid) + if local_obj is not None: + update_object_from_pulled_object( + db, local_obj, updated_object, str(local_event.uuid), user ) - for attribute in updated_object.attributes: - attribute.object_id = db_object.id - - # process attributes - local_event = attributes_repository.update_attributes_from_fetched_event( - db, local_event, updated_object.attributes, feed, user - ) - - # TODO: process galaxies - # TODO: process attribute sightings - # TODO: process analyst notes - - # process object references - for updated_object in updated_objects: - for reference in updated_object.references: - referenced = ( - db.query(object_models.Object) - .filter_by(uuid=reference.referenced_uuid) - .first() + for reference in updated_object.references: + referenced_type = None + referenced = get_object_from_opensearch(reference.referenced_uuid) + if referenced is None: + os_attr = attributes_repository.get_attribute_from_opensearch( + reference.referenced_uuid ) - if referenced is None: - referenced = ( - db.query(attribute_models.Attribute) - .filter_by(uuid=reference.referenced_uuid) - .first() - ) - if referenced is None: - logger.error( - f"Referenced entity not found, skipping object reference uuid: {reference.uuid}" - ) - break + if os_attr is not None: + referenced = os_attr + referenced_type = object_reference_schemas.ReferencedType.ATTRIBUTE + if referenced is None: + logger.error( + f"Referenced entity not found, skipping object reference uuid: {reference.uuid}" + ) + continue + if referenced_type is None: + referenced_type = object_reference_schemas.ReferencedType.OBJECT - db_object_reference = object_reference_models.ObjectReference( + object_references_repository.create_object_reference( + db, + object_reference_schemas.ObjectReferenceCreate( uuid=reference.uuid, - event_uuid=str(local_event.uuid), - object_id=db_object.id, - referenced_uuid=referenced.uuid, - referenced_id=referenced.id if referenced else None, + event_uuid=UUID(str(local_event.uuid)), + source_uuid=getattr(reference, "object_uuid", None), + referenced_uuid=reference.referenced_uuid, + referenced_id=None, relationship_type=reference.relationship_type, - timestamp=int(reference.timestamp), - referenced_type=( - object_reference_models.ReferencedType.ATTRIBUTE - if referenced.__class__.__name__ == "Attribute" - else object_reference_models.ReferencedType.OBJECT - ), - comment=reference.comment, + timestamp=int(reference.timestamp) if reference.timestamp else 0, + referenced_type=referenced_type, + comment=reference.comment or "", deleted=referenced.deleted, - ) - db.add(db_object_reference) - - db.commit() - - db.commit() - - # # TODO: process shadow_attributes - - # db.commit() + ), + ) return local_event diff --git a/api/app/repositories/sync.py b/api/app/repositories/sync.py index b14094e4..12ddb49a 100644 --- a/api/app/repositories/sync.py +++ b/api/app/repositories/sync.py @@ -6,7 +6,6 @@ from app.models import tag as tag_models from app.schemas import event as event_schemas from app.models import user as user_models -from app.models import attribute as attribute_models from app.models import object as object_models from app.repositories import attributes as attributes_repository from app.repositories import objects as objects_repository @@ -95,7 +94,7 @@ def create_pulled_event_reports( def create_pulled_event_attributes( db: Session, event_uuid: str, - attributes: list[attribute_models.Attribute], + attributes: list[MISPAttribute], user: user_models.User, ): hashes_dict = {} @@ -104,13 +103,10 @@ def create_pulled_event_attributes( (str(attribute.value) + attribute.type + attribute.category).encode("utf-8") ).hexdigest() if hash not in hashes_dict: - local_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, attribute, event_uuid, user - ) + attributes_repository.create_attribute_from_pulled_attribute( + db, attribute, event_uuid, user ) hashes_dict[hash] = True - db.add(local_attribute) db.commit() @@ -160,12 +156,9 @@ def update_pulled_event_attributes( ) if local_attribute is None: - local_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_attribute, event_uuid, user - ) + attributes_repository.create_attribute_from_pulled_attribute( + db, pulled_attribute, event_uuid, user ) - db.add(local_attribute) else: attributes_repository.update_attribute_from_pulled_attribute( db, local_attribute, pulled_attribute, event_uuid, user diff --git a/api/app/schemas/attribute.py b/api/app/schemas/attribute.py index e3209161..1573dcbe 100644 --- a/api/app/schemas/attribute.py +++ b/api/app/schemas/attribute.py @@ -1,3 +1,4 @@ +import logging from enum import Enum from typing import Optional from uuid import UUID @@ -6,6 +7,8 @@ from app.schemas.tag import Tag from pydantic import BaseModel, ConfigDict +logger = logging.getLogger(__name__) + class AttributeBase(BaseModel): object_id: Optional[int] = None @@ -33,6 +36,38 @@ class Attribute(AttributeBase): expanded: Optional[dict] = None model_config = ConfigDict(from_attributes=True) + def to_misp_format(self) -> dict: + from app.services.attachments import get_b64_attachment + from app.settings import get_settings + + attr_json = { + "id": None, + "object_id": self.object_id, + "object_relation": self.object_relation, + "category": self.category, + "type": self.type, + "value": self.value, + "to_ids": self.to_ids, + "uuid": str(self.uuid), + "timestamp": self.timestamp, + "distribution": self.distribution, + "sharing_group_id": self.sharing_group_id, + "comment": self.comment, + "deleted": self.deleted, + "disable_correlation": self.disable_correlation, + "first_seen": self.first_seen, + "last_seen": self.last_seen, + "Tag": [tag.model_dump() for tag in self.tags], + } + + if self.type in ["malware-sample", "attachment"]: + try: + attr_json["data"] = get_b64_attachment(str(self.uuid), get_settings()) + except Exception as e: + logger.error(f"Error fetching attachment: {e}") + + return attr_json + class AttributeCreate(AttributeBase): pass diff --git a/api/app/schemas/object.py b/api/app/schemas/object.py index 1dd0bc69..952e9fdf 100644 --- a/api/app/schemas/object.py +++ b/api/app/schemas/object.py @@ -30,6 +30,26 @@ class Object(ObjectBase): object_references: list[ObjectReference] = [] model_config = ConfigDict(from_attributes=True) + def to_misp_format(self) -> dict: + return { + "id": None, + "name": self.name, + "meta-category": self.meta_category, + "description": self.description if self.description else self.name, + "template_uuid": self.template_uuid, + "template_version": self.template_version, + "uuid": str(self.uuid), + "timestamp": self.timestamp, + "distribution": self.distribution, + "sharing_group_id": self.sharing_group_id, + "comment": self.comment, + "deleted": self.deleted, + "first_seen": self.first_seen, + "last_seen": self.last_seen, + "Attribute": [attr.to_misp_format() for attr in self.attributes], + "ObjectReference": [ref.to_misp_format() for ref in self.object_references], + } + class ObjectCreate(ObjectBase): event_uuid: Optional[UUID] = None diff --git a/api/app/schemas/object_reference.py b/api/app/schemas/object_reference.py index feb3f355..c4eea0ef 100644 --- a/api/app/schemas/object_reference.py +++ b/api/app/schemas/object_reference.py @@ -1,32 +1,62 @@ +import enum from typing import Optional from uuid import UUID -from app.models.object_reference import ReferencedType from pydantic import BaseModel, ConfigDict +class ReferencedType(enum.Enum): + ATTRIBUTE = 0 + OBJECT = 1 + + class ObjectReferenceBase(BaseModel): uuid: UUID - object_id: int + object_id: Optional[int] = None event_uuid: Optional[UUID] = None source_uuid: Optional[UUID] = None referenced_uuid: Optional[UUID] = None - timestamp: int + timestamp: Optional[int] = None referenced_id: Optional[int] = None referenced_type: Optional[ReferencedType] = None relationship_type: Optional[str] = None comment: Optional[str] = None - deleted: bool + deleted: bool = False model_config = ConfigDict(use_enum_values=True) class ObjectReference(ObjectReferenceBase): - id: int model_config = ConfigDict(from_attributes=True) + def to_misp_format(self) -> dict: + ref_type = self.referenced_type + if isinstance(ref_type, ReferencedType): + ref_type_name = ref_type.name + elif ref_type is not None: + try: + ref_type_name = ReferencedType(ref_type).name + except ValueError: + ref_type_name = str(ref_type) + else: + ref_type_name = None + + return { + "id": None, + "uuid": str(self.uuid), + "timestamp": self.timestamp, + "object_id": self.object_id, + "event_uuid": str(self.event_uuid) if self.event_uuid else None, + "source_uuid": str(self.source_uuid) if self.source_uuid else None, + "referenced_uuid": str(self.referenced_uuid) if self.referenced_uuid else None, + "referenced_id": self.referenced_id, + "referenced_type": ref_type_name, + "relationship_type": self.relationship_type, + "comment": self.comment, + "deleted": self.deleted, + } + class ObjectReferenceCreate(ObjectReferenceBase): - object_id: Optional[int] = None referenced_type: Optional[ReferencedType] = None comment: Optional[str] = "" diff --git a/api/app/tests/api/test_attributes.py b/api/app/tests/api/test_attributes.py index 13d454b4..bc9c929f 100644 --- a/api/app/tests/api/test_attributes.py +++ b/api/app/tests/api/test_attributes.py @@ -1,6 +1,5 @@ import pytest from app.auth import auth -from app.models import attribute as attribute_models from app.models import tag as tag_models from app.tests.api_tester import ApiTester from fastapi import status @@ -13,7 +12,7 @@ class TestAttributesResource(ApiTester): def test_get_attributes( self, client: TestClient, - attribute_1: attribute_models.Attribute, + attribute_1, auth_token: auth.Token, ): response = client.get( @@ -98,7 +97,7 @@ def test_create_attribute_incomplete( def test_update_attribute( self, client: TestClient, - attribute_1: attribute_models.Attribute, + attribute_1, auth_token: auth.Token, ): response = client.patch( @@ -119,7 +118,7 @@ def test_update_attribute( def test_delete_attribute( self, client: TestClient, - attribute_1: attribute_models.Attribute, + attribute_1, auth_token: auth.Token, ): response = client.delete( @@ -134,7 +133,7 @@ def test_tag_attribute( self, client: TestClient, event_1: object, - attribute_1: attribute_models.Attribute, + attribute_1, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, @@ -157,7 +156,7 @@ def test_untag_event( self, client: TestClient, event_1: object, - attribute_1: attribute_models.Attribute, + attribute_1, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, diff --git a/api/app/tests/api/test_events.py b/api/app/tests/api/test_events.py index d95ee756..2a4155c6 100644 --- a/api/app/tests/api/test_events.py +++ b/api/app/tests/api/test_events.py @@ -1,6 +1,5 @@ import pytest from app.auth import auth -from app.models import attribute as attribute_models from app.models import organisation as organisation_models from app.models import tag as tag_models from app.models import user as user_models diff --git a/api/app/tests/api/test_objects.py b/api/app/tests/api/test_objects.py index 7eeaf9d0..2b64216d 100644 --- a/api/app/tests/api/test_objects.py +++ b/api/app/tests/api/test_objects.py @@ -1,6 +1,5 @@ import pytest from app.auth import auth -from app.models import object as object_models from app.tests.api_tester import ApiTester from fastapi import status from fastapi.testclient import TestClient @@ -11,7 +10,7 @@ class TestObjectsResource(ApiTester): def test_get_objects( self, client: TestClient, - object_1: object_models.Object, + object_1, auth_token: auth.Token, ): response = client.get( @@ -95,7 +94,7 @@ def test_create_object_incomplete( def test_update_object( self, client: TestClient, - object_1: object_models.Object, + object_1, auth_token: auth.Token, ): response = client.patch( @@ -116,7 +115,7 @@ def test_update_object( def test_delete_object( self, client: TestClient, - object_1: object_models.Object, + object_1, auth_token: auth.Token, ): response = client.delete( diff --git a/api/app/tests/api_tester.py b/api/app/tests/api_tester.py index 85a25fc7..cc56fa87 100644 --- a/api/app/tests/api_tester.py +++ b/api/app/tests/api_tester.py @@ -5,14 +5,11 @@ from app.auth import auth from app.db.session import get_db from app.main import app -from app.models import attribute as attribute_models from app.models import event as event_models from app.models import feed as feed_models from app.models import galaxy as galaxy_models from app.models import hunt as hunt_models from app.models import module as module_models -from app.models import object as object_models -from app.models import object_reference as object_reference_models from app.models import organisation as organisation_models from app.models import server as server_models from app.models import sharing_groups as sharing_groups_models @@ -72,9 +69,6 @@ def teardown_db(self, db: Session): db.query(tag_models.AttributeTag).delete(synchronize_session=False) db.query(tag_models.EventTag).delete(synchronize_session=False) db.query(tag_models.Tag).delete(synchronize_session=False) - db.query(attribute_models.Attribute).delete(synchronize_session=False) - db.query(object_reference_models.ObjectReference).delete(synchronize_session=False) - db.query(object_models.Object).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupOrganisation).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupServer).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroup).delete(synchronize_session=False) @@ -192,125 +186,78 @@ def event_1( @pytest.fixture(scope="class") def attribute_1(self, db: Session, event_1): - from datetime import datetime - from app.services.opensearch import get_opensearch_client + from uuid import UUID + from app.repositories import attributes as attributes_repository + from app.schemas import attribute as attribute_schemas - attribute_1 = attribute_models.Attribute( + attr_create = attribute_schemas.AttributeCreate( category="Network activity", type="ip-src", value="127.0.0.1", - uuid="7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0", + uuid=UUID("7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0"), timestamp=157783680, + event_uuid=event_1.uuid, + deleted=False, + to_ids=False, + disable_correlation=False, ) - db.add(attribute_1) - db.commit() - db.refresh(attribute_1) - - client = get_opensearch_client() - client.index( - index="misp-attributes", - id=str(attribute_1.uuid), - body={ - "uuid": str(attribute_1.uuid), - "event_uuid": str(event_1.uuid), - "category": attribute_1.category, - "type": attribute_1.type, - "value": attribute_1.value, - "timestamp": attribute_1.timestamp, - "@timestamp": datetime.fromtimestamp(attribute_1.timestamp).isoformat(), - "deleted": attribute_1.deleted or False, - "to_ids": attribute_1.to_ids or False, - "disable_correlation": attribute_1.disable_correlation or False, - "distribution": attribute_1.distribution.value if attribute_1.distribution else 0, - "sharing_group_id": attribute_1.sharing_group_id, - "comment": attribute_1.comment, - "first_seen": attribute_1.first_seen, - "last_seen": attribute_1.last_seen, - "data": "", - "tags": [], - }, - refresh=True, - ) - - yield attribute_1 + yield attributes_repository.create_attribute(db, attr_create) @pytest.fixture(scope="class") def object_1(self, db: Session, event_1): from datetime import datetime + from uuid import UUID as _UUID from app.services.opensearch import get_opensearch_client - object_1 = object_models.Object( - uuid="90e06ef6-26f8-40dd-9fb7-75897445e2a0", - name="test object", - template_version=0, - timestamp=1577836800, - deleted=False, - ) - db.add(object_1) - db.commit() - db.refresh(object_1) + obj_uuid = "90e06ef6-26f8-40dd-9fb7-75897445e2a0" + obj_doc = { + "uuid": obj_uuid, + "event_uuid": str(event_1.uuid), + "name": "test object", + "meta_category": None, + "template_uuid": None, + "template_version": 0, + "timestamp": 1577836800, + "@timestamp": datetime.fromtimestamp(1577836800).isoformat(), + "deleted": False, + "distribution": 0, + "sharing_group_id": None, + "first_seen": None, + "last_seen": None, + "comment": "", + "object_references": [], + } client = get_opensearch_client() - client.index( - index="misp-objects", - id=str(object_1.uuid), - body={ - "uuid": str(object_1.uuid), - "event_uuid": str(event_1.uuid), - "name": object_1.name, - "meta_category": object_1.meta_category, - "template_uuid": object_1.template_uuid, - "template_version": object_1.template_version, - "timestamp": object_1.timestamp, - "@timestamp": datetime.fromtimestamp(object_1.timestamp).isoformat(), - "deleted": object_1.deleted, - "distribution": object_1.distribution.value if object_1.distribution else 0, - "sharing_group_id": object_1.sharing_group_id, - "first_seen": object_1.first_seen, - "last_seen": object_1.last_seen, - "object_references": [], - }, - refresh=True, - ) + client.index(index="misp-objects", id=obj_uuid, body=obj_doc, refresh=True) - yield object_1 + from app.schemas import object as object_schemas + yield object_schemas.Object.model_validate(obj_doc) @pytest.fixture(scope="class") def object_attribute_1( - self, db: Session, event_1, object_1: object_models.Object + self, db: Session, event_1, object_1 ): - from datetime import datetime + from uuid import UUID + from app.repositories import attributes as attributes_repository + from app.schemas import attribute as attribute_schemas from app.services.opensearch import get_opensearch_client - object_attribute_1 = attribute_models.Attribute( - object_id=object_1.id, + attr_create = attribute_schemas.AttributeCreate( category="Network activity", type="ip-src", value="127.0.0.2", - uuid="1355e435-aa0f-4f06-acd3-b44498131e82", + uuid=UUID("1355e435-aa0f-4f06-acd3-b44498131e82"), timestamp=1577836800, + event_uuid=event_1.uuid, + deleted=False, ) - db.add(object_attribute_1) - db.commit() - db.refresh(object_attribute_1) + object_attribute_1 = attributes_repository.create_attribute(db, attr_create) - client = get_opensearch_client() - client.index( + get_opensearch_client().update( index="misp-attributes", id=str(object_attribute_1.uuid), - body={ - "uuid": str(object_attribute_1.uuid), - "event_uuid": str(event_1.uuid), - "object_uuid": str(object_1.uuid), - "category": object_attribute_1.category, - "type": object_attribute_1.type, - "value": object_attribute_1.value, - "timestamp": object_attribute_1.timestamp, - "@timestamp": datetime.fromtimestamp(object_attribute_1.timestamp).isoformat(), - "deleted": False, - "data": "", - "tags": [], - }, + body={"doc": {"object_uuid": str(object_1.uuid)}}, refresh=True, ) diff --git a/api/app/tests/repositories/test_feeds.py b/api/app/tests/repositories/test_feeds.py index fa6540ff..71779bf1 100644 --- a/api/app/tests/repositories/test_feeds.py +++ b/api/app/tests/repositories/test_feeds.py @@ -2,14 +2,13 @@ from uuid import UUID -from app.models import attribute as attribute_models from app.models import feed as feed_models -from app.models import object as object_models -from app.models import object_reference as object_reference_models from app.models import tag as tag_models from app.models import user as user_models +from app.repositories import attributes as attributes_repository from app.repositories import events as events_repository from app.repositories import feeds as feeds_repository +from app.repositories import objects as objects_repository from app.tests.api_tester import ApiTester from app.tests.scenarios import feed_fetch_scenarios from sqlalchemy.orm import Session @@ -51,40 +50,27 @@ def test_fetch_feed_by_id_new_event( assert os_event is not None # check that the attributes were created - attributes = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid.in_( - [ - "317e63e6-b95d-4dd1-b4fd-de2f64f33fd8", - "8be7a04d-c10b-4ef6-854f-2072e67f6cd5", - ] - ) - ) - .all() - ) - assert len(attributes) == 2 + attributes = [ + attributes_repository.get_attribute_from_opensearch(UUID(u)) + for u in [ + "317e63e6-b95d-4dd1-b4fd-de2f64f33fd8", + "8be7a04d-c10b-4ef6-854f-2072e67f6cd5", + ] + ] + assert len([a for a in attributes if a is not None]) == 2 # check the objects were created - objects = ( - db.query(object_models.Object) - .filter( - object_models.Object.uuid == "df23d3be-1179-4824-ac03-471f0bc6d92d" - ) - .all() + obj = objects_repository.get_object_from_opensearch( + UUID("df23d3be-1179-4824-ac03-471f0bc6d92d") ) - assert len(objects) == 1 + assert obj is not None # check the object references were created - object_references = ( - db.query(object_reference_models.ObjectReference) - .filter( - object_reference_models.ObjectReference.uuid - == "d7e57f39-4dd5-4b87-b040-75561fa8289e" - ) - .all() + from app.repositories import object_references as object_references_repository + object_reference = object_references_repository.get_object_reference_by_uuid( + db, UUID("d7e57f39-4dd5-4b87-b040-75561fa8289e") ) - assert len(object_references) == 1 + assert object_reference is not None # check the tags were created tags = db.query(tag_models.Tag).all() @@ -117,9 +103,9 @@ def test_fetch_feed_by_id_existing_event( db: Session, feed_1: feed_models.Feed, event_1, - attribute_1: attribute_models.Attribute, + attribute_1, object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, + object_attribute_1, user_1: user_models.User, ): # mock remote Feed API calls @@ -154,14 +140,10 @@ def test_fetch_feed_by_id_existing_event( assert os_event.timestamp == 1577836801 # check that the attribute was updated - attribute = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid - == "7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0" - ) - .first() + attribute = attributes_repository.get_attribute_from_opensearch( + UUID("7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0") ) + assert attribute is not None assert attribute.value == "7edc546f741eff3e13590a62ce2856bb39d8f71d" assert attribute.timestamp == 1577836801 @@ -177,14 +159,10 @@ def test_fetch_feed_by_id_existing_event( assert object.timestamp == 1577836801 # check the object attribute was added - object_attribute = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid - == "011aca4f-eaf0-4a06-8133-b69f3806cbe8" - ) - .first() + object_attribute = attributes_repository.get_attribute_from_opensearch( + UUID("011aca4f-eaf0-4a06-8133-b69f3806cbe8") ) + assert object_attribute is not None assert object_attribute.value == "Foobar12345" assert object_attribute.timestamp == 1577836801 diff --git a/api/app/tests/repositories/test_servers.py b/api/app/tests/repositories/test_servers.py index 87ced78b..5a9dc45d 100644 --- a/api/app/tests/repositories/test_servers.py +++ b/api/app/tests/repositories/test_servers.py @@ -3,8 +3,8 @@ import pytest from uuid import UUID -from app.models import attribute as attribute_models from app.models import object as object_models +from app.repositories import attributes as attributes_repository from app.models import object_reference as object_reference_models from app.models import organisation as organisations_models from app.models import server as server_models @@ -16,7 +16,6 @@ from app.settings import Settings from app.tests.api_tester import ApiTester from app.tests.scenarios import server_pull_scenarios -from sqlalchemy import and_ from sqlalchemy.orm import Session @@ -79,15 +78,11 @@ def test_pull_server_by_id( assert len(os_events) == len(scenario["expected_result"]["event_uuids"]) # check that the attributes were created - attributes = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid.in_( - scenario["expected_result"]["attribute_uuids"] - ) - ) - .all() - ) + attributes = [ + attributes_repository.get_attribute_from_opensearch(UUID(u)) + for u in scenario["expected_result"]["attribute_uuids"] + ] + attributes = [a for a in attributes if a is not None] assert len(attributes) == len( scenario["expected_result"]["attribute_uuids"] ) @@ -167,15 +162,7 @@ def test_pull_server_by_id( attribute_tags = ( db.query(tag_models.Tag) .join(tag_models.AttributeTag) - .filter( - and_( - tag_models.Tag.name.in_(attribute_tag["tags"]), - attribute_models.Attribute.uuid - == attribute_tag["attribute_uuid"], - attribute_models.Attribute.id - == tag_models.AttributeTag.attribute_id, - ) - ) + .filter(tag_models.Tag.name.in_(attribute_tag["tags"])) .all() ) assert len(attribute_tags) == len(attribute_tag["tags"]) From 613c35121372d5df1fbd12c35b6648bb45823794 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Fri, 27 Mar 2026 15:10:27 +0100 Subject: [PATCH 08/38] chg: [refactor] remove references to event_id, object_id, attribute_id, adjust tests accordingly --- api/README.md | 2 +- api/app/models/__init__.py | 4 +- api/app/models/object.py | 52 ---------- api/app/models/object_reference.py | 52 ---------- api/app/models/tag.py | 23 ----- api/app/repositories/attributes.py | 97 +++++++------------ api/app/repositories/events.py | 50 +++++----- api/app/repositories/notifications.py | 18 ---- api/app/repositories/object_references.py | 2 +- api/app/repositories/objects.py | 61 ++++++------ api/app/repositories/reports.py | 10 +- api/app/repositories/servers.py | 8 +- api/app/repositories/sync.py | 5 +- api/app/repositories/tags.py | 8 +- api/app/routers/attributes.py | 19 ++-- api/app/routers/events.py | 35 ++++++- api/app/routers/mcp.py | 13 ++- api/app/routers/objects.py | 20 ++-- api/app/schemas/attribute.py | 6 +- api/app/schemas/event.py | 2 +- api/app/schemas/object_reference.py | 6 +- api/app/schemas/tag.py | 21 ---- api/app/tests/api/test_events.py | 32 ------ api/app/tests/api_tester.py | 2 - api/app/tests/repositories/test_feeds.py | 89 +++++++---------- api/app/tests/repositories/test_servers.py | 62 +++++------- api/app/worker/tasks.py | 22 ++--- docs/features/enrichments.md | 2 +- .../components/attachments/AttachmentIcon.vue | 2 +- .../attachments/UploadAttachmentsWidget.vue | 4 +- .../attributes/AttributeActions.vue | 6 +- .../components/attributes/AttributeView.vue | 2 +- .../attributes/DeleteAttributeModal.vue | 10 +- .../attributes/EnrichAttributeModal.vue | 2 +- .../misc/AttributesPropertiesModal.vue | 11 ++- .../components/objects/DeleteObjectModal.vue | 10 +- .../src/components/objects/ObjectActions.vue | 2 +- .../objects/ObjectAttributesList.vue | 10 +- .../src/components/objects/ObjectsIndex.vue | 2 +- .../components/objects/ObjectsIndexRemote.vue | 2 +- .../src/components/objects/ViewObject.vue | 2 +- .../index-patterns/index-patterns.ndjson | 6 +- opensearch/mappings/misp-attributes.json | 8 +- opensearch/mappings/misp-event-reports.json | 10 +- opensearch/mappings/misp-objects.json | 8 +- 45 files changed, 294 insertions(+), 526 deletions(-) delete mode 100644 api/app/models/object.py delete mode 100644 api/app/models/object_reference.py diff --git a/api/README.md b/api/README.md index c87ec8d6..91ee7ab5 100644 --- a/api/README.md +++ b/api/README.md @@ -209,7 +209,7 @@ Example: ``` from app.worker import tasks -tasks.handle_created_attribute.delay(pulled_attribute.id, pulled_attribute.object_id, pulled_attribute.event_id) +tasks.handle_created_attribute.delay(pulled_attribute.id, pulled_attribute.object_uuid, pulled_attribute.event_uuid) ``` If you add a new task, you have to restart the celery `worker` container, otherwise you will get `NotRegistered('app.worker.tasks.new_task') ` exception. diff --git a/api/app/models/__init__.py b/api/app/models/__init__.py index c03974b4..531335b9 100644 --- a/api/app/models/__init__.py +++ b/api/app/models/__init__.py @@ -1,7 +1,5 @@ from app.models.hunt import Hunt # noqa from app.models.module import ModuleSettings # noqa -from app.models.object import Object # noqa -from app.models.object_reference import ObjectReference # noqa from app.models.organisation import Organisation # noqa from app.models.role import Role # noqa from app.models.server import Server # noqa @@ -10,7 +8,7 @@ SharingGroupOrganisation, SharingGroupServer, ) -from app.models.tag import AttributeTag, EventTag, Tag # noqa +from app.models.tag import Tag # noqa from app.models.user import User # noqa # fixes: sqlalchemy.exc.InvalidRequestError: When initializing mapper mapped class AAA->bbb, expression 'XXXX' failed to locate a name ('XXXX'). If this is a class name, consider adding this relationship() to the class after both dependent classes have been defined. diff --git a/api/app/models/object.py b/api/app/models/object.py deleted file mode 100644 index d53ff65b..00000000 --- a/api/app/models/object.py +++ /dev/null @@ -1,52 +0,0 @@ -import uuid - -from app.database import Base -from app.models.event import DistributionLevel -from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - - -class Object(Base): - __tablename__ = "objects" - - id = Column(Integer, primary_key=True, index=True) - name = Column(String) - meta_category = Column(String) - description = Column(String) - template_uuid = Column(String) - template_version = Column(Integer, nullable=False) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - timestamp = Column(Integer, nullable=False) - distribution: Mapped[DistributionLevel] = mapped_column( - Enum(DistributionLevel, name="distribution_level"), - nullable=False, - default=DistributionLevel.INHERIT_EVENT, - ) - sharing_group_id = Column(Integer, ForeignKey("sharing_groups.id")) - comment = Column(String) - deleted = Column(Boolean, nullable=False, default=False) - first_seen = Column(Integer) - last_seen = Column(Integer) - object_references = relationship("ObjectReference", lazy="subquery", cascade="all, delete-orphan") - - def to_misp_format(self): - """Convert the Object to a MISP-compatible dictionary representation.""" - return { - "id": self.id, - "name": self.name, - "meta-category": self.meta_category, - "description": self.description if self.description else self.name, - "template_uuid": self.template_uuid, - "template_version": self.template_version, - "uuid": str(self.uuid), - "timestamp": self.timestamp, - "distribution": self.distribution.value if self.distribution else None, - "sharing_group_id": self.sharing_group_id if self.sharing_group_id else None, - "comment": self.comment, - "deleted": self.deleted, - "first_seen": self.first_seen, - "last_seen": self.last_seen, - "Attribute": [], - "ObjectReference": [ref.to_misp_format() for ref in self.object_references], - } \ No newline at end of file diff --git a/api/app/models/object_reference.py b/api/app/models/object_reference.py deleted file mode 100644 index 9e417238..00000000 --- a/api/app/models/object_reference.py +++ /dev/null @@ -1,52 +0,0 @@ -import enum -import uuid - -from app.database import Base -from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column - - -class ReferencedType(enum.Enum): - """ - Enum for the Referenced Entity Type - """ - - ATTRIBUTE = 0 - OBJECT = 1 - - -class ObjectReference(Base): - __tablename__ = "object_references" - - id = Column(Integer, primary_key=True, index=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - timestamp = Column(Integer, nullable=False) - object_id = Column(Integer, ForeignKey("objects.id"), nullable=False) - event_uuid = Column(UUID(as_uuid=True), nullable=True) - source_uuid = Column(UUID(as_uuid=True)) - referenced_uuid = Column(UUID(as_uuid=True)) - referenced_id = Column(Integer, nullable=True) - referenced_type: Mapped[ReferencedType] = mapped_column( - Enum(ReferencedType, name="referenced_type"), nullable=False - ) - relationship_type = Column(String) - comment = Column(String, nullable=False) - deleted = Column(Boolean, nullable=False, default=False) - - def to_misp_format(self): - """Convert the ObjectReference to a MISP-compatible dictionary representation.""" - return { - "id": self.id, - "uuid": str(self.uuid), - "timestamp": self.timestamp, - "object_id": self.object_id, - "event_uuid": str(self.event_uuid) if self.event_uuid else None, - "source_uuid": str(self.source_uuid) if self.source_uuid else None, - "referenced_uuid": str(self.referenced_uuid) if self.referenced_uuid else None, - "referenced_id": self.referenced_id, - "referenced_type": self.referenced_type.name if self.referenced_type else None, - "relationship_type": self.relationship_type, - "comment": self.comment, - "deleted": self.deleted, - } \ No newline at end of file diff --git a/api/app/models/tag.py b/api/app/models/tag.py index 70378ca4..11f0a74d 100644 --- a/api/app/models/tag.py +++ b/api/app/models/tag.py @@ -1,7 +1,5 @@ from app.database import Base from sqlalchemy import Boolean, Column, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import relationship class Tag(Base): @@ -34,24 +32,3 @@ def to_misp_format(self): "is_custom_galaxy": self.is_custom_galaxy, "local_only": self.local_only, } - - -class EventTag(Base): - __tablename__ = "event_tags" - - id = Column(Integer, primary_key=True, index=True) - event_uuid = Column(UUID(as_uuid=True), nullable=True) - tag_id = Column(Integer, ForeignKey("tags.id"), nullable=False) - tag = relationship("Tag", lazy="subquery", overlaps="tags") - local = Column(Boolean, nullable=False, default=False) - - -class AttributeTag(Base): - __tablename__ = "attribute_tags" - - id = Column(Integer, primary_key=True, index=True) - attribute_id = Column(Integer, nullable=True) - event_uuid = Column(UUID(as_uuid=True), nullable=True) - tag_id = Column(Integer, ForeignKey("tags.id"), nullable=False) - tag = relationship("Tag", lazy="subquery", overlaps="tags") - local = Column(Boolean, nullable=False, default=False) diff --git a/api/app/repositories/attributes.py b/api/app/repositories/attributes.py index e20164f5..b38dc279 100644 --- a/api/app/repositories/attributes.py +++ b/api/app/repositories/attributes.py @@ -1,12 +1,13 @@ import math import time from datetime import datetime -from typing import Iterable, Optional, Union +from typing import Iterable, Optional from uuid import UUID, uuid4 from app.models.event import DistributionLevel from app.services.opensearch import get_opensearch_client from app.models import tag as tag_models from app.models import user as user_models +from app.schemas import tag as tag_schemas from app.repositories import attachments as attachments_repository from app.schemas import attribute as attribute_schemas from app.schemas import event as event_schemas @@ -58,7 +59,7 @@ def get_attributes_from_opensearch( params: Params, event_uuid: str = None, deleted: bool = None, - object_id: int = None, + object_uuid: UUID = None, type: str = None, ) -> Page[attribute_schemas.Attribute]: client = get_opensearch_client() @@ -71,13 +72,13 @@ def get_attributes_from_opensearch( if type is not None: must_clauses.append({"term": {"type.keyword": type}}) - # Mirror SQL behaviour: always filter by object_id (None → standalone attributes only) - if object_id is None: + # None → standalone attributes only (no parent object) + if object_uuid is None: must_clauses.append( - {"bool": {"must_not": [{"exists": {"field": "object_id"}}]}} + {"bool": {"must_not": [{"exists": {"field": "object_uuid"}}]}} ) else: - must_clauses.append({"term": {"object_id": object_id}}) + must_clauses.append({"term": {"object_uuid": str(object_uuid)}}) query_body = { "query": {"bool": {"must": must_clauses}}, @@ -100,35 +101,19 @@ def get_attributes_from_opensearch( def get_attribute_from_opensearch( - attribute_id: Union[int, UUID], + attribute_uuid: UUID, ) -> Optional[attribute_schemas.Attribute]: client = get_opensearch_client() - if isinstance(attribute_id, int): - response = client.search( - index="misp-attributes", - body={"query": {"term": {"id": attribute_id}}, "size": 1}, - ) - hits = response["hits"]["hits"] - if not hits: - return None - source = hits[0]["_source"] - else: - try: - doc = client.get(index="misp-attributes", id=str(attribute_id)) - source = doc["_source"] - except NotFoundError: - return None + try: + doc = client.get(index="misp-attributes", id=str(attribute_uuid)) + source = doc["_source"] + except NotFoundError: + return None return attribute_schemas.Attribute.model_validate(source) -def get_attribute_by_id( - db: Session, attribute_id: int -) -> Optional[attribute_schemas.Attribute]: - return get_attribute_from_opensearch(attribute_id) - - def get_attribute_by_uuid( db: Session, attribute_uuid: UUID ) -> Optional[attribute_schemas.Attribute]: @@ -150,8 +135,7 @@ def create_attribute( attr_doc = { "uuid": attribute_uuid, "event_uuid": event_uuid, - "object_id": attribute.object_id if attribute.object_id and attribute.object_id > 0 else None, - "object_uuid": None, + "object_uuid": str(attribute.object_uuid) if attribute.object_uuid else None, "object_relation": attribute.object_relation, "category": attribute.category, "type": attribute.type, @@ -172,7 +156,7 @@ def create_attribute( client.index(index="misp-attributes", id=attribute_uuid, body=attr_doc, refresh=True) - tasks.handle_created_attribute(attribute_uuid, attr_doc["object_id"], event_uuid) + tasks.handle_created_attribute(attribute_uuid, attr_doc["object_uuid"], event_uuid) return attribute_schemas.Attribute.model_validate(attr_doc) @@ -233,7 +217,7 @@ def create_attribute_from_pulled_attribute( # TODO: process sightings # TODO: process galaxies - capture_attribute_tags(db, pulled_attribute.tags, event_uuid, user) + capture_attribute_tags(db, pulled_attribute.tags, user, str(local_attribute.uuid)) return local_attribute @@ -242,7 +226,6 @@ def update_attribute_from_pulled_attribute( db: Session, local_attribute: attribute_schemas.Attribute, pulled_attribute: MISPAttribute, - event_uuid: str, user: user_models.User, ) -> attribute_schemas.Attribute: @@ -285,14 +268,14 @@ def update_attribute_from_pulled_attribute( str(pulled_attribute.uuid), pulled_attribute.data.getvalue() ) - capture_attribute_tags(db, pulled_attribute.tags, event_uuid, user) + capture_attribute_tags(db, pulled_attribute.tags, user, str(local_attribute.uuid)) # TODO: process sightings # TODO: process galaxies tasks.handle_updated_attribute.delay( str(local_attribute.uuid), - local_attribute.object_id, + local_attribute.object_uuid, str(local_attribute.event_uuid) if local_attribute.event_uuid else None, ) @@ -300,10 +283,10 @@ def update_attribute_from_pulled_attribute( def update_attribute( - db: Session, attribute_id: Union[int, UUID], attribute: attribute_schemas.AttributeUpdate + db: Session, attribute_uuid: UUID, attribute: attribute_schemas.AttributeUpdate ) -> attribute_schemas.Attribute: client = get_opensearch_client() - os_attr = get_attribute_from_opensearch(attribute_id) + os_attr = get_attribute_from_opensearch(attribute_uuid) if os_attr is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") @@ -314,21 +297,15 @@ def update_attribute( client.update(index="misp-attributes", id=str(os_attr.uuid), body={"doc": patch}, refresh=True) - tasks.handle_updated_attribute(str(os_attr.uuid), os_attr.object_id, str(os_attr.event_uuid) if os_attr.event_uuid else None) + tasks.handle_updated_attribute(str(os_attr.uuid), os_attr.object_uuid, str(os_attr.event_uuid) if os_attr.event_uuid else None) return get_attribute_from_opensearch(os_attr.uuid) -def delete_attribute(db: Session, attribute_id: Union[int, str, UUID]) -> None: +def delete_attribute(db: Session, attribute_uuid: UUID) -> None: client = get_opensearch_client() - if isinstance(attribute_id, str): - try: - os_attr = get_attribute_from_opensearch(UUID(attribute_id)) - except ValueError: - os_attr = None - else: - os_attr = get_attribute_from_opensearch(attribute_id) + os_attr = get_attribute_from_opensearch(attribute_uuid) if os_attr is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") @@ -340,14 +317,14 @@ def delete_attribute(db: Session, attribute_id: Union[int, str, UUID]) -> None: refresh=True, ) - tasks.handle_deleted_attribute(str(os_attr.uuid), os_attr.object_id, str(os_attr.event_uuid) if os_attr.event_uuid else None) + tasks.handle_deleted_attribute(str(os_attr.uuid), os_attr.object_uuid, str(os_attr.event_uuid) if os_attr.event_uuid else None) def capture_attribute_tags( db: Session, tags: list[MISPTag], - event_uuid: str, user: user_models.User, + attribute_uuid: str = None, ): tag_name_to_db_tag = {} @@ -380,20 +357,18 @@ def capture_attribute_tags( db.add_all(new_tags) db.commit() - for tag in tags: - if tag.local: - continue - - db_tag = tag_name_to_db_tag[tag.name] - - db_attribute_tag = tag_models.AttributeTag( - event_uuid=event_uuid, - tag_id=db_tag.id, - local=tag.local, + if attribute_uuid and tag_name_to_db_tag: + client = get_opensearch_client() + tag_dicts = [ + tag_schemas.Tag.model_validate(db_tag).model_dump() + for db_tag in tag_name_to_db_tag.values() + ] + client.update( + index="misp-attributes", + id=attribute_uuid, + body={"doc": {"tags": tag_dicts}}, + refresh=True, ) - db.add(db_attribute_tag) - - db.commit() def get_vulnerability_attributes( diff --git a/api/app/repositories/events.py b/api/app/repositories/events.py index 29028298..f1da84d6 100644 --- a/api/app/repositories/events.py +++ b/api/app/repositories/events.py @@ -3,7 +3,7 @@ import time from datetime import datetime from uuid import UUID, uuid4 -from typing import Optional, Union, Iterable +from typing import Optional, Iterable from app.worker import tasks from app.services.opensearch import get_opensearch_client from app.services.vulnerability_lookup import lookup as vulnerability_lookup @@ -65,24 +65,14 @@ def get_events_from_opensearch( return Page(items=items, total=total, page=params.page, size=params.size, pages=pages) -def get_event_from_opensearch(event_id: Union[int, UUID]) -> Optional[event_schemas.Event]: +def get_event_from_opensearch(event_uuid: UUID) -> Optional[event_schemas.Event]: client = get_opensearch_client() - if isinstance(event_id, int): - response = client.search( - index="misp-events", - body={"query": {"term": {"id": event_id}}, "size": 1}, - ) - hits = response["hits"]["hits"] - if not hits: - return None - source = hits[0]["_source"] - else: - try: - doc = client.get(index="misp-events", id=str(event_id)) - source = doc["_source"] - except NotFoundError: - return None + try: + doc = client.get(index="misp-events", id=str(event_uuid)) + source = doc["_source"] + except NotFoundError: + return None source.setdefault("attributes", []) source.setdefault("objects", []) @@ -175,6 +165,16 @@ def get_event_by_info(info: str) -> Optional[event_schemas.Event]: return event_schemas.Event.model_validate(source) +def get_event_by_uuid(db, event_uuid) -> Optional[event_schemas.Event]: + """Alias for get_event_from_opensearch used by feeds/servers/mcp.""" + return get_event_from_opensearch(UUID(str(event_uuid))) + + +def get_events_by_uuids(db, uuids) -> list[event_schemas.Event]: + """Alias for get_events_by_uuids_from_opensearch used by feeds.""" + return get_events_by_uuids_from_opensearch(uuids) + + def get_event_uuids_from_opensearch() -> list[str]: """Return all event UUIDs from OpenSearch (used for server push).""" client = get_opensearch_client() @@ -269,14 +269,14 @@ def create_event_from_pulled_event(pulled_event: MISPEvent) -> event_schemas.Eve "timestamp": ts, "distribution": int(pulled_event.distribution) if pulled_event.distribution is not None else 0, "sharing_group_id": int(pulled_event.sharing_group_id) if pulled_event.sharing_group_id and int(pulled_event.sharing_group_id) > 0 else None, - "proposal_email_lock": pulled_event.proposal_email_lock or False, - "locked": pulled_event.locked or False, + "proposal_email_lock": getattr(pulled_event, "proposal_email_lock", False) or False, + "locked": getattr(pulled_event, "locked", False) or False, "threat_level": int(pulled_event.threat_level_id) if pulled_event.threat_level_id else 4, "publish_timestamp": int(pulled_event.publish_timestamp.timestamp()), - "disable_correlation": pulled_event.disable_correlation or False, + "disable_correlation": getattr(pulled_event, "disable_correlation", False) or False, "extends_uuid": str(pulled_event.extends_uuid) if pulled_event.extends_uuid else None, "protected": getattr(pulled_event, "protected", False) or False, - "deleted": pulled_event.deleted or False, + "deleted": getattr(pulled_event, "deleted", False) or False, "tags": [], "attributes": [], "objects": [], @@ -446,9 +446,9 @@ def update_event_from_fetched_event( return get_event_from_opensearch(UUID(event_uuid)) -def update_event(db: Session, event_id: Union[int, UUID], event: event_schemas.EventUpdate) -> event_schemas.Event: +def update_event(db: Session, event_uuid: UUID, event: event_schemas.EventUpdate) -> event_schemas.Event: client = get_opensearch_client() - os_event = get_event_from_opensearch(event_id) + os_event = get_event_from_opensearch(event_uuid) if os_event is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") @@ -463,9 +463,9 @@ def update_event(db: Session, event_id: Union[int, UUID], event: event_schemas.E return get_event_from_opensearch(os_event.uuid) -def delete_event(db: Session, event_id: Union[int, UUID], force: bool = False) -> None: +def delete_event(db: Session, event_uuid: UUID, force: bool = False) -> None: client = get_opensearch_client() - os_event = get_event_from_opensearch(event_id) + os_event = get_event_from_opensearch(event_uuid) if os_event is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") diff --git a/api/app/repositories/notifications.py b/api/app/repositories/notifications.py index 838cf156..083c8f93 100644 --- a/api/app/repositories/notifications.py +++ b/api/app/repositories/notifications.py @@ -4,7 +4,6 @@ from app.models import hunt as hunt_models from app.models import notification as notification_models from app.models import organisation as organisation_models -from app.models import object as object_models from app.repositories import user_settings as user_settings_repository from sqlalchemy import select, update, text from sqlalchemy.orm import Session @@ -314,23 +313,6 @@ def create_attribute_notifications( for follower in attr_followers ] - # If the attribute is linked to an object, we can also notify followers of that object - if attribute.object_id: - object = ( - db.query(object_models.Object) - .filter(object_models.Object.id == attribute.object_id) - .first() - ) - if object: - notifications = build_attribute_notification( - db, - f"object.attribute.{type}", - attribute=attribute, - event=event, - object=object, - ) - return notifications - # Followers of the event event_followers = get_followers_for(db, "events", event.uuid) notifications += [ diff --git a/api/app/repositories/object_references.py b/api/app/repositories/object_references.py index 3be9ac1e..62fe3aba 100644 --- a/api/app/repositories/object_references.py +++ b/api/app/repositories/object_references.py @@ -17,7 +17,7 @@ def create_object_reference( ref_doc = { "uuid": ref_uuid, - "object_id": object_reference.object_id, + "object_uuid": str(object_reference.object_uuid) if object_reference.object_uuid else None, "event_uuid": str(object_reference.event_uuid) if object_reference.event_uuid else None, "source_uuid": str(object_reference.source_uuid) if object_reference.source_uuid else None, "referenced_uuid": str(object_reference.referenced_uuid) if object_reference.referenced_uuid else None, diff --git a/api/app/repositories/objects.py b/api/app/repositories/objects.py index 1862d776..93000ffb 100644 --- a/api/app/repositories/objects.py +++ b/api/app/repositories/objects.py @@ -1,7 +1,7 @@ import logging import math import time -from typing import Optional, Union +from typing import Optional from uuid import UUID, uuid4 from app.models import feed as feed_models @@ -85,7 +85,11 @@ def get_objects_from_opensearch( "sort": [{"timestamp": {"order": "desc"}}], } - response = client.search(index="misp-objects", body=query_body) + try: + response = client.search(index="misp-objects", body=query_body) + except NotFoundError: + return Page(items=[], total=0, page=params.page, size=params.size, pages=0) + total = response["hits"]["total"]["value"] hits = response["hits"]["hits"] @@ -128,25 +132,15 @@ def get_objects_from_opensearch( def get_object_from_opensearch( - object_id: Union[int, UUID], + object_uuid: UUID, ) -> Optional[object_schemas.Object]: client = get_opensearch_client() - if isinstance(object_id, int): - response = client.search( - index="misp-objects", - body={"query": {"term": {"id": object_id}}, "size": 1}, - ) - hits = response["hits"]["hits"] - if not hits: - return None - source = hits[0]["_source"] - else: - try: - doc = client.get(index="misp-objects", id=str(object_id)) - source = doc["_source"] - except NotFoundError: - return None + try: + doc = client.get(index="misp-objects", id=str(object_uuid)) + source = doc["_source"] + except NotFoundError: + return None obj_uuid = str(source.get("uuid", "")) attr_response = client.search( @@ -166,14 +160,25 @@ def get_object_from_opensearch( return object_schemas.Object.model_validate(source) -def get_object_by_id(db: Session, object_id: int): - return get_object_from_opensearch(object_id) - - def get_object_by_uuid(db: Session, object_uuid: UUID): return get_object_from_opensearch(object_uuid) +def get_objects( + db: Session, + event_uuid=None, + deleted: bool = False, + template_uuid: list = None, +) -> Page[object_schemas.Object]: + params = Params(page=1, size=100) + return get_objects_from_opensearch( + params, + event_uuid=str(event_uuid) if event_uuid else None, + deleted=deleted, + template_uuid=template_uuid, + ) + + def create_object( db: Session, object: object_schemas.ObjectCreate ) -> object_schemas.Object: @@ -210,7 +215,6 @@ def create_object( built_attrs = [] for attr in (object.attributes or []): - attr.object_id = None attr.event_uuid = event_uuid attr_schema = attributes_repository.create_attribute(db, attr) client.update( @@ -325,7 +329,7 @@ def update_object_from_pulled_object( ) else: attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_attr, event_uuid, user + db, local_attribute, pulled_attr, user ) for uuid_to_delete in local_attr_uuids - pulled_attr_uuids: @@ -371,10 +375,10 @@ def update_object_from_pulled_object( def update_object( - db: Session, object_id: Union[int, UUID], object: object_schemas.ObjectUpdate + db: Session, object_uuid: UUID, object: object_schemas.ObjectUpdate ) -> object_schemas.Object: client = get_opensearch_client() - os_obj = get_object_from_opensearch(object_id) + os_obj = get_object_from_opensearch(object_uuid) if os_obj is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Object not found") @@ -390,7 +394,6 @@ def update_object( client.update(index="misp-objects", id=str(os_obj.uuid), body={"doc": patch}, refresh=True) for attr in (object.new_attributes or []): - attr.object_id = None attr.event_uuid = os_obj.event_uuid attr_schema = attributes_repository.create_attribute(db, attr) client.update( @@ -411,9 +414,9 @@ def update_object( return get_object_from_opensearch(os_obj.uuid) -def delete_object(db: Session, object_id: Union[int, UUID]) -> None: +def delete_object(db: Session, object_uuid: UUID) -> None: client = get_opensearch_client() - os_obj = get_object_from_opensearch(object_id) + os_obj = get_object_from_opensearch(object_uuid) if os_obj is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Object not found") diff --git a/api/app/repositories/reports.py b/api/app/repositories/reports.py index 81f416dd..62472198 100644 --- a/api/app/repositories/reports.py +++ b/api/app/repositories/reports.py @@ -1,5 +1,6 @@ from app.services.opensearch import get_opensearch_client from app.schemas import event as event_schemas +from opensearchpy.exceptions import NotFoundError import logging import uuid import time @@ -20,7 +21,10 @@ def get_event_reports_by_event_uuid(event_uuid: uuid.UUID): } } - response = OpenSearchClient.search(index="misp-event-reports", body=search_body) + try: + response = OpenSearchClient.search(index="misp-event-reports", body=search_body) + except NotFoundError: + return [] return response["hits"]["hits"] @@ -38,12 +42,10 @@ def create_event_report(event: event_schemas.Event, report: dict): "sharing_group_id": event.sharing_group_id, "name": report["name"], "content": report["content"], - "id": None, - "event_id": event.id, + "event_uuid": str(event.uuid), "timestamp": int(time.time()), "@timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), "deleted": False, - "event_uuid": str(event.uuid), } response = OpenSearchClient.index( diff --git a/api/app/repositories/servers.py b/api/app/repositories/servers.py index a5d802a7..6a0009b6 100644 --- a/api/app/repositories/servers.py +++ b/api/app/repositories/servers.py @@ -137,7 +137,7 @@ def pull_server_by_id_full( user: user_models.User, ): - # get a list of the event_ids on the server + # get a list of the event_uuids on the server event_uuids = get_event_uuids_from_server(server, remote_misp) # TODO apply MISP.enableEventBlocklisting / removeBlockedEvents @@ -164,9 +164,9 @@ def get_event_uuids_from_server(server: server_schemas.Server, remote_misp: PyMI timestamp = server.pull_rules.get("timestamp", None) events = remote_misp.search_index(minimal=True, published=True, timestamp=timestamp) - event_ids = [event["uuid"] for event in events] + event_uuids = [event["uuid"] for event in events] - return event_ids + return event_uuids def pull_event_by_uuid( @@ -384,7 +384,7 @@ def create_or_update_pulled_event( else: event.sharing_group_id = None - created = events_repository.create_event_from_pulled_event(db, event) + created = events_repository.create_event_from_pulled_event(event) if created: sync_repository.create_pulled_event_tags(db, created, event.tags, user) diff --git a/api/app/repositories/sync.py b/api/app/repositories/sync.py index 12ddb49a..bb38ebba 100644 --- a/api/app/repositories/sync.py +++ b/api/app/repositories/sync.py @@ -6,7 +6,6 @@ from app.models import tag as tag_models from app.schemas import event as event_schemas from app.models import user as user_models -from app.models import object as object_models from app.repositories import attributes as attributes_repository from app.repositories import objects as objects_repository from app.repositories import tags as tags_repository @@ -114,7 +113,7 @@ def create_pulled_event_attributes( def create_pulled_event_objects( db: Session, event_uuid: str, - objects: list[object_models.Object], + objects: list[MISPObject], user: user_models.User, ): for object in objects: @@ -161,5 +160,5 @@ def update_pulled_event_attributes( ) else: attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_attribute, event_uuid, user + db, local_attribute, pulled_attribute, user ) diff --git a/api/app/repositories/tags.py b/api/app/repositories/tags.py index 369d9644..8b88d597 100644 --- a/api/app/repositories/tags.py +++ b/api/app/repositories/tags.py @@ -141,12 +141,12 @@ def untag_attribute( try: doc = client.get(index="misp-attributes", id=attr_uuid) except NotFoundError: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="AttributeTag not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") current_tags = doc["_source"].get("tags", []) new_tags = [t for t in current_tags if t.get("name") != tag.name] if len(new_tags) == len(current_tags): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="AttributeTag not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tag not found") client.update( index="misp-attributes", @@ -194,12 +194,12 @@ def untag_event( try: doc = client.get(index="misp-events", id=event_uuid) except NotFoundError: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="EventTag not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") current_tags = doc["_source"].get("tags", []) new_tags = [t for t in current_tags if t.get("name") != tag.name] if len(new_tags) == len(current_tags): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="EventTag not found") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tag not found") client.update( index="misp-events", diff --git a/api/app/routers/attributes.py b/api/app/routers/attributes.py index 527521e7..f168994f 100644 --- a/api/app/routers/attributes.py +++ b/api/app/routers/attributes.py @@ -19,13 +19,13 @@ async def get_attributes_parameters( event_uuid: Optional[str] = None, deleted: Optional[bool] = None, - object_id: Optional[int] = None, + object_uuid: Optional[UUID] = None, type: Optional[str] = None, ): return { "event_uuid": event_uuid, "deleted": deleted, - "object_id": object_id, + "object_uuid": object_uuid, "type": type, } @@ -42,7 +42,7 @@ def get_attributes( page_params, params["event_uuid"], params["deleted"], - params["object_id"], + params["object_uuid"], params["type"], ) @@ -105,16 +105,13 @@ def create_attribute( get_current_active_user, scopes=["attributes:create"] ), ) -> attribute_schemas.Attribute: - if attribute.event_uuid is None and attribute.event_id is None: + if attribute.event_uuid is None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail="Event ID or UUID must be provided", + detail="Event UUID must be provided", ) - if attribute.event_uuid is None: - event = events_repository.get_event_from_opensearch(attribute.event_id) - else: - event = events_repository.get_event_from_opensearch(attribute.event_uuid) + event = events_repository.get_event_from_opensearch(attribute.event_uuid) if event is None: raise HTTPException( @@ -135,7 +132,7 @@ def update_attribute( ), ) -> attribute_schemas.Attribute: return attributes_repository.update_attribute( - db=db, attribute_id=attribute_uuid, attribute=attribute + db=db, attribute_uuid=attribute_uuid, attribute=attribute ) @@ -147,7 +144,7 @@ def delete_attribute( get_current_active_user, scopes=["attributes:delete"] ), ): - attributes_repository.delete_attribute(db=db, attribute_id=attribute_uuid) + attributes_repository.delete_attribute(db=db, attribute_uuid=attribute_uuid) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/api/app/routers/events.py b/api/app/routers/events.py index 15192bcb..f3a55e8c 100644 --- a/api/app/routers/events.py +++ b/api/app/routers/events.py @@ -89,6 +89,37 @@ async def export_events( ) +@router.post("/events/force-index", status_code=status.HTTP_202_ACCEPTED) +def force_index( + uuid: Optional[UUID] = Query(None), + db: Session = Depends(get_db), + user: user_schemas.User = Security( + get_current_active_user, scopes=["events:update"] + ), +): + from app.worker import tasks as _tasks + + if uuid is not None: + os_event = events_repository.get_event_from_opensearch(uuid) + if os_event is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" + ) + _tasks.handle_updated_event(str(os_event.uuid)) + return JSONResponse( + content={"message": f"Event {uuid} has been re-indexed."}, + status_code=status.HTTP_202_ACCEPTED, + ) + + uuids = events_repository.get_event_uuids_from_opensearch() + for event_uuid in uuids: + _tasks.handle_updated_event(str(event_uuid)) + return JSONResponse( + content={"message": "Indexing job dispatched for all events."}, + status_code=status.HTTP_202_ACCEPTED, + ) + + @router.get("/events/{event_uuid}", response_model=event_schemas.Event) def get_event_by_uuid( event_uuid: UUID, @@ -133,7 +164,7 @@ def update_event( get_current_active_user, scopes=["events:update"] ), ) -> event_schemas.Event: - return events_repository.update_event(db=db, event_id=event_uuid, event=event) + return events_repository.update_event(db=db, event_uuid=event_uuid, event=event) @router.delete("/events/{event_uuid}", status_code=status.HTTP_204_NO_CONTENT) @@ -145,7 +176,7 @@ def delete_event( get_current_active_user, scopes=["events:delete"] ), ): - return events_repository.delete_event(db=db, event_id=event_uuid, force=force) + return events_repository.delete_event(db=db, event_uuid=event_uuid, force=force) @router.post( diff --git a/api/app/routers/mcp.py b/api/app/routers/mcp.py index e4fd2fdf..7ecdc61a 100644 --- a/api/app/routers/mcp.py +++ b/api/app/routers/mcp.py @@ -379,7 +379,6 @@ def search_attributes(query: str, page: int = 1, size: int = 10) -> dict: - category (str): e.g. "Network activity", "Payload delivery", "External analysis", "Persistence" - event_uuid (str): UUID of parent event - - event_id (int): ID of parent event - to_ids (bool): whether this is an IDS-exportable indicator - comment (str): analyst comment - tags.name (str): associated tag names @@ -1094,9 +1093,13 @@ def search_event_reports( "sort": [{"@timestamp": {"order": "desc"}}], } - response = client.search(index="misp-event-reports", body=query_body) - hits = response["hits"]["hits"] - total = response["hits"]["total"]["value"] + from opensearchpy.exceptions import NotFoundError as _NotFoundError + try: + response = client.search(index="misp-event-reports", body=query_body) + hits = response["hits"]["hits"] + total = response["hits"]["total"]["value"] + except _NotFoundError: + hits, total = [], 0 results = [h["_source"] for h in hits] logger.debug( f"Event reports search returned {total} total, {len(results)} in page {page}" @@ -1478,7 +1481,7 @@ def query_syntax() -> str: "- attribute_count, object_count, tags.name\n\n" "## Attribute fields\n" "- value (default), type, category, uuid\n" - "- event_uuid, event_id, to_ids (bool)\n" + "- event_uuid, to_ids (bool)\n" "- comment, tags.name, deleted, disable_correlation\n" "- expanded.ip2geo.country_iso_code (GeoIP enrichment)\n" ) diff --git a/api/app/routers/objects.py b/api/app/routers/objects.py index 5b04c39a..d37f63c6 100644 --- a/api/app/routers/objects.py +++ b/api/app/routers/objects.py @@ -1,5 +1,5 @@ from uuid import UUID -from typing import Optional, Union +from typing import Optional from app.auth.security import get_current_active_user from app.db.session import get_db @@ -39,14 +39,14 @@ def get_objects( ) -@router.get("/objects/{object_id}", response_model=object_schemas.Object) +@router.get("/objects/{object_uuid}", response_model=object_schemas.Object) def get_object_by_id( - object_id: Union[int, UUID], + object_uuid: UUID, user: user_schemas.User = Security( get_current_active_user, scopes=["objects:read"] ), ): - os_object = objects_repository.get_object_from_opensearch(object_id) + os_object = objects_repository.get_object_from_opensearch(object_uuid) if os_object is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" @@ -82,26 +82,26 @@ def create_object( return objects_repository.create_object(db=db, object=object) -@router.patch("/objects/{object_id}", response_model=object_schemas.Object) +@router.patch("/objects/{object_uuid}", response_model=object_schemas.Object) def update_object( - object_id: Union[int, UUID], + object_uuid: UUID, object: object_schemas.ObjectUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:update"] ), ): - return objects_repository.update_object(db=db, object_id=object_id, object=object) + return objects_repository.update_object(db=db, object_uuid=object_uuid, object=object) -@router.delete("/objects/{object_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/objects/{object_uuid}", status_code=status.HTTP_204_NO_CONTENT) def delete_object( - object_id: Union[int, UUID], + object_uuid: UUID, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:delete"] ), ): - objects_repository.delete_object(db=db, object_id=object_id) + objects_repository.delete_object(db=db, object_uuid=object_uuid) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/api/app/schemas/attribute.py b/api/app/schemas/attribute.py index 1573dcbe..23eca85d 100644 --- a/api/app/schemas/attribute.py +++ b/api/app/schemas/attribute.py @@ -11,7 +11,7 @@ class AttributeBase(BaseModel): - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None event_uuid: Optional[UUID] = None object_relation: Optional[str] = None category: str @@ -42,7 +42,7 @@ def to_misp_format(self) -> dict: attr_json = { "id": None, - "object_id": self.object_id, + "object_uuid": str(self.object_uuid) if self.object_uuid else None, "object_relation": self.object_relation, "category": self.category, "type": self.type, @@ -74,7 +74,7 @@ class AttributeCreate(AttributeBase): class AttributeUpdate(BaseModel): - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None object_relation: Optional[str] = None category: Optional[str] = None type: Optional[str] = None diff --git a/api/app/schemas/event.py b/api/app/schemas/event.py index 4f28b6c2..ddd994a7 100644 --- a/api/app/schemas/event.py +++ b/api/app/schemas/event.py @@ -42,7 +42,7 @@ class Event(EventBase): objects: list[Object] = [] sharing_group: Optional[SharingGroup] = None tags: list[Tag] = [] - organisation: Organisation + organisation: Optional[Organisation] = None model_config = ConfigDict(from_attributes=True) diff --git a/api/app/schemas/object_reference.py b/api/app/schemas/object_reference.py index c4eea0ef..eb0c51ba 100644 --- a/api/app/schemas/object_reference.py +++ b/api/app/schemas/object_reference.py @@ -12,7 +12,7 @@ class ReferencedType(enum.Enum): class ObjectReferenceBase(BaseModel): uuid: UUID - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None event_uuid: Optional[UUID] = None source_uuid: Optional[UUID] = None referenced_uuid: Optional[UUID] = None @@ -44,7 +44,7 @@ def to_misp_format(self) -> dict: "id": None, "uuid": str(self.uuid), "timestamp": self.timestamp, - "object_id": self.object_id, + "object_uuid": str(self.object_uuid) if self.object_uuid else None, "event_uuid": str(self.event_uuid) if self.event_uuid else None, "source_uuid": str(self.source_uuid) if self.source_uuid else None, "referenced_uuid": str(self.referenced_uuid) if self.referenced_uuid else None, @@ -62,7 +62,7 @@ class ObjectReferenceCreate(ObjectReferenceBase): class ObjectReferenceUpdate(ObjectReferenceBase): - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None source_uuid: Optional[UUID] = None referenced_uuid: Optional[UUID] = None timestamp: Optional[int] = None diff --git a/api/app/schemas/tag.py b/api/app/schemas/tag.py index c73c833f..17e3ef2c 100644 --- a/api/app/schemas/tag.py +++ b/api/app/schemas/tag.py @@ -38,24 +38,3 @@ class TagUpdate(TagBase): local_only: Optional[bool] = None -class AttributeTagBase(BaseModel): - attribute_id: int - event_id: int - tag_id: int - local: bool - - -class AttributeTag(AttributeTagBase): - id: int - model_config = ConfigDict(from_attributes=True) - - -class EventTagBase(BaseModel): - event_id: int - tag_id: int - local: bool - - -class EventTag(EventTagBase): - id: int - model_config = ConfigDict(from_attributes=True) diff --git a/api/app/tests/api/test_events.py b/api/app/tests/api/test_events.py index 2a4155c6..19947fe9 100644 --- a/api/app/tests/api/test_events.py +++ b/api/app/tests/api/test_events.py @@ -626,38 +626,6 @@ def test_force_index_by_uuid( assert response.status_code == status.HTTP_202_ACCEPTED assert str(event_1.uuid) in data["message"] - @pytest.mark.parametrize("scopes", [["events:update"]]) - def test_force_index_by_id( - self, - client: TestClient, - event_1: object, - auth_token: auth.Token, - ): - response = client.post( - "/events/force-index", - params={"id": event_1.id}, - headers={"Authorization": "Bearer " + auth_token}, - ) - data = response.json() - - assert response.status_code == status.HTTP_202_ACCEPTED - assert str(event_1.uuid) in data["message"] - - @pytest.mark.parametrize("scopes", [["events:update"]]) - def test_force_index_by_id_not_found( - self, - client: TestClient, - auth_token: auth.Token, - ): - response = client.post( - "/events/force-index", - params={"id": 999999}, - headers={"Authorization": "Bearer " + auth_token}, - ) - - assert response.status_code == status.HTTP_404_NOT_FOUND - assert response.json()["detail"] == "Event not found" - @pytest.mark.parametrize("scopes", [["events:update"]]) def test_force_index_all_events( self, diff --git a/api/app/tests/api_tester.py b/api/app/tests/api_tester.py index cc56fa87..1c11eb94 100644 --- a/api/app/tests/api_tester.py +++ b/api/app/tests/api_tester.py @@ -66,8 +66,6 @@ def teardown_db(self, db: Session): db.query(galaxy_models.GalaxyCluster).delete(synchronize_session=False) db.query(galaxy_models.Galaxy).delete(synchronize_session=False) db.query(feed_models.Feed).delete(synchronize_session=False) - db.query(tag_models.AttributeTag).delete(synchronize_session=False) - db.query(tag_models.EventTag).delete(synchronize_session=False) db.query(tag_models.Tag).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupOrganisation).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupServer).delete(synchronize_session=False) diff --git a/api/app/tests/repositories/test_feeds.py b/api/app/tests/repositories/test_feeds.py index 71779bf1..62afe169 100644 --- a/api/app/tests/repositories/test_feeds.py +++ b/api/app/tests/repositories/test_feeds.py @@ -8,6 +8,7 @@ from app.repositories import attributes as attributes_repository from app.repositories import events as events_repository from app.repositories import feeds as feeds_repository +from app.repositories import object_references as object_references_repository from app.repositories import objects as objects_repository from app.tests.api_tester import ApiTester from app.tests.scenarios import feed_fetch_scenarios @@ -66,7 +67,6 @@ def test_fetch_feed_by_id_new_event( assert obj is not None # check the object references were created - from app.repositories import object_references as object_references_repository object_reference = object_references_repository.get_object_reference_by_uuid( db, UUID("d7e57f39-4dd5-4b87-b040-75561fa8289e") ) @@ -77,26 +77,23 @@ def test_fetch_feed_by_id_new_event( assert len(tags) == 4 # check the event tags were created - event_tags = ( - db.query(tag_models.Tag) - .join(tag_models.EventTag) - .filter( - tag_models.Tag.name.in_(["type:OSINT", "tlp:clear", "tlp:white"]), - ) - .all() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_new_event["Event"]["uuid"]) ) - assert len(event_tags) == 3 + event_tag_names = {t.name for t in (os_event.tags or [])} + assert {"type:OSINT", "tlp:clear", "tlp:white"}.issubset(event_tag_names) # check the attribute tags were created - attribute_tags = ( - db.query(tag_models.Tag) - .join(tag_models.AttributeTag) - .filter( - tag_models.Tag.name.in_(["tlp:red"]), - ) - .all() - ) - assert len(attribute_tags) == 1 + all_attribute_tag_names = set() + for uuid in [ + "317e63e6-b95d-4dd1-b4fd-de2f64f33fd8", + "8be7a04d-c10b-4ef6-854f-2072e67f6cd5", + ]: + attr = attributes_repository.get_attribute_from_opensearch(UUID(uuid)) + if attr: + for t in (attr.tags or []): + all_attribute_tag_names.add(t.name) + assert "tlp:red" in all_attribute_tag_names def test_fetch_feed_by_id_existing_event( self, @@ -104,7 +101,7 @@ def test_fetch_feed_by_id_existing_event( feed_1: feed_models.Feed, event_1, attribute_1, - object_1: object_models.Object, + object_1, object_attribute_1, user_1: user_models.User, ): @@ -148,12 +145,8 @@ def test_fetch_feed_by_id_existing_event( assert attribute.timestamp == 1577836801 # check the object was updated - object = ( - db.query(object_models.Object) - .filter( - object_models.Object.uuid == "90e06ef6-26f8-40dd-9fb7-75897445e2a0" - ) - .first() + object = objects_repository.get_object_from_opensearch( + UUID("90e06ef6-26f8-40dd-9fb7-75897445e2a0") ) assert object.comment == "Object comment updated by Feed fetch" assert object.timestamp == 1577836801 @@ -167,13 +160,8 @@ def test_fetch_feed_by_id_existing_event( assert object_attribute.timestamp == 1577836801 # check the object references were created - object_reference = ( - db.query(object_reference_models.ObjectReference) - .filter( - object_reference_models.ObjectReference.uuid - == "4d4c12b9-e514-496e-a8a6-06d5c6815b97" - ) - .first() + object_reference = object_references_repository.get_object_reference_by_uuid( + db, UUID("4d4c12b9-e514-496e-a8a6-06d5c6815b97") ) assert ( str(object_reference.referenced_uuid) @@ -181,28 +169,21 @@ def test_fetch_feed_by_id_existing_event( ) # check the event tags were created - event_tags = ( - db.query(tag_models.Tag) - .join(tag_models.EventTag) - .filter( - tag_models.Tag.name.in_(["EVENT_FEED_ADDED_TAG"]), - ) - .all() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_update_event["Event"]["uuid"]) ) - assert len(event_tags) == 1 + event_tag_names = {t.name for t in (os_event.tags or [])} + assert "EVENT_FEED_ADDED_TAG" in event_tag_names # check the attribute tags were created - attribute_tags = ( - db.query(tag_models.Tag) - .join(tag_models.AttributeTag) - .filter( - tag_models.Tag.name.in_( - [ - "ATTRIBUTE_EVENT_FEED_ADDED_TAG", - "OBJECT_ATTRIBUTE_EVENT_FEED_ADDED_TAG", - ] - ), - ) - .all() - ) - assert len(attribute_tags) == 2 + all_attribute_tag_names = set() + for uuid in [ + "7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0", + "011aca4f-eaf0-4a06-8133-b69f3806cbe8", + ]: + attr = attributes_repository.get_attribute_from_opensearch(UUID(uuid)) + if attr: + for t in (attr.tags or []): + all_attribute_tag_names.add(t.name) + assert "ATTRIBUTE_EVENT_FEED_ADDED_TAG" in all_attribute_tag_names + assert "OBJECT_ATTRIBUTE_EVENT_FEED_ADDED_TAG" in all_attribute_tag_names diff --git a/api/app/tests/repositories/test_servers.py b/api/app/tests/repositories/test_servers.py index 5a9dc45d..c1f4ff7f 100644 --- a/api/app/tests/repositories/test_servers.py +++ b/api/app/tests/repositories/test_servers.py @@ -3,9 +3,9 @@ import pytest from uuid import UUID -from app.models import object as object_models from app.repositories import attributes as attributes_repository -from app.models import object_reference as object_reference_models +from app.repositories import object_references as object_references_repository +from app.repositories import objects as objects_repository from app.models import organisation as organisations_models from app.models import server as server_models from app.models import sharing_groups as sharing_groups_models @@ -35,8 +35,6 @@ def test_pull_server_by_id( scenario: dict, ): # clear the database - db.query(tag_models.AttributeTag).delete() - db.query(tag_models.EventTag).delete() db.query(tag_models.Tag).delete() # mock remote MISP API calls @@ -88,27 +86,19 @@ def test_pull_server_by_id( ) # check the objects were created - objects = ( - db.query(object_models.Object) - .filter( - object_models.Object.uuid.in_( - scenario["expected_result"]["object_uuids"] - ) - ) - .all() - ) + objects = [ + objects_repository.get_object_from_opensearch(UUID(uuid)) + for uuid in scenario["expected_result"]["object_uuids"] + ] + objects = [o for o in objects if o is not None] assert len(objects) == len(scenario["expected_result"]["object_uuids"]) # check the object references were created - object_references = ( - db.query(object_reference_models.ObjectReference) - .filter( - object_reference_models.ObjectReference.uuid.in_( - scenario["expected_result"]["object_reference_uuids"] - ) - ) - .all() - ) + object_references = [ + object_references_repository.get_object_reference_by_uuid(db, UUID(uuid)) + for uuid in scenario["expected_result"]["object_reference_uuids"] + ] + object_references = [r for r in object_references if r is not None] assert len(object_references) == len( scenario["expected_result"]["object_reference_uuids"] ) @@ -147,22 +137,18 @@ def test_pull_server_by_id( assert len(tags) == len(scenario["expected_result"]["tags"]) # check the event tags were created - event_tags = ( - db.query(tag_models.Tag) - .join(tag_models.EventTag) - .filter( - tag_models.Tag.name.in_(scenario["expected_result"]["event_tags"]) - ) - .all() - ) - assert len(event_tags) == len(scenario["expected_result"]["event_tags"]) + event_tag_names = set() + for event in os_events: + for t in (event.tags or []): + event_tag_names.add(t.name) + for tag_name in scenario["expected_result"]["event_tags"]: + assert tag_name in event_tag_names # check the attribute tags were created + all_attribute_tag_names = set() + for attr in attributes: + for t in (attr.tags or []): + all_attribute_tag_names.add(t.name) for attribute_tag in scenario["expected_result"]["attribute_tags"]: - attribute_tags = ( - db.query(tag_models.Tag) - .join(tag_models.AttributeTag) - .filter(tag_models.Tag.name.in_(attribute_tag["tags"])) - .all() - ) - assert len(attribute_tags) == len(attribute_tag["tags"]) + for tag_name in attribute_tag["tags"]: + assert tag_name in all_attribute_tag_names diff --git a/api/app/worker/tasks.py b/api/app/worker/tasks.py index 1f5dc600..b2b639d0 100644 --- a/api/app/worker/tasks.py +++ b/api/app/worker/tasks.py @@ -183,10 +183,10 @@ def handle_deleted_event(event_uuid: str): @celery_app.task -def handle_created_attribute(attribute_uuid: str, object_id, event_uuid: str | None): +def handle_created_attribute(attribute_uuid: str, object_uuid, event_uuid: str | None): logger.info("handling created attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - if object_id is None and event_uuid: + if object_uuid is None and event_uuid: events_repository.increment_attribute_count(db, event_uuid) os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) @@ -197,7 +197,7 @@ def handle_created_attribute(attribute_uuid: str, object_id, event_uuid: str | N @celery_app.task -def handle_updated_attribute(attribute_uuid: str, object_id, event_uuid: str | None): +def handle_updated_attribute(attribute_uuid: str, object_uuid, event_uuid: str | None): logger.info("handling updated attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) @@ -208,10 +208,10 @@ def handle_updated_attribute(attribute_uuid: str, object_id, event_uuid: str | N @celery_app.task -def handle_deleted_attribute(attribute_uuid: str, object_id, event_uuid: str | None): +def handle_deleted_attribute(attribute_uuid: str, object_uuid, event_uuid: str | None): logger.info("handling deleted attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - if object_id is None and event_uuid: + if object_uuid is None and event_uuid: events_repository.decrement_attribute_count(db, event_uuid) os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) @@ -691,7 +691,7 @@ def delete_indexed_event(event_uuid: str): # delete indexed objects response = OpenSearchClient.delete_by_query( - index="misp-objects", body=query, refresh=True + index="misp-objects", body=query, refresh=True, ignore=[404] ) logger.info( "deleted %s indexed objects for event uuid=%s", @@ -710,13 +710,14 @@ def index_attribute(attribute_uuid: str): OpenSearchClient = get_opensearch_client() - # Preserve event_uuid and tags from existing indexed doc (event_id FK removed from SQL) existing_event_uuid = None existing_tags = [] + existing_object_uuid = None try: existing_doc = OpenSearchClient.get(index="misp-attributes", id=attribute_uuid) existing_event_uuid = existing_doc["_source"].get("event_uuid") existing_tags = existing_doc["_source"].get("tags", []) + existing_object_uuid = existing_doc["_source"].get("object_uuid") except Exception: pass @@ -726,14 +727,10 @@ def index_attribute(attribute_uuid: str): raise Exception("Attribute with uuid=%s not found", attribute_uuid) attribute = event_schemas.Attribute.model_validate(db_attribute) - object_uuid = None - if db_attribute.object_id: - db_object = objects_repository.get_object_by_id(db, db_attribute.object_id) - object_uuid = db_object.uuid if db_object else None attribute_raw = attribute.model_dump() attribute_raw["event_uuid"] = existing_event_uuid - attribute_raw["object_uuid"] = str(object_uuid) if object_uuid else None + attribute_raw["object_uuid"] = existing_object_uuid attribute_raw["tags"] = existing_tags # convert timestamp to datetime so it can be indexed @@ -835,7 +832,6 @@ def index_object(object_uuid: str): OpenSearchClient = get_opensearch_client() - # Preserve event_uuid from existing indexed doc (event_id FK was removed from SQL) existing_event_uuid = None try: existing_doc = OpenSearchClient.get(index="misp-objects", id=object_uuid) diff --git a/docs/features/enrichments.md b/docs/features/enrichments.md index 677fb633..72c8ce2c 100644 --- a/docs/features/enrichments.md +++ b/docs/features/enrichments.md @@ -82,7 +82,7 @@ Click **query** on an enabled module to send a test query directly from the sett 6. Use the checkboxes to select which results to keep, or click **Select All Enrichments**. 7. Click **Add** to create the selected attributes and objects in the event. -New attributes inherit the source attribute's `event_id`, `distribution`, and `sharing_group_id`. +New attributes inherit the source attribute's `event_uuid`, `distribution`, and `sharing_group_id`. ## API reference diff --git a/frontend/src/components/attachments/AttachmentIcon.vue b/frontend/src/components/attachments/AttachmentIcon.vue index c989a487..4e522b17 100644 --- a/frontend/src/components/attachments/AttachmentIcon.vue +++ b/frontend/src/components/attachments/AttachmentIcon.vue @@ -206,7 +206,7 @@ async function downloadAttachment() { :id="`deleteObjectModal_${attachment.id}`" @object-deleted="handleObjectDeleted" :modal="deleteObjectModal" - :object_id="attachment.id" + :object_uuid="attachment.uuid" /> diff --git a/frontend/src/components/attachments/UploadAttachmentsWidget.vue b/frontend/src/components/attachments/UploadAttachmentsWidget.vue index b538ff15..1d2c5008 100644 --- a/frontend/src/components/attachments/UploadAttachmentsWidget.vue +++ b/frontend/src/components/attachments/UploadAttachmentsWidget.vue @@ -86,8 +86,8 @@ function handleAttachmentDeleted(attachment_id) { ); } -function handleObjectDeleted(object_id) { - emit("object-deleted", object_id); +function handleObjectDeleted(object_uuid) { + emit("object-deleted", object_uuid); } diff --git a/frontend/src/components/attributes/AttributeActions.vue b/frontend/src/components/attributes/AttributeActions.vue index fcbcb8b9..f676d6c2 100644 --- a/frontend/src/components/attributes/AttributeActions.vue +++ b/frontend/src/components/attributes/AttributeActions.vue @@ -279,11 +279,11 @@ function followAttribute() { diff --git a/frontend/src/components/attributes/DeleteAttributeModal.vue b/frontend/src/components/attributes/DeleteAttributeModal.vue index 0e20ebb4..72ba8682 100644 --- a/frontend/src/components/attributes/DeleteAttributeModal.vue +++ b/frontend/src/components/attributes/DeleteAttributeModal.vue @@ -5,14 +5,14 @@ import { storeToRefs } from "pinia"; const attributesStore = useAttributesStore(); const { status } = storeToRefs(attributesStore); -const props = defineProps(["attribute_id", "modal"]); +const props = defineProps(["attribute_uuid", "modal"]); const emit = defineEmits(["attribute-deleted"]); function deleteAttribute() { return attributesStore - .delete(props.attribute_id) + .delete(props.attribute_uuid) .then(() => { - emit("attribute-deleted", { attribute_id: props.attribute_id }); + emit("attribute-deleted", { attribute_uuid: props.attribute_uuid }); props.modal.hide(); }) .catch((error) => (status.error = error)); @@ -20,11 +20,11 @@ function deleteAttribute() {