diff --git a/api/README.md b/api/README.md index c87ec8d6..91ee7ab5 100644 --- a/api/README.md +++ b/api/README.md @@ -209,7 +209,7 @@ Example: ``` from app.worker import tasks -tasks.handle_created_attribute.delay(pulled_attribute.id, pulled_attribute.object_id, pulled_attribute.event_id) +tasks.handle_created_attribute.delay(pulled_attribute.id, pulled_attribute.object_uuid, pulled_attribute.event_uuid) ``` If you add a new task, you have to restart the celery `worker` container, otherwise you will get `NotRegistered('app.worker.tasks.new_task') ` exception. diff --git a/api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py b/api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py new file mode 100644 index 00000000..52221019 --- /dev/null +++ b/api/alembic/versions/c8d7e6f5a4b3_detach_object_references_fks.py @@ -0,0 +1,44 @@ +"""detach object_references FK constraints to events and objects + +These are the last DB-level FK constraints that reference the events and objects +tables from an external table, blocking their eventual removal. + +Revision ID: c8d7e6f5a4b3 +Revises: f3e2d1c0b9a8 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c8d7e6f5a4b3" +down_revision = "f3e2d1c0b9a8" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint( + "object_references_event_id_fkey", "object_references", type_="foreignkey" + ) + op.drop_constraint( + "object_references_object_id_fkey", "object_references", type_="foreignkey" + ) + + +def downgrade(): + op.create_foreign_key( + "object_references_event_id_fkey", + "object_references", + "events", + ["event_id"], + ["id"], + ) + op.create_foreign_key( + "object_references_object_id_fkey", + "object_references", + "objects", + ["object_id"], + ["id"], + ) diff --git a/api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py b/api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py new file mode 100644 index 00000000..acaece1b --- /dev/null +++ b/api/alembic/versions/d4e5f6a7b8c9_detach_attributes_objects_from_events.py @@ -0,0 +1,30 @@ +"""remove event_id columns from attributes and objects tables + +Revision ID: d4e5f6a7b8c9 +Revises: c8d7e6f5a4b3 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +from alembic import op + +revision = "d4e5f6a7b8c9" +down_revision = "c8d7e6f5a4b3" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_index("ix_attributes_event_id", table_name="attributes", if_exists=True) + op.drop_column("attributes", "event_id") + op.drop_index("ix_objects_event_id", table_name="objects", if_exists=True) + op.drop_column("objects", "event_id") + + +def downgrade(): + import sqlalchemy as sa + + op.add_column("objects", sa.Column("event_id", sa.Integer(), nullable=True)) + op.create_index("ix_objects_event_id", "objects", ["event_id"]) + op.add_column("attributes", sa.Column("event_id", sa.Integer(), nullable=True)) + op.create_index("ix_attributes_event_id", "attributes", ["event_id"]) diff --git a/api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py b/api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py new file mode 100644 index 00000000..cc8e8d53 --- /dev/null +++ b/api/alembic/versions/e1a2b3c4d5f6_drop_events_table.py @@ -0,0 +1,75 @@ +"""drop events table and migrate feeds.event_id to uuid string + +Revision ID: e1a2b3c4d5f6 +Revises: d4e5f6a7b8c9 +Create Date: 2026-03-26 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "e1a2b3c4d5f6" +down_revision = "d4e5f6a7b8c9" +branch_labels = None +depends_on = None + + +def upgrade(): + # Replace event_id (INTEGER) with event_uuid (VARCHAR) in object_references + op.drop_column("object_references", "event_id") + op.add_column("object_references", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + + # Replace event_id (INTEGER) with event_uuid (VARCHAR) in event_tags and attribute_tags + op.drop_column("event_tags", "event_id") + op.add_column("event_tags", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + op.drop_column("attribute_tags", "event_id") + op.add_column("attribute_tags", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + + # Rename feeds.event_id (INTEGER) to feeds.event_uuid (VARCHAR for UUID string) + op.drop_column("feeds", "event_id") + op.add_column("feeds", sa.Column("event_uuid", postgresql.UUID(as_uuid=True), nullable=True)) + + # Drop the events table (all FK constraints were removed in prior migrations) + op.drop_table("events") + + +def downgrade(): + # Recreate events table (minimal schema for rollback) + op.create_table( + "events", + sa.Column("id", sa.Integer(), nullable=False, autoincrement=True), + sa.Column("org_id", sa.Integer(), nullable=False), + sa.Column("date", sa.Date(), nullable=False), + sa.Column("info", sa.String(), nullable=True), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("published", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("attribute_count", sa.Integer(), nullable=True), + sa.Column("object_count", sa.Integer(), nullable=True), + sa.Column("orgc_id", sa.Integer(), nullable=False), + sa.Column("timestamp", sa.Integer(), nullable=False, server_default="0"), + sa.Column("sharing_group_id", sa.Integer(), nullable=True), + sa.Column("proposal_email_lock", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("locked", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("publish_timestamp", sa.Integer(), nullable=False, server_default="0"), + sa.Column("sighting_timestamp", sa.Integer(), nullable=True), + sa.Column("disable_correlation", sa.Boolean(), nullable=True), + sa.Column("extends_uuid", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("protected", sa.Boolean(), nullable=False, server_default="false"), + sa.Column("deleted", sa.Boolean(), nullable=False, server_default="false"), + sa.PrimaryKeyConstraint("id"), + ) + + # Revert feeds.event_uuid back to event_id INTEGER + op.drop_column("feeds", "event_uuid") + op.add_column("feeds", sa.Column("event_id", sa.Integer(), nullable=True)) + + # Revert object_references, event_tags, attribute_tags + op.drop_column("object_references", "event_uuid") + op.add_column("object_references", sa.Column("event_id", sa.Integer(), nullable=False)) + op.drop_column("event_tags", "event_uuid") + op.add_column("event_tags", sa.Column("event_id", sa.Integer(), nullable=True)) + op.drop_column("attribute_tags", "event_uuid") + op.add_column("attribute_tags", sa.Column("event_id", sa.Integer(), nullable=True)) diff --git a/api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py b/api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py new file mode 100644 index 00000000..8dc54593 --- /dev/null +++ b/api/alembic/versions/f3e2d1c0b9a8_detach_event_attribute_fks_from_tag_tables.py @@ -0,0 +1,61 @@ +"""detach event/attribute FK constraints from event_tags and attribute_tags + +These FK constraints block dropping the events and attributes tables. +The tag junction tables will remain, referencing only the tags table. + +Revision ID: f3e2d1c0b9a8 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-25 00:00:00.000000 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f3e2d1c0b9a8" +down_revision = "a1b2c3d4e5f6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint( + "event_tags_event_id_fkey", "event_tags", type_="foreignkey" + ) + op.drop_constraint( + "attribute_tags_attribute_id_fkey", "attribute_tags", type_="foreignkey" + ) + op.drop_constraint( + "attribute_tags_event_id_fkey", "attribute_tags", type_="foreignkey" + ) + # Allow NULLs now that FK constraints are removed (events/attributes may not exist in SQL) + op.alter_column("event_tags", "event_id", nullable=True) + op.alter_column("attribute_tags", "event_id", nullable=True) + op.alter_column("attribute_tags", "attribute_id", nullable=True) + + +def downgrade(): + op.alter_column("attribute_tags", "attribute_id", nullable=False) + op.alter_column("attribute_tags", "event_id", nullable=False) + op.alter_column("event_tags", "event_id", nullable=False) + op.create_foreign_key( + "event_tags_event_id_fkey", + "event_tags", + "events", + ["event_id"], + ["id"], + ) + op.create_foreign_key( + "attribute_tags_attribute_id_fkey", + "attribute_tags", + "attributes", + ["attribute_id"], + ["id"], + ) + op.create_foreign_key( + "attribute_tags_event_id_fkey", + "attribute_tags", + "events", + ["event_id"], + ["id"], + ) diff --git a/api/app/models/__init__.py b/api/app/models/__init__.py index 517da763..531335b9 100644 --- a/api/app/models/__init__.py +++ b/api/app/models/__init__.py @@ -1,9 +1,5 @@ -from app.models.attribute import Attribute # noqa -from app.models.event import Event # noqa from app.models.hunt import Hunt # noqa from app.models.module import ModuleSettings # noqa -from app.models.object import Object # noqa -from app.models.object_reference import ObjectReference # noqa from app.models.organisation import Organisation # noqa from app.models.role import Role # noqa from app.models.server import Server # noqa @@ -12,7 +8,7 @@ SharingGroupOrganisation, SharingGroupServer, ) -from app.models.tag import AttributeTag, EventTag, Tag # noqa +from app.models.tag import Tag # noqa from app.models.user import User # noqa # fixes: sqlalchemy.exc.InvalidRequestError: When initializing mapper mapped class AAA->bbb, expression 'XXXX' failed to locate a name ('XXXX'). If this is a class name, consider adding this relationship() to the class after both dependent classes have been defined. diff --git a/api/app/models/attribute.py b/api/app/models/attribute.py deleted file mode 100644 index 9a54cb94..00000000 --- a/api/app/models/attribute.py +++ /dev/null @@ -1,86 +0,0 @@ -import uuid - -import logging -from app.database import Base -from app.models.event import DistributionLevel -from app.services.attachments import get_b64_attachment -from sqlalchemy import BigInteger, Boolean, Column, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship -from app.settings import Settings, get_settings - - -logger = logging.getLogger(__name__) - - -class Attribute(Base): - __tablename__ = "attributes" - - id = Column( - Integer, primary_key=True, index=True, autoincrement=True, nullable=False - ) - event_id = Column(Integer, ForeignKey("events.id"), index=True, nullable=False) - object_id = Column(Integer, ForeignKey("objects.id")) - object_relation = Column(String(255), index=True) - category = Column(String(255), index=True) - type = Column(String(100), index=True) - value = Column(String()) - to_ids = Column(Boolean, default=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - timestamp = Column(Integer, nullable=False, default=0) - distribution: Mapped[DistributionLevel] = mapped_column( - Enum(DistributionLevel, name="distribution_level"), - nullable=False, - default=DistributionLevel.INHERIT_EVENT, - ) - sharing_group_id = Column( - Integer, ForeignKey("sharing_groups.id"), index=True, nullable=True - ) - comment = Column(String()) - deleted = Column(Boolean, default=False) - disable_correlation = Column(Boolean, default=False) - first_seen = Column(BigInteger(), index=True) - last_seen = Column(BigInteger(), index=True) - event = relationship( - "Event", - lazy="joined", - viewonly=True - ) - tags = relationship("Tag", secondary="attribute_tags", lazy="subquery") - - def to_misp_format( - self, - settings: Settings = get_settings(), - ): - """Convert the Attribute to a MISP-compatible dictionary representation.""" - - attr_json = { - "id": self.id, - "event_id": self.event_id, - "object_id": self.object_id, - "object_relation": self.object_relation, - "category": self.category, - "type": self.type, - "value": self.value, - "to_ids": self.to_ids, - "uuid": str(self.uuid), - "timestamp": self.timestamp, - "distribution": self.distribution.value if self.distribution else DistributionLevel.INHERIT_EVENT, - "sharing_group_id": self.sharing_group_id if self.sharing_group_id else None, - "comment": self.comment, - "deleted": self.deleted, - "disable_correlation": self.disable_correlation, - "first_seen": self.first_seen, - "last_seen": self.last_seen, - "Tag": [tag.to_misp_format() for tag in self.tags], - } - - # if its a file attribute, we need to handle it differently - if self.type in ["malware-sample", "attachment"]: - try: - attr_json["data"] = get_b64_attachment(self.uuid, settings) - except Exception as e: - logger.error(f"Error storing attachment: {str(e)}") - print(f"Error fetching file from storage: {str(e)}") - - return attr_json diff --git a/api/app/models/event.py b/api/app/models/event.py index 926e4366..117a250f 100644 --- a/api/app/models/event.py +++ b/api/app/models/event.py @@ -1,10 +1,4 @@ import enum -import uuid - -from app.database import Base -from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship class DistributionLevel(enum.Enum): @@ -41,87 +35,3 @@ class AnalysisLevel(enum.Enum): COMPLETE = 2 -class Event(Base): - __tablename__ = "events" - - id = Column( - Integer, primary_key=True, index=True, autoincrement=True, nullable=False - ) - org_id = Column(Integer, ForeignKey("organisations.id"), index=True, nullable=False) - date = Column(Date, nullable=False) - info = Column(String, index=True) - user_id = Column(Integer, ForeignKey("users.id"), nullable=False) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - published = Column(Boolean, default=False, nullable=False) - analysis: Mapped[AnalysisLevel] = mapped_column( - Enum(AnalysisLevel, name="analysis_level"), - nullable=False, - default=AnalysisLevel.INITIAL, - ) - attribute_count = Column(Integer, default=0) - object_count = Column(Integer, default=0) - orgc_id = Column( - Integer, ForeignKey("organisations.id"), index=True, nullable=False - ) - timestamp = Column(Integer, nullable=False, default=0) - distribution: Mapped[DistributionLevel] = mapped_column( - Enum(DistributionLevel, name="distribution_level"), - nullable=False, - default=DistributionLevel.ORGANISATION_ONLY, - ) - sharing_group_id = Column( - Integer, ForeignKey("sharing_groups.id"), index=True, nullable=True - ) - proposal_email_lock = Column(Boolean, nullable=False, default=False) - locked = Column(Boolean, nullable=False, default=False) - threat_level: Mapped[ThreatLevel] = mapped_column( - Enum(ThreatLevel, name="threat_level"), - nullable=False, - default=ThreatLevel.UNDEFINED, - ) - publish_timestamp = Column(Integer, nullable=False, default=0) - sighting_timestamp = Column(Integer, nullable=True) - disable_correlation = Column(Boolean, default=False) - extends_uuid = Column(UUID(as_uuid=True), index=True, nullable=True) - protected = Column(Boolean, nullable=False, default=False) - deleted = Column(Boolean, nullable=False, default=False) - - attributes = relationship("Attribute", lazy="subquery", cascade="all, delete-orphan") - objects = relationship("Object", lazy="subquery", cascade="all, delete-orphan") - sharing_group = relationship("SharingGroup", lazy="joined") - tags = relationship("Tag", secondary="event_tags", lazy="joined") - organisation = relationship("Organisation", lazy="joined", uselist=False, foreign_keys=[org_id]) - - def to_misp_format(self): - """Convert the Event to a MISP-compatible dictionary representation.""" - - return { - "id": self.id, - "org_id": self.org_id, - "date": self.date.isoformat(), - "info": self.info, - "user_id": self.user_id, - "uuid": str(self.uuid), - "published": self.published, - "analysis": self.analysis.value, - "orgc_id": self.orgc_id, - "timestamp": self.timestamp, - "distribution": self.distribution.value, - "sharing_group_id": self.sharing_group_id, - "proposal_email_lock": self.proposal_email_lock, - "locked": self.locked, - "threat_level": self.threat_level.value, - "publish_timestamp": self.publish_timestamp, - "disable_correlation": self.disable_correlation, - "extends_uuid": str(self.extends_uuid) if self.extends_uuid else None, - "protected": self.protected, - "deleted": self.deleted, - "Attribute": [attribute.to_misp_format() for attribute in self.attributes if attribute.object_id == None and not attribute.deleted], - "Object": [obj.to_misp_format() for obj in self.objects if not obj.deleted], - "Tag": [tag.to_misp_format() for tag in self.tags], - "Organisation": { - "id": self.organisation.id, - "name": self.organisation.name, - "uuid": str(self.organisation.uuid), - } if self.organisation else None - } diff --git a/api/app/models/feed.py b/api/app/models/feed.py index 84482d41..2ab7258a 100644 --- a/api/app/models/feed.py +++ b/api/app/models/feed.py @@ -1,6 +1,7 @@ from app.database import Base from app.models.event import DistributionLevel from sqlalchemy import JSON, Boolean, Column, Enum, Float, ForeignKey, Integer, String +from sqlalchemy.dialects.postgresql import UUID as PG_UUID from sqlalchemy.orm import Mapped, mapped_column @@ -24,7 +25,7 @@ class Feed(Base): source_format = Column(String, nullable=False) fixed_event = Column(Boolean, nullable=False, default=False) delta_merge = Column(Boolean, nullable=False, default=False) - event_id = Column(Integer, ForeignKey("events.id"), nullable=True) + event_uuid = Column(PG_UUID(as_uuid=True), nullable=True) publish = Column(Boolean, nullable=False, default=False) override_ids = Column(Boolean, nullable=False, default=False) settings = Column(JSON, nullable=False, default={}) diff --git a/api/app/models/object.py b/api/app/models/object.py deleted file mode 100644 index 1ea4996d..00000000 --- a/api/app/models/object.py +++ /dev/null @@ -1,60 +0,0 @@ -import uuid - -from app.database import Base -from app.models.event import DistributionLevel -from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column, relationship - - -class Object(Base): - __tablename__ = "objects" - - id = Column(Integer, primary_key=True, index=True) - name = Column(String) - meta_category = Column(String) - description = Column(String) - template_uuid = Column(String) - template_version = Column(Integer, nullable=False) - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - timestamp = Column(Integer, nullable=False) - distribution: Mapped[DistributionLevel] = mapped_column( - Enum(DistributionLevel, name="distribution_level"), - nullable=False, - default=DistributionLevel.INHERIT_EVENT, - ) - sharing_group_id = Column(Integer, ForeignKey("sharing_groups.id")) - comment = Column(String) - deleted = Column(Boolean, nullable=False, default=False) - first_seen = Column(Integer) - last_seen = Column(Integer) - event = relationship( - "Event", - lazy="joined", - viewonly=True - ) - attributes = relationship("Attribute", lazy="subquery", cascade="all, delete-orphan") - object_references = relationship("ObjectReference", lazy="subquery", cascade="all, delete-orphan") - - def to_misp_format(self): - """Convert the Object to a MISP-compatible dictionary representation.""" - return { - "id": self.id, - "name": self.name, - "meta-category": self.meta_category, - "description": self.description if self.description else self.name, - "template_uuid": self.template_uuid, - "template_version": self.template_version, - "event_id": self.event_id, - "uuid": str(self.uuid), - "timestamp": self.timestamp, - "distribution": self.distribution.value if self.distribution else None, - "sharing_group_id": self.sharing_group_id if self.sharing_group_id else None, - "comment": self.comment, - "deleted": self.deleted, - "first_seen": self.first_seen, - "last_seen": self.last_seen, - "Attribute": [attribute.to_misp_format() for attribute in self.attributes], - "ObjectReference": [ref.to_misp_format() for ref in self.object_references], - } \ No newline at end of file diff --git a/api/app/models/object_reference.py b/api/app/models/object_reference.py deleted file mode 100644 index 0b5a186c..00000000 --- a/api/app/models/object_reference.py +++ /dev/null @@ -1,52 +0,0 @@ -import enum -import uuid - -from app.database import Base -from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.orm import Mapped, mapped_column - - -class ReferencedType(enum.Enum): - """ - Enum for the Referenced Entity Type - """ - - ATTRIBUTE = 0 - OBJECT = 1 - - -class ObjectReference(Base): - __tablename__ = "object_references" - - id = Column(Integer, primary_key=True, index=True) - uuid = Column(UUID(as_uuid=True), unique=True, default=uuid.uuid4) - timestamp = Column(Integer, nullable=False) - object_id = Column(Integer, ForeignKey("objects.id"), nullable=False) - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) - source_uuid = Column(UUID(as_uuid=True)) - referenced_uuid = Column(UUID(as_uuid=True)) - referenced_id = Column(Integer, nullable=True) - referenced_type: Mapped[ReferencedType] = mapped_column( - Enum(ReferencedType, name="referenced_type"), nullable=False - ) - relationship_type = Column(String) - comment = Column(String, nullable=False) - deleted = Column(Boolean, nullable=False, default=False) - - def to_misp_format(self): - """Convert the ObjectReference to a MISP-compatible dictionary representation.""" - return { - "id": self.id, - "uuid": str(self.uuid), - "timestamp": self.timestamp, - "object_id": self.object_id, - "event_id": self.event_id, - "source_uuid": str(self.source_uuid) if self.source_uuid else None, - "referenced_uuid": str(self.referenced_uuid) if self.referenced_uuid else None, - "referenced_id": self.referenced_id, - "referenced_type": self.referenced_type.name if self.referenced_type else None, - "relationship_type": self.relationship_type, - "comment": self.comment, - "deleted": self.deleted, - } \ No newline at end of file diff --git a/api/app/models/tag.py b/api/app/models/tag.py index 54876f1b..11f0a74d 100644 --- a/api/app/models/tag.py +++ b/api/app/models/tag.py @@ -1,6 +1,5 @@ from app.database import Base from sqlalchemy import Boolean, Column, ForeignKey, Integer, String -from sqlalchemy.orm import relationship class Tag(Base): @@ -33,26 +32,3 @@ def to_misp_format(self): "is_custom_galaxy": self.is_custom_galaxy, "local_only": self.local_only, } - - -class EventTag(Base): - __tablename__ = "event_tags" - - id = Column(Integer, primary_key=True, index=True) - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) - event = relationship("Event", lazy="subquery", overlaps="tags") - tag_id = Column(Integer, ForeignKey("tags.id"), nullable=False) - tag = relationship("Tag", lazy="subquery", overlaps="tags") - local = Column(Boolean, nullable=False, default=False) - - -class AttributeTag(Base): - __tablename__ = "attribute_tags" - - id = Column(Integer, primary_key=True, index=True) - attribute_id = Column(Integer, ForeignKey("attributes.id"), nullable=False) - attribute = relationship("Attribute", lazy="subquery", overlaps="tags") - event_id = Column(Integer, ForeignKey("events.id"), nullable=False) - tag_id = Column(Integer, ForeignKey("tags.id"), nullable=False) - tag = relationship("Tag", lazy="subquery", overlaps="tags") - local = Column(Boolean, nullable=False, default=False) diff --git a/api/app/repositories/attachments.py b/api/app/repositories/attachments.py index 5d3cade6..9d0d2c78 100644 --- a/api/app/repositories/attachments.py +++ b/api/app/repositories/attachments.py @@ -77,12 +77,12 @@ def upload_attachments_to_event( meta_category=template["meta_category"], template_version=template["version"], comment=attachment.filename, - event_id=event.id, + event_uuid=event.uuid, timestamp=int(time.time()), ) filename_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="filename", category=attachment_meta.get("category", "External analysis"), type="filename", @@ -100,7 +100,7 @@ def upload_attachments_to_event( sha1.update(file_content) sha1 = sha1.hexdigest() sha1_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="sha1", category="External analysis", type="sha1", @@ -115,7 +115,7 @@ def upload_attachments_to_event( sha256.update(file_content) sha256 = sha256.hexdigest() sha256_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="sha256", category="External analysis", type="sha256", @@ -128,7 +128,7 @@ def upload_attachments_to_event( # get file md5 md5sum = hashlib.md5(file_content).hexdigest() md5_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="md5", category="External analysis", type="md5", @@ -141,7 +141,7 @@ def upload_attachments_to_event( # get file size size = len(file_content) size_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="size-in-bytes", disable_correlation=True, category="Other", @@ -154,7 +154,7 @@ def upload_attachments_to_event( # attachment attribute attachment_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="attachment", category="Payload delivery", type="attachment", @@ -167,7 +167,7 @@ def upload_attachments_to_event( # malware analysis if attachment_meta.get("is_malware", False): malware_attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, object_relation="malware", category="External analysis", type="malware", diff --git a/api/app/repositories/attributes.py b/api/app/repositories/attributes.py index 38b7e075..d6320476 100644 --- a/api/app/repositories/attributes.py +++ b/api/app/repositories/attributes.py @@ -1,22 +1,21 @@ +import math import time -from typing import Iterable, Union -from uuid import UUID +from datetime import datetime +from typing import Iterable, Optional +from uuid import UUID, uuid4 from app.models.event import DistributionLevel from app.services.opensearch import get_opensearch_client -from app.models import attribute as attribute_models from app.models import tag as tag_models from app.models import user as user_models +from app.schemas import tag as tag_schemas from app.repositories import attachments as attachments_repository -from app.repositories import events as events_repository from app.schemas import attribute as attribute_schemas from app.schemas import event as event_schemas from app.worker import tasks from fastapi import HTTPException, status -from fastapi_pagination.ext.sqlalchemy import paginate +from fastapi_pagination import Page, Params from pymisp import MISPAttribute, MISPTag from sqlalchemy.orm import Session -from sqlalchemy.sql import select -from fastapi_pagination import Page from collections import defaultdict from opensearchpy.exceptions import NotFoundError @@ -56,108 +55,133 @@ def enrich_attributes_page_with_correlations( return attributes_page -def get_attributes( - db: Session, +def get_attributes_from_opensearch( + params: Params, event_uuid: str = None, deleted: bool = None, - object_id: int = None, + object_uuid: UUID = None, type: str = None, ) -> Page[attribute_schemas.Attribute]: - query = select(attribute_models.Attribute) + client = get_opensearch_client() + must_clauses = [] if event_uuid is not None: - db_event = events_repository.get_event_by_uuid(event_uuid=event_uuid, db=db) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - query = query.where(attribute_models.Attribute.event_id == db_event.id) - + must_clauses.append({"term": {"event_uuid.keyword": event_uuid}}) if deleted is not None: - query = query.where(attribute_models.Attribute.deleted == deleted) - + must_clauses.append({"term": {"deleted": deleted}}) if type is not None: - query = query.where(attribute_models.Attribute.type == type) + must_clauses.append({"term": {"type.keyword": type}}) + + # None → standalone attributes only (no parent object) + if object_uuid is None: + must_clauses.append( + {"bool": {"must_not": [{"exists": {"field": "object_uuid"}}]}} + ) + else: + must_clauses.append({"term": {"object_uuid": str(object_uuid)}}) - query = query.where(attribute_models.Attribute.object_id == object_id) + query_body = { + "query": {"bool": {"must": must_clauses}}, + "from": (params.page - 1) * params.size, + "size": params.size, + "sort": [{"timestamp": {"order": "desc"}}], + } - page_results = paginate(db, query) + try: + response = client.search(index="misp-attributes", body=query_body) + except NotFoundError: + return Page(items=[], total=0, page=params.page, size=params.size, pages=0) - return enrich_attributes_page_with_correlations(page_results) + total = response["hits"]["total"]["value"] + hits = response["hits"]["hits"] + items = [attribute_schemas.Attribute.model_validate(hit["_source"]) for hit in hits] -def get_attribute_by_id( - db: Session, attribute_id: int -) -> Union[attribute_models.Attribute, None]: - return ( - db.query(attribute_models.Attribute) - .filter(attribute_models.Attribute.id == attribute_id) - .first() + pages = math.ceil(total / params.size) if params.size > 0 else 0 + attributes_page = Page( + items=items, total=total, page=params.page, size=params.size, pages=pages ) + return enrich_attributes_page_with_correlations(attributes_page) + + +def get_attribute_from_opensearch( + attribute_uuid: UUID, +) -> Optional[attribute_schemas.Attribute]: + client = get_opensearch_client() + + try: + doc = client.get(index="misp-attributes", id=str(attribute_uuid)) + source = doc["_source"] + except NotFoundError: + return None + + return attribute_schemas.Attribute.model_validate(source) def get_attribute_by_uuid( db: Session, attribute_uuid: UUID -) -> Union[attribute_models.Attribute, None]: - return ( - db.query(attribute_models.Attribute) - .filter(attribute_models.Attribute.uuid == attribute_uuid) - .first() - ) +) -> Optional[attribute_schemas.Attribute]: + return get_attribute_from_opensearch(attribute_uuid) def create_attribute( db: Session, attribute: attribute_schemas.AttributeCreate -) -> attribute_models.Attribute: - db_attribute = attribute_models.Attribute( - event_id=attribute.event_id, - object_id=( - attribute.object_id - if attribute.object_id is not None and attribute.object_id > 0 - else None - ), - object_relation=attribute.object_relation, - category=attribute.category, - type=attribute.type, - value=attribute.value, - to_ids=attribute.to_ids, - uuid=attribute.uuid, - timestamp=attribute.timestamp or time.time(), - distribution=( - event_schemas.DistributionLevel(attribute.distribution) - if attribute.distribution is not None - else event_schemas.DistributionLevel.INHERIT_EVENT - ), - sharing_group_id=attribute.sharing_group_id, - comment=attribute.comment, - deleted=attribute.deleted, - disable_correlation=attribute.disable_correlation, - first_seen=attribute.first_seen, - last_seen=attribute.last_seen, - ) - db.add(db_attribute) - db.commit() - db.refresh(db_attribute) +) -> attribute_schemas.Attribute: + client = get_opensearch_client() + attribute_uuid = str(attribute.uuid or uuid4()) + now = int(time.time()) + + event_uuid = str(attribute.event_uuid) if attribute.event_uuid else None + + dist = attribute.distribution + dist_val = dist.value if hasattr(dist, "value") else (dist if dist is not None else 5) + + attr_doc = { + "uuid": attribute_uuid, + "event_uuid": event_uuid, + "object_uuid": str(attribute.object_uuid) if attribute.object_uuid else None, + "object_relation": attribute.object_relation, + "category": attribute.category, + "type": attribute.type, + "value": attribute.value, + "to_ids": attribute.to_ids if attribute.to_ids is not None else True, + "timestamp": attribute.timestamp or now, + "distribution": dist_val, + "sharing_group_id": attribute.sharing_group_id, + "comment": attribute.comment or "", + "deleted": attribute.deleted or False, + "disable_correlation": attribute.disable_correlation or False, + "first_seen": attribute.first_seen, + "last_seen": attribute.last_seen, + "data": "", + "tags": [], + "@timestamp": datetime.fromtimestamp(attribute.timestamp or now).isoformat(), + } - if db_attribute is not None: - tasks.handle_created_attribute.delay( - db_attribute.id, db_attribute.object_id, db_attribute.event_id - ) + client.index(index="misp-attributes", id=attribute_uuid, body=attr_doc, refresh=True) - return db_attribute + tasks.handle_created_attribute.delay(attribute_uuid, attr_doc["object_uuid"], event_uuid) + + return attribute_schemas.Attribute.model_validate(attr_doc) def create_attribute_from_pulled_attribute( db: Session, pulled_attribute: MISPAttribute, - local_event_id: int, + event_uuid: str, user: user_models.User, -) -> attribute_models.Attribute: +) -> attribute_schemas.Attribute: # TODO: process sharing group // captureSG # TODO: enforce warninglist - local_attribute = attribute_models.Attribute( - event_id=local_event_id, + dist = pulled_attribute.distribution + dist_val = ( + event_schemas.DistributionLevel(dist) + if dist is not None + else event_schemas.DistributionLevel.INHERIT_EVENT + ) + + attr_create = attribute_schemas.AttributeCreate( category=pulled_attribute.category, type=pulled_attribute.type, value=( @@ -167,59 +191,50 @@ def create_attribute_from_pulled_attribute( ), to_ids=pulled_attribute.to_ids, uuid=pulled_attribute.uuid, - timestamp=pulled_attribute.timestamp.timestamp(), - distribution=event_schemas.DistributionLevel( - event_schemas.DistributionLevel(pulled_attribute.distribution) - if pulled_attribute.distribution is not None - else event_schemas.DistributionLevel.INHERIT_EVENT - ), + timestamp=int(pulled_attribute.timestamp.timestamp()), + distribution=dist_val, comment=pulled_attribute.comment, sharing_group_id=None, deleted=pulled_attribute.deleted, disable_correlation=pulled_attribute.disable_correlation, object_relation=getattr(pulled_attribute, "object_relation", None), first_seen=( - pulled_attribute.first_seen.timestamp() - if hasattr(pulled_attribute, "first_seen") + int(pulled_attribute.first_seen.timestamp()) + if hasattr(pulled_attribute, "first_seen") and pulled_attribute.first_seen else None ), last_seen=( - pulled_attribute.last_seen.timestamp() - if hasattr(pulled_attribute, "last_seen") + int(pulled_attribute.last_seen.timestamp()) + if hasattr(pulled_attribute, "last_seen") and pulled_attribute.last_seen else None ), + event_uuid=event_uuid, ) + local_attribute = create_attribute(db, attr_create) + if pulled_attribute.data is not None: - # store file attachments_repository.store_attachment( str(pulled_attribute.uuid), pulled_attribute.data.getvalue() ) - # TODO: process sigthings + # TODO: process sightings # TODO: process galaxies - capture_attribute_tags( - db, local_attribute, pulled_attribute.tags, local_event_id, user - ) + capture_attribute_tags(db, pulled_attribute.tags, user, str(local_attribute.uuid)) return local_attribute def update_attribute_from_pulled_attribute( db: Session, - local_attribute: attribute_models.Attribute, + local_attribute: attribute_schemas.Attribute, pulled_attribute: MISPAttribute, - local_event_id: int, user: user_models.User, -) -> attribute_models.Attribute: - - pulled_attribute.id = local_attribute.id - pulled_attribute.event_id = local_event_id +) -> attribute_schemas.Attribute: if local_attribute.timestamp < pulled_attribute.timestamp.timestamp(): attribute_patch = attribute_schemas.AttributeUpdate( - event_id=local_event_id, category=pulled_attribute.category, type=pulled_attribute.type, value=( @@ -241,92 +256,73 @@ def update_attribute_from_pulled_attribute( ), first_seen=( pulled_attribute.first_seen.timestamp() - if hasattr(pulled_attribute, "first_seen") + if hasattr(pulled_attribute, "first_seen") and pulled_attribute.first_seen else local_attribute.first_seen ), last_seen=( pulled_attribute.last_seen.timestamp() - if hasattr(pulled_attribute, "last_seen") + if hasattr(pulled_attribute, "last_seen") and pulled_attribute.last_seen else local_attribute.last_seen ), ) - update_attribute(db, local_attribute.id, attribute_patch) + update_attribute(db, local_attribute.uuid, attribute_patch) if pulled_attribute.data is not None: - # store file attachments_repository.store_attachment( str(pulled_attribute.uuid), pulled_attribute.data.getvalue() ) - capture_attribute_tags( - db, local_attribute, pulled_attribute.tags, local_event_id, user - ) + capture_attribute_tags(db, pulled_attribute.tags, user, str(local_attribute.uuid)) - # TODO: process sigthings + # TODO: process sightings # TODO: process galaxies - tasks.handle_updated_attribute.delay( - local_attribute.id, local_attribute.object_id, local_attribute.event_id - ) - - return local_attribute + return get_attribute_from_opensearch(local_attribute.uuid) def update_attribute( - db: Session, attribute_id: int, attribute: attribute_schemas.AttributeUpdate -) -> attribute_models.Attribute: - # TODO: Attribute::beforeValidate() && Attribute::$validate - db_attribute = get_attribute_by_id(db, attribute_id=attribute_id) - - if db_attribute is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" - ) - - attribute_patch = attribute.model_dump(exclude_unset=True) - for key, value in attribute_patch.items(): - setattr(db_attribute, key, value) - - db.add(db_attribute) - db.commit() - db.refresh(db_attribute) + db: Session, attribute_uuid: UUID, attribute: attribute_schemas.AttributeUpdate +) -> attribute_schemas.Attribute: + client = get_opensearch_client() + os_attr = get_attribute_from_opensearch(attribute_uuid) + if os_attr is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") - tasks.handle_updated_attribute.delay( - db_attribute.id, db_attribute.object_id, db_attribute.event_id - ) + patch = attribute.model_dump(exclude_unset=True) + for k, v in list(patch.items()): + if hasattr(v, "value"): + patch[k] = v.value - return db_attribute + client.update(index="misp-attributes", id=str(os_attr.uuid), body={"doc": patch}, refresh=True) + tasks.handle_updated_attribute.delay(str(os_attr.uuid), os_attr.object_uuid, str(os_attr.event_uuid) if os_attr.event_uuid else None) -def delete_attribute(db: Session, attribute_id: int | str) -> None: + return get_attribute_from_opensearch(os_attr.uuid) - if isinstance(attribute_id, str): - db_attribute = get_attribute_by_uuid(db, attribute_uuid=UUID(attribute_id)) - else: - db_attribute = get_attribute_by_id(db, attribute_id=attribute_id) - if db_attribute is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" - ) +def delete_attribute(db: Session, attribute_uuid: UUID) -> None: + client = get_opensearch_client() - db_attribute.deleted = True + os_attr = get_attribute_from_opensearch(attribute_uuid) - db.add(db_attribute) - db.commit() - db.refresh(db_attribute) + if os_attr is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") - tasks.handle_deleted_attribute.delay( - db_attribute.id, db_attribute.object_id, db_attribute.event_id + client.update( + index="misp-attributes", + id=str(os_attr.uuid), + body={"doc": {"deleted": True}}, + refresh=True, ) + tasks.handle_deleted_attribute.delay(str(os_attr.uuid), os_attr.object_uuid, str(os_attr.event_uuid) if os_attr.event_uuid else None) + def capture_attribute_tags( db: Session, - db_attribute: attribute_models.Attribute, tags: list[MISPTag], - local_event_id: int, user: user_models.User, + attribute_uuid: str = None, ): tag_name_to_db_tag = {} @@ -359,41 +355,39 @@ def capture_attribute_tags( db.add_all(new_tags) db.commit() - for tag in tags: - if tag.local: - continue - - db_tag = tag_name_to_db_tag[tag.name] - - db_attribute_tag = tag_models.AttributeTag( - attribute=db_attribute, - event_id=local_event_id, - tag_id=db_tag.id, - local=tag.local, + if attribute_uuid and tag_name_to_db_tag: + client = get_opensearch_client() + tag_dicts = [ + tag_schemas.Tag.model_validate(db_tag).model_dump() + for db_tag in tag_name_to_db_tag.values() + ] + client.update( + index="misp-attributes", + id=attribute_uuid, + body={"doc": {"tags": tag_dicts}}, + refresh=True, ) - db.add(db_attribute_tag) - - db.commit() def get_vulnerability_attributes( db: Session, event_uuid: str = None ) -> list[attribute_schemas.Attribute]: - query = select(attribute_models.Attribute).where( - attribute_models.Attribute.type == "vulnerability" - ) - + client = get_opensearch_client() + must_clauses = [{"term": {"type.keyword": "vulnerability"}}] if event_uuid is not None: - db_event = events_repository.get_event_by_uuid(event_uuid=event_uuid, db=db) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - query = query.where(attribute_models.Attribute.event_id == db_event.id) + must_clauses.append({"term": {"event_uuid.keyword": event_uuid}}) - results = db.execute(query).scalars().unique().all() - - return results + try: + response = client.search( + index="misp-attributes", + body={"query": {"bool": {"must": must_clauses}}, "size": 10000}, + ) + except NotFoundError: + return [] + return [ + attribute_schemas.Attribute.model_validate(h["_source"]) + for h in response["hits"]["hits"] + ] def search_attributes( @@ -407,7 +401,12 @@ def search_attributes( OpenSearchClient = get_opensearch_client() search_body = { - "query": {"query_string": {"query": query, "default_field": "value"}}, + "query": { + "bool": { + "must": {"query_string": {"query": query, "default_field": "value"}}, + "filter": {"term": {"deleted": False}}, + } + }, "from": from_value, "size": size, "sort": [{sort_by: {"order": sort_order}}], @@ -461,4 +460,4 @@ def export_attributes( if format == "json": yield hit - search_after = hits[-1].get("sort") \ No newline at end of file + search_after = hits[-1].get("sort") diff --git a/api/app/repositories/events.py b/api/app/repositories/events.py index 43fb77ee..c6b2dd94 100644 --- a/api/app/repositories/events.py +++ b/api/app/repositories/events.py @@ -1,55 +1,82 @@ import logging +import math import time from datetime import datetime -from uuid import UUID -from typing import Union, Iterable +from uuid import UUID, uuid4 +from typing import Optional, Iterable from app.worker import tasks from app.services.opensearch import get_opensearch_client from app.services.vulnerability_lookup import lookup as vulnerability_lookup from app.services.rulezet import lookup as rulezet_lookup -from app.models import event as event_models from app.models import feed as feed_models from app.models import tag as tag_models +from app.models import organisation as org_models from app.repositories import tags as tags_repository from app.repositories import attributes as attributes_repository from app.schemas import event as event_schemas from app.schemas import user as user_schemas +from app.schemas import organisations as org_schemas import app.schemas.attribute as attribute_schemas import app.schemas.vulnerability as vulnerability_schemas from fastapi import HTTPException, status, Query -from fastapi_pagination.ext.sqlalchemy import paginate +from fastapi_pagination import Page, Params +from opensearchpy.exceptions import NotFoundError from pymisp import MISPEvent, MISPOrganisation -from sqlalchemy.orm import Session, noload -from sqlalchemy.sql import select +from sqlalchemy.orm import Session logger = logging.getLogger(__name__) -def get_events(db: Session, info: str = Query(None), deleted: bool = Query(None), uuid: str = Query(None), include_attributes: bool = Query(False)): - query = select(event_models.Event) - - if include_attributes: - query = select(event_models.Event) - else: - # avoid loading child relationships (attributes/objects) to keep the query lightweight - query = select(event_models.Event).options( - noload(event_models.Event.attributes), noload(event_models.Event.objects) - ) +def get_events_from_opensearch( + params: Params, + info: str = None, + deleted: bool = None, + uuid: str = None, +) -> Page[event_schemas.Event]: + client = get_opensearch_client() + must_clauses = [] if info is not None: - search = f"%{info}%" - query = query.where(event_models.Event.info.like(search)) - + must_clauses.append({"match": {"info": info}}) if deleted is not None: - query = query.where(event_models.Event.deleted == deleted) - + must_clauses.append({"term": {"deleted": deleted}}) if uuid is not None: - query = query.where(event_models.Event.uuid == uuid) + must_clauses.append({"term": {"uuid.keyword": uuid}}) + + query_body = { + "query": {"bool": {"must": must_clauses}}, + "from": (params.page - 1) * params.size, + "size": params.size, + "sort": [{"timestamp": {"order": "desc"}}], + } + + response = client.search(index="misp-events", body=query_body) + total = response["hits"]["total"]["value"] + hits = response["hits"]["hits"] - # Sort the query by timestamp in descending order - query = query.order_by(event_models.Event.timestamp.desc()) + items = [] + for hit in hits: + source = hit["_source"] + source.setdefault("attributes", []) + source.setdefault("objects", []) + items.append(event_schemas.Event.model_validate(source)) - return paginate(db, query) + pages = math.ceil(total / params.size) if params.size > 0 else 0 + return Page(items=items, total=total, page=params.page, size=params.size, pages=pages) + + +def get_event_from_opensearch(event_uuid: UUID) -> Optional[event_schemas.Event]: + client = get_opensearch_client() + + try: + doc = client.get(index="misp-events", id=str(event_uuid)) + source = doc["_source"] + except NotFoundError: + return None + + source.setdefault("attributes", []) + source.setdefault("objects", []) + return event_schemas.Event.model_validate(source) def search_events( @@ -59,16 +86,23 @@ def search_events( size: int = 10, sort_by: str = "@timestamp", sort_order: str = "desc", + searchAttributes: bool = False, ): OpenSearchClient = get_opensearch_client() + index = "misp-attributes" if searchAttributes else "misp-events" search_body = { - "query": {"query_string": {"query": query, "default_field": "info"}}, + "query": { + "bool": { + "must": {"query_string": {"query": query, "default_field": "info"}}, + "filter": {"term": {"deleted": False}}, + } + }, "from": from_value, "size": size, "sort": [{sort_by: {"order": sort_order}}], } - response = OpenSearchClient.search(index="misp-events", body=search_body) + response = OpenSearchClient.search(index=index, body=search_body) return { "page": page, @@ -121,237 +155,189 @@ def export_events( search_after = hits[-1].get("sort") -def get_event_by_id(db: Session, event_id: int): - return ( - db.query(event_models.Event).filter(event_models.Event.id == event_id).first() +def get_event_by_info(info: str) -> Optional[event_schemas.Event]: + client = get_opensearch_client() + response = client.search( + index="misp-events", + body={"query": {"term": {"info.keyword": info}}, "size": 1}, ) + hits = response["hits"]["hits"] + if not hits: + return None + source = hits[0]["_source"] + source.setdefault("attributes", []) + source.setdefault("objects", []) + return event_schemas.Event.model_validate(source) -def get_event_by_uuid(db: Session, event_uuid: str): - return ( - db.query(event_models.Event) - .filter(event_models.Event.uuid == event_uuid) - .first() - ) +def get_event_by_uuid(db, event_uuid) -> Optional[event_schemas.Event]: + """Alias for get_event_from_opensearch used by feeds/servers/mcp.""" + return get_event_from_opensearch(UUID(str(event_uuid))) -def get_user_by_info(db: Session, info: str): - return db.query(event_models.Event).filter(event_models.Event.info == info).first() - - -def create_event(db: Session, event: event_schemas.EventCreate) -> event_models.Event: - # TODO: Event::beforeValidate() && Event::$validate - db_event = event_models.Event( - org_id=event.org_id, - date=event.date or datetime.now(), - info=event.info, - user_id=event.user_id, - uuid=event.uuid, - published=event.published, - analysis=event_models.AnalysisLevel(event.analysis), - attribute_count=event.attribute_count, - object_count=event.object_count, - orgc_id=event.orgc_id or event.org_id, - timestamp=event.timestamp or time.time(), - distribution=event_models.DistributionLevel(event.distribution), - sharing_group_id=event.sharing_group_id, - proposal_email_lock=event.proposal_email_lock, - locked=event.locked, - threat_level=event_models.ThreatLevel(event.threat_level), - publish_timestamp=event.publish_timestamp, - sighting_timestamp=event.sighting_timestamp, - disable_correlation=event.disable_correlation, - extends_uuid=event.extends_uuid, - protected=event.protected, - deleted=event.deleted, - ) - db.add(db_event) - db.commit() - db.flush() - db.refresh(db_event) - - tasks.handle_created_event.delay(str(db_event.uuid)) - - return db_event - - -def create_event_from_pulled_event(db: Session, pulled_event: MISPEvent): - event = event_models.Event( - org_id=pulled_event.org_id, - date=pulled_event.date, - info=pulled_event.info, - user_id=pulled_event.user_id, - uuid=pulled_event.uuid, - published=pulled_event.published, - analysis=event_models.AnalysisLevel(pulled_event.analysis), - attribute_count=pulled_event.attribute_count, - object_count=len(pulled_event.objects), - orgc_id=pulled_event.orgc_id, - timestamp=pulled_event.timestamp.timestamp(), - distribution=event_models.DistributionLevel(pulled_event.distribution), - sharing_group_id=( - pulled_event.sharing_group_id - if pulled_event.sharing_group_id is not None - and int(pulled_event.sharing_group_id) > 0 - else None - ), - proposal_email_lock=pulled_event.proposal_email_lock, - locked=pulled_event.locked, - threat_level=event_models.ThreatLevel(pulled_event.threat_level_id), - publish_timestamp=pulled_event.publish_timestamp.timestamp(), - # sighting_timestamp=pulled_event.sighting_timestamp, # TODO: add sighting_timestamp - disable_correlation=pulled_event.disable_correlation, - extends_uuid=pulled_event.extends_uuid or None, - # protected=pulled_event.protected # TODO: add protected [pymisp] - ) - db.add(event) - db.commit() - db.refresh(event) +def get_events_by_uuids(db, uuids) -> list[event_schemas.Event]: + """Alias for get_events_by_uuids_from_opensearch used by feeds.""" + return get_events_by_uuids_from_opensearch(uuids) - tasks.handle_created_event.delay(str(event.uuid)) - return event +def get_event_uuids_from_opensearch() -> list[str]: + """Return all event UUIDs from OpenSearch (used for server push).""" + client = get_opensearch_client() + uuids = [] + search_after = None + while True: + body = { + "query": {"match_all": {}}, + "_source": ["uuid"], + "size": 500, + "sort": [{"uuid.keyword": "asc"}], + } + if search_after: + body["search_after"] = search_after + response = client.search(index="misp-events", body=body) + hits = response["hits"]["hits"] + if not hits: + break + uuids.extend(hit["_source"]["uuid"] for hit in hits) + if len(hits) < 500: + break + search_after = hits[-1]["sort"] + return uuids -def update_event_from_pulled_event( - db: Session, existing_event: event_models.Event, pulled_event: MISPEvent -): - existing_event.date = pulled_event.date - existing_event.info = pulled_event.info - existing_event.uuid = pulled_event.uuid - existing_event.published = pulled_event.published - existing_event.attribute_count = pulled_event.attribute_count - existing_event.object_count = len(pulled_event.objects) - existing_event.analysis = event_models.AnalysisLevel(pulled_event.analysis) - existing_event.timestamp = pulled_event.timestamp.timestamp() or time.time() - existing_event.distribution = event_models.DistributionLevel( - pulled_event.distribution - ) - existing_event.sharing_group_id = ( - pulled_event.sharing_group_id - if int(pulled_event.sharing_group_id) > 0 - else None +def get_events_by_uuids_from_opensearch(uuids) -> list[event_schemas.Event]: + """Return events matching the given UUIDs from OpenSearch.""" + client = get_opensearch_client() + uuids = [str(u) for u in uuids] + if not uuids: + return [] + response = client.search( + index="misp-events", + body={"query": {"terms": {"uuid.keyword": uuids}}, "size": len(uuids)}, ) - existing_event.threat_level = event_models.ThreatLevel(pulled_event.threat_level_id) - existing_event.disable_correlation = pulled_event.disable_correlation - existing_event.extends_uuid = pulled_event.extends_uuid or None - db.commit() - db.refresh(existing_event) - - tasks.handle_updated_event.delay(str(existing_event.uuid)) - - return existing_event - - -def update_event(db: Session, event_id: int, event: event_schemas.EventUpdate): - # TODO: Event::beforeValidate() && Event::$validate - db_event = get_event_by_id(db, event_id=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - event_patch = event.model_dump(exclude_unset=True) - for key, value in event_patch.items(): - setattr(db_event, key, value) - - db.commit() - db.refresh(db_event) - - tasks.handle_updated_event.delay(str(db_event.uuid)) - - return db_event - - -def delete_event(db: Session, event_id: Union[int, UUID], force: bool = False) -> None: - - if isinstance(event_id, int): - db_event = get_event_by_id(db, event_id=event_id) - else: - db_event = get_event_by_uuid(db, event_uuid=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - db_event.deleted = True - - if force: - event_uuid = str(db_event.uuid) - db.delete(db_event) - db.commit() - tasks.delete_indexed_event.delay(event_uuid) - return - - db.commit() - db.refresh(db_event) - - tasks.handle_deleted_event.delay(str(db_event.uuid)) - - -def increment_attribute_count( - db: Session, event_id: int, attributes_count: int = 1 -) -> None: - db_event = get_event_by_id(db, event_id=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) + events = [] + for hit in response["hits"]["hits"]: + source = hit["_source"] + source.setdefault("attributes", []) + source.setdefault("objects", []) + events.append(event_schemas.Event.model_validate(source)) + return events - db_event.attribute_count += attributes_count - db.commit() - db.refresh(db_event) - - -def decrement_attribute_count( - db: Session, event_id: int, attributes_count: int = 1 -) -> None: - db_event = get_event_by_id(db, event_id=event_id) - - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - if db_event.attribute_count > 0: - db_event.attribute_count -= attributes_count - db.commit() - db.refresh(db_event) +def create_event(db: Session, event: event_schemas.EventCreate) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(event.uuid or uuid4()) + now = int(time.time()) + ts = event.timestamp or now + + org = db.query(org_models.Organisation).filter_by(id=event.org_id).first() + org_dict = org_schemas.Organisation.model_validate(org).model_dump(mode="json") if org else None + + event_doc = { + "uuid": event_uuid, + "org_id": event.org_id, + "date": (event.date or datetime.now()).strftime("%Y-%m-%d") if event.date else datetime.now().strftime("%Y-%m-%d"), + "info": event.info, + "user_id": event.user_id, + "published": event.published or False, + "analysis": event.analysis.value if hasattr(event.analysis, "value") else (event.analysis or 0), + "attribute_count": 0, + "object_count": 0, + "orgc_id": event.orgc_id or event.org_id, + "timestamp": ts, + "distribution": event.distribution.value if hasattr(event.distribution, "value") else (event.distribution or 0), + "sharing_group_id": event.sharing_group_id, + "sharing_group": None, + "proposal_email_lock": event.proposal_email_lock or False, + "locked": event.locked or False, + "threat_level": event.threat_level.value if hasattr(event.threat_level, "value") else (event.threat_level or 4), + "publish_timestamp": event.publish_timestamp or 0, + "sighting_timestamp": event.sighting_timestamp, + "disable_correlation": event.disable_correlation or False, + "extends_uuid": str(event.extends_uuid) if event.extends_uuid else None, + "protected": event.protected or False, + "deleted": event.deleted or False, + "tags": [], + "attributes": [], + "objects": [], + "organisation": org_dict, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } + client.index(index="misp-events", id=event_uuid, body=event_doc, refresh=True) + tasks.handle_created_event.delay(event_uuid) -def increment_object_count(db: Session, event_id: int, objects_count: int = 1) -> None: - db_event = get_event_by_id(db, event_id=event_id) + return event_schemas.Event.model_validate(event_doc) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - db_event.object_count += objects_count - - db.commit() - db.refresh(db_event) +def create_event_from_pulled_event(pulled_event: MISPEvent) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(pulled_event.uuid) + ts = int(pulled_event.timestamp.timestamp()) + + event_doc = { + "uuid": event_uuid, + "org_id": pulled_event.org_id, + "orgc_id": pulled_event.orgc_id or pulled_event.org_id, + "date": pulled_event.date.isoformat() if hasattr(pulled_event.date, "isoformat") else str(pulled_event.date), + "info": pulled_event.info, + "user_id": pulled_event.user_id, + "published": pulled_event.published or False, + "analysis": int(pulled_event.analysis) if pulled_event.analysis is not None else 0, + "attribute_count": pulled_event.attribute_count or 0, + "object_count": len(pulled_event.objects), + "timestamp": ts, + "distribution": int(pulled_event.distribution) if pulled_event.distribution is not None else 0, + "sharing_group_id": int(pulled_event.sharing_group_id) if pulled_event.sharing_group_id and int(pulled_event.sharing_group_id) > 0 else None, + "proposal_email_lock": getattr(pulled_event, "proposal_email_lock", False) or False, + "locked": getattr(pulled_event, "locked", False) or False, + "threat_level": int(pulled_event.threat_level_id) if pulled_event.threat_level_id else 4, + "publish_timestamp": int(pulled_event.publish_timestamp.timestamp()), + "disable_correlation": getattr(pulled_event, "disable_correlation", False) or False, + "extends_uuid": str(pulled_event.extends_uuid) if pulled_event.extends_uuid else None, + "protected": getattr(pulled_event, "protected", False) or False, + "deleted": getattr(pulled_event, "deleted", False) or False, + "tags": [], + "attributes": [], + "objects": [], + "organisation": None, + "sharing_group": None, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } + client.index(index="misp-events", id=event_uuid, body=event_doc, refresh=True) + tasks.handle_created_event.delay(event_uuid) -def decrement_object_count(db: Session, event_id: int, objects_count: int = 1) -> None: - db_event = get_event_by_id(db, event_id=event_id) + return event_schemas.Event.model_validate(event_doc) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - db_event.object_count -= objects_count +def update_event_from_pulled_event( + existing_event: event_schemas.Event, pulled_event: MISPEvent +) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(existing_event.uuid) + ts = int(pulled_event.timestamp.timestamp()) + + patch = { + "date": pulled_event.date.isoformat() if hasattr(pulled_event.date, "isoformat") else str(pulled_event.date), + "info": pulled_event.info, + "published": pulled_event.published or False, + "analysis": int(pulled_event.analysis) if pulled_event.analysis is not None else 0, + "attribute_count": pulled_event.attribute_count or 0, + "object_count": len(pulled_event.objects), + "timestamp": ts, + "distribution": int(pulled_event.distribution) if pulled_event.distribution is not None else 0, + "sharing_group_id": int(pulled_event.sharing_group_id) if pulled_event.sharing_group_id and int(pulled_event.sharing_group_id) > 0 else None, + "threat_level": int(pulled_event.threat_level_id) if pulled_event.threat_level_id else 4, + "disable_correlation": pulled_event.disable_correlation or False, + "extends_uuid": str(pulled_event.extends_uuid) if pulled_event.extends_uuid else None, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } - if db_event.object_count < 0: - db_event.object_count = 0 + client.update(index="misp-events", id=event_uuid, body={"doc": patch}, refresh=True) + tasks.handle_updated_event.delay(event_uuid) - db.commit() - db.refresh(db_event) + return get_event_from_opensearch(UUID(event_uuid)) def create_event_from_fetched_event( @@ -360,72 +346,64 @@ def create_event_from_fetched_event( Orgc: MISPOrganisation, feed: feed_models.Feed, user: user_schemas.User, -) -> event_models.Event: - db_event = event_models.Event( - org_id=Orgc.id, - date=fetched_event.date, - info=fetched_event.info, - user_id=user.id, - uuid=fetched_event.uuid, - published=fetched_event.published, - analysis=event_models.AnalysisLevel(fetched_event.analysis), - object_count=len(fetched_event.objects), - orgc_id=Orgc.id, - timestamp=fetched_event.timestamp.timestamp(), - distribution=feed.distribution, - sharing_group_id=feed.sharing_group_id, - locked=(fetched_event.locked if hasattr(fetched_event, "locked") else False), - threat_level=event_models.ThreatLevel(fetched_event.threat_level_id), - publish_timestamp=fetched_event.publish_timestamp.timestamp(), - # sighting_timestamp=fetched_event.sighting_timestamp, # TODO: add sighting_timestamp - disable_correlation=getattr(fetched_event, "disable_correlation", False), - extends_uuid=( - fetched_event.extends_uuid - if hasattr(fetched_event, "extends_uuid") - and fetched_event.extends_uuid != "" +) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(fetched_event.uuid) + ts = int(fetched_event.timestamp.timestamp()) + + event_doc = { + "uuid": event_uuid, + "org_id": Orgc.id, + "orgc_id": Orgc.id, + "date": fetched_event.date.isoformat() if hasattr(fetched_event.date, "isoformat") else str(fetched_event.date), + "info": fetched_event.info, + "user_id": user.id, + "published": fetched_event.published or False, + "analysis": int(fetched_event.analysis) if fetched_event.analysis is not None else 0, + "attribute_count": 0, + "object_count": len(fetched_event.objects), + "timestamp": ts, + "distribution": feed.distribution.value if hasattr(feed.distribution, "value") else int(feed.distribution), + "sharing_group_id": feed.sharing_group_id, + "locked": (fetched_event.locked if hasattr(fetched_event, "locked") else False), + "threat_level": int(fetched_event.threat_level_id) if fetched_event.threat_level_id else 4, + "publish_timestamp": int(fetched_event.publish_timestamp.timestamp()), + "disable_correlation": getattr(fetched_event, "disable_correlation", False), + "extends_uuid": ( + str(fetched_event.extends_uuid) + if hasattr(fetched_event, "extends_uuid") and fetched_event.extends_uuid != "" else None ), - # protected=fetched_event.protected # TODO: add protected [pymisp] - ) + "protected": False, + "deleted": False, + "proposal_email_lock": False, + "tags": [], + "attributes": [], + "objects": [], + "organisation": None, + "sharing_group": None, + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } - db.add(db_event) + client.index(index="misp-events", id=event_uuid, body=event_doc, refresh=True) - # process tags + # process tags into OS event doc for tag in fetched_event.tags: db_tag = tags_repository.get_tag_by_name(db, tag.name) - if db_tag is None: - # create tag if not exists db_tag = tag_models.Tag( name=tag.name, colour=tag.colour, org_id=user.org_id, user_id=user.id, local_only=tag.local, - # exportable=tag.exportable, - # hide_tag=tag.hide_tag, - # numerical_value=tag.numerical_value, - # is_galaxy=tag.is_galaxy, - # is_custom_galaxy=tag.is_custom_galaxy, ) db.add(db_tag) + db.commit() + db.refresh(db_tag) + tags_repository.tag_event(db, event_schemas.Event.model_validate(event_doc), db_tag) - db_event_tag = tag_models.EventTag( - event=db_event, - tag=db_tag, - local=tag.local, - ) - db.add(db_event_tag) - - # TODO: process galaxies - # TODO: process reports - # TODO: process analyst notes - - db.commit() - db.flush() - db.refresh(db_event) - - return db_event + return event_schemas.Event.model_validate(event_doc) def update_event_from_fetched_event( @@ -434,143 +412,183 @@ def update_event_from_fetched_event( Orgc: MISPOrganisation, feed: feed_models.Feed, user: user_schemas.User, -) -> event_models.Event: - db_event = get_event_by_uuid(db, fetched_event.uuid) +) -> event_schemas.Event: + client = get_opensearch_client() + event_uuid = str(fetched_event.uuid) + + existing = get_event_from_opensearch(UUID(event_uuid)) + if existing is None: + logger.error(f"Event {event_uuid} not found in OpenSearch") + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + ts = int(fetched_event.timestamp.timestamp()) + patch = { + "date": fetched_event.date.isoformat() if hasattr(fetched_event.date, "isoformat") else str(fetched_event.date), + "info": fetched_event.info, + "published": fetched_event.published or False, + "analysis": int(fetched_event.analysis) if fetched_event.analysis is not None else 0, + "object_count": len(fetched_event.objects), + "org_id": Orgc.id, + "orgc_id": Orgc.id, + "timestamp": ts, + "distribution": feed.distribution.value if hasattr(feed.distribution, "value") else int(feed.distribution), + "sharing_group_id": feed.sharing_group_id, + "locked": (fetched_event.locked if hasattr(fetched_event, "locked") else False), + "threat_level": int(fetched_event.threat_level_id) if fetched_event.threat_level_id else 4, + "publish_timestamp": int(fetched_event.publish_timestamp.timestamp()), + "disable_correlation": getattr(fetched_event, "disable_correlation", False), + "extends_uuid": ( + str(fetched_event.extends_uuid) + if hasattr(fetched_event, "extends_uuid") and fetched_event.extends_uuid != "" + else None + ), + "@timestamp": datetime.fromtimestamp(ts).isoformat(), + } - if db_event is None: - logger.error(f"Event {fetched_event.uuid} not found") - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - - db_event.date = fetched_event.date - db_event.info = fetched_event.info - db_event.published = fetched_event.published - db_event.analysis = event_models.AnalysisLevel(fetched_event.analysis) - db_event.object_count = len(fetched_event.objects) - db_event.org_id = Orgc.id - db_event.orgc_id = Orgc.id - db_event.timestamp = fetched_event.timestamp.timestamp() - db_event.distribution = feed.distribution - db_event.sharing_group_id = feed.sharing_group_id - db_event.locked = ( - fetched_event.locked if hasattr(fetched_event, "locked") else False - ) - db_event.threat_level = event_models.ThreatLevel(fetched_event.threat_level_id) - db_event.publish_timestamp = fetched_event.publish_timestamp.timestamp() - db_event.disable_correlation = getattr(fetched_event, "disable_correlation", False) - db_event.extends_uuid = ( - fetched_event.extends_uuid - if hasattr(fetched_event, "extends_uuid") and fetched_event.extends_uuid != "" - else None - ) + client.update(index="misp-events", id=event_uuid, body={"doc": patch}, refresh=True) # process tags for tag in fetched_event.tags: db_tag = tags_repository.get_tag_by_name(db, tag.name) - if db_tag is None: - # create tag if not exists db_tag = tag_models.Tag( name=tag.name, colour=tag.colour, org_id=user.org_id, user_id=user.id, local_only=tag.local, - # exportable=tag.exportable, - # hide_tag=tag.hide_tag, - # numerical_value=tag.numerical_value, - # is_galaxy=tag.is_galaxy, - # is_custom_galaxy=tag.is_custom_galaxy, ) db.add(db_tag) + db.commit() + db.refresh(db_tag) + tags_repository.tag_event(db, existing, db_tag) - db_event_tag = tag_models.EventTag( - event=db_event, - tag=db_tag, - local=tag.local, - ) - db.add(db_event_tag) + return get_event_from_opensearch(UUID(event_uuid)) - # remove tags that are not in fetched event - event_tags = db.query(tag_models.EventTag).filter( - tag_models.EventTag.event_id == db_event.id - ) - for event_tag in event_tags: - if event_tag.tag.name not in [tag.name in fetched_event.tags]: - db.delete(event_tag) - # TODO: process galaxies - # TODO: process reports - # TODO: process analyst notes +def update_event(db: Session, event_uuid: UUID, event: event_schemas.EventUpdate) -> event_schemas.Event: + client = get_opensearch_client() + os_event = get_event_from_opensearch(event_uuid) + if os_event is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + patch = event.model_dump(exclude_unset=True) + for k, v in list(patch.items()): + if hasattr(v, "value"): + patch[k] = v.value - db.commit() - db.flush() - db.refresh(db_event) + client.update(index="misp-events", id=str(os_event.uuid), body={"doc": patch}, refresh=True) + tasks.handle_updated_event.delay(str(os_event.uuid)) - return db_event + return get_event_from_opensearch(os_event.uuid) -def get_event_uuids(db: Session) -> list[UUID]: - return db.query(event_models.Event.uuid).all() +def delete_event(db: Session, event_uuid: UUID, force: bool = False) -> None: + client = get_opensearch_client() + os_event = get_event_from_opensearch(event_uuid) + if os_event is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + event_uuid = str(os_event.uuid) -def get_events_by_uuids(db: Session, uuids: list[UUID]) -> list[event_models.Event]: - return ( - db.query(event_models.Event) - .options(noload("*")) - .filter(event_models.Event.uuid.in_(uuids)) - .all() + if force: + tasks.delete_indexed_event(event_uuid) + return + + # Soft delete: mark deleted=True in OS but keep the document so it remains searchable + client.update(index="misp-events", id=event_uuid, body={"doc": {"deleted": True}}, refresh=True) + + +def increment_attribute_count(db: Session, event_uuid: str, attributes_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": f"ctx._source.attribute_count += {attributes_count}", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, ) -def publish_event(db: Session, db_event: event_models.Event) -> event_models.Event: +def decrement_attribute_count(db: Session, event_uuid: str, attributes_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": f"if (ctx._source.attribute_count > 0) {{ ctx._source.attribute_count -= {attributes_count}; }}", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, + ) + - if db_event.published: - return db_event +def increment_object_count(db: Session, event_uuid: str, objects_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": f"ctx._source.object_count += {objects_count}", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, + ) - db_event.published = True - db_event.publish_timestamp = time.time() - db.commit() - db.refresh(db_event) +def decrement_object_count(db: Session, event_uuid: str, objects_count: int = 1) -> None: + client = get_opensearch_client() + client.update_by_query( + index="misp-events", + body={ + "script": {"source": "if (ctx._source.object_count > 0) { ctx._source.object_count -= 1; } else { ctx._source.object_count = 0; }", "lang": "painless"}, + "query": {"term": {"uuid.keyword": event_uuid}}, + }, + refresh=True, + ) - tasks.handle_published_event.delay(str(db_event.uuid)) - return db_event +def publish_event(event: event_schemas.Event) -> event_schemas.Event: + client = get_opensearch_client() + if event.published: + return event + patch = {"published": True, "publish_timestamp": int(time.time())} + client.update(index="misp-events", id=str(event.uuid), body={"doc": patch}, refresh=True) -def unpublish_event(db: Session, db_event: event_models.Event) -> event_models.Event: + tasks.handle_published_event.delay(str(event.uuid)) - if not db_event.published: - return db_event + return get_event_from_opensearch(event.uuid) - db_event.published = False - db.commit() - db.refresh(db_event) +def unpublish_event(event: event_schemas.Event) -> event_schemas.Event: + client = get_opensearch_client() + if not event.published: + return event - tasks.handle_unpublished_event.delay(str(db_event.uuid)) + client.update(index="misp-events", id=str(event.uuid), body={"doc": {"published": False}}, refresh=True) - return db_event + tasks.handle_unpublished_event.delay(str(event.uuid)) + return get_event_from_opensearch(event.uuid) -def toggle_event_correlation( - db: Session, db_event: event_models.Event -) -> event_models.Event: - db_event.disable_correlation = not db_event.disable_correlation - db.commit() - db.refresh(db_event) +def toggle_event_correlation(event: event_schemas.Event) -> event_schemas.Event: + client = get_opensearch_client() + new_val = not event.disable_correlation - tasks.handle_toggled_event_correlation.delay( - str(db_event.uuid), db_event.disable_correlation + client.update( + index="misp-events", + id=str(event.uuid), + body={"doc": {"disable_correlation": new_val}}, + refresh=True, ) - return db_event + tasks.handle_toggled_event_correlation.delay(str(event.uuid), new_val) + + return get_event_from_opensearch(event.uuid) -def import_data(db: Session, event: event_models.Event, data: dict): +def import_data(db: Session, event: event_schemas.Event, data: dict): total_imported_attributes = 0 total_attributes = 0 @@ -581,7 +599,7 @@ def import_data(db: Session, event: event_models.Event, data: dict): for raw_attribute in data["attributes"]: try: attribute = attribute_schemas.AttributeCreate( - event_id=event.id, + event_uuid=event.uuid, category=raw_attribute.get("category", "External analysis"), type=raw_attribute["type"], value=raw_attribute["value"], diff --git a/api/app/repositories/feeds.py b/api/app/repositories/feeds.py index e1025974..0c67ccdd 100644 --- a/api/app/repositories/feeds.py +++ b/api/app/repositories/feeds.py @@ -46,7 +46,7 @@ def create_feed(db: Session, feed: feed_schemas.FeedCreate): source_format=feed.source_format, fixed_event=feed.fixed_event, delta_merge=feed.delta_merge, - event_id=feed.event_id, + event_uuid=feed.event_uuid, publish=feed.publish, override_ids=feed.override_ids, settings=feed.settings, @@ -158,12 +158,12 @@ def process_feed_event( # process objects sync_repository.create_pulled_event_objects( - db, local_event.id, event.objects, user + db, str(local_event.uuid), event.objects, user ) # process attributes sync_repository.create_pulled_event_attributes( - db, local_event.id, event.attributes, user + db, str(local_event.uuid), event.attributes, user ) else: @@ -179,22 +179,16 @@ def process_feed_event( # process objects sync_repository.update_pulled_event_objects( - db, local_event.id, event.objects, user + db, str(local_event.uuid), event.objects, user ) # process attributes sync_repository.update_pulled_event_attributes( - db, local_event.id, event.attributes, user + db, str(local_event.uuid), event.attributes, user ) - local_event.attribute_count = len(local_event.attributes) - local_event.object_count = len(local_event.objects) - db.add(local_event) - db.commit() - tasks.index_event.delay(str(local_event.uuid), full_reindex=True) - return {"result": "success", "message": "Event processed"} @@ -736,21 +730,21 @@ def get_or_create_feed_event( ): label = _feed_import_label(db_feed) if db_feed.fixed_event: - if db_feed.event_id is None: - db_event = _create_feed_event(db, db_feed, user, label) - db_feed.event_id = db_event.id + if db_feed.event_uuid is None: + os_event = _create_feed_event(db, db_feed, user, label) + db_feed.event_uuid = str(os_event.uuid) db.commit() db.refresh(db_feed) else: - db_event = events_repository.get_event_by_id(db, db_feed.event_id) + os_event = events_repository.get_event_from_opensearch(db_feed.event_uuid) - if db_event is None or db_event.deleted: - db_event = _create_feed_event(db, db_feed, user, label) - db_feed.event_id = db_event.id + if os_event is None or os_event.deleted: + os_event = _create_feed_event(db, db_feed, user, label) + db_feed.event_uuid = str(os_event.uuid) db.commit() db.refresh(db_feed) else: - db_event = events_repository.create_event( + os_event = events_repository.create_event( db, event_schemas.EventCreate( info="%s: %s - %s" % (label, db_feed.name, datetime.now().isoformat()), @@ -762,7 +756,7 @@ def get_or_create_feed_event( ), ) - return db_event + return os_event def _create_feed_event(db, db_feed, user, label: str): diff --git a/api/app/repositories/notifications.py b/api/app/repositories/notifications.py index da58089d..083c8f93 100644 --- a/api/app/repositories/notifications.py +++ b/api/app/repositories/notifications.py @@ -4,9 +4,6 @@ from app.models import hunt as hunt_models from app.models import notification as notification_models from app.models import organisation as organisation_models -from app.models import object as object_models -from app.models import event as event_models -from app.models import attribute as attribute_models from app.repositories import user_settings as user_settings_repository from sqlalchemy import select, update, text from sqlalchemy.orm import Session @@ -186,7 +183,7 @@ def build_event_notification( ) -def create_event_notifications(db: Session, type: str, event: event_models.Event): +def create_event_notifications(db: Session, type: str, event): """Create notifications for users following the organisation or event.""" # Get event organisation @@ -288,16 +285,16 @@ def build_attribute_notification( def create_attribute_notifications( - db: Session, type: str, attribute: attribute_models.Attribute + db: Session, type: str, attribute ): """Create notifications for users following event of the attribute.""" - # get event - event = ( - db.query(event_models.Event) - .filter(event_models.Event.id == attribute.event_id) - .first() - ) + event = None + event_uuid_val = getattr(attribute, "event_uuid", None) + if event_uuid_val: + from app.repositories import events as events_repository_local + from uuid import UUID as _UUID + event = events_repository_local.get_event_from_opensearch(_UUID(str(event_uuid_val))) if not event: return [] @@ -316,23 +313,6 @@ def create_attribute_notifications( for follower in attr_followers ] - # If the attribute is linked to an object, we can also notify followers of that object - if attribute.object_id: - object = ( - db.query(object_models.Object) - .filter(object_models.Object.id == attribute.object_id) - .first() - ) - if object: - notifications = build_attribute_notification( - db, - f"object.attribute.{type}", - attribute=attribute, - event=event, - object=object, - ) - return notifications - # Followers of the event event_followers = get_followers_for(db, "events", event.uuid) notifications += [ @@ -368,15 +348,15 @@ def build_object_notification( ) -def create_object_notifications(db: Session, type: str, object: object_models.Object): +def create_object_notifications(db: Session, type: str, object): """Create notifications for users following event of the object.""" - # get event - event = ( - db.query(event_models.Event) - .filter(event_models.Event.id == object.event_id) - .first() - ) + event = None + event_uuid_val = getattr(object, "event_uuid", None) + if event_uuid_val: + from app.repositories import events as events_repository_local + from uuid import UUID as _UUID + event = events_repository_local.get_event_from_opensearch(_UUID(str(event_uuid_val))) if not event: return [] diff --git a/api/app/repositories/object_references.py b/api/app/repositories/object_references.py index 917a4b55..62fe3aba 100644 --- a/api/app/repositories/object_references.py +++ b/api/app/repositories/object_references.py @@ -1,71 +1,91 @@ import time +from typing import Optional +from uuid import UUID -from app.models import object_reference as object_reference_models from app.schemas import object_reference as object_reference_schemas +from app.services.opensearch import get_opensearch_client +from opensearchpy.exceptions import NotFoundError from pymisp import MISPObjectReference from sqlalchemy.orm import Session def create_object_reference( db: Session, object_reference: object_reference_schemas.ObjectReferenceCreate -): - # TODO: ObjectReference::beforeValidate() && ObjectReference::$validate - db_object_reference = object_reference_models.ObjectReference( - uuid=object_reference.uuid, - object_id=object_reference.object_id, - event_id=object_reference.event_id, - source_uuid=object_reference.source_uuid, - referenced_uuid=object_reference.referenced_uuid, - timestamp=object_reference.timestamp or time.time(), - referenced_id=object_reference.referenced_id, - referenced_type=object_reference.referenced_type, - relationship_type=object_reference.relationship_type, - comment=object_reference.comment, - deleted=object_reference.deleted, - ) +) -> object_reference_schemas.ObjectReference: + client = get_opensearch_client() + ref_uuid = str(object_reference.uuid) + + ref_doc = { + "uuid": ref_uuid, + "object_uuid": str(object_reference.object_uuid) if object_reference.object_uuid else None, + "event_uuid": str(object_reference.event_uuid) if object_reference.event_uuid else None, + "source_uuid": str(object_reference.source_uuid) if object_reference.source_uuid else None, + "referenced_uuid": str(object_reference.referenced_uuid) if object_reference.referenced_uuid else None, + "timestamp": object_reference.timestamp or int(time.time()), + "referenced_id": object_reference.referenced_id, + "referenced_type": object_reference.referenced_type, + "relationship_type": object_reference.relationship_type, + "comment": object_reference.comment or "", + "deleted": object_reference.deleted or False, + } - db.add(db_object_reference) - db.commit() - db.refresh(db_object_reference) + client.index(index="misp-object-references", id=ref_uuid, body=ref_doc, refresh=True) - return db_object_reference + return object_reference_schemas.ObjectReference.model_validate(ref_doc) def create_object_reference_from_pulled_object_reference( - db: Session, pulled_object_reference: MISPObjectReference, local_event_id: int -): - db_object_refence = object_reference_models.ObjectReference( - uuid=pulled_object_reference.uuid, - event_id=local_event_id, - source_uuid=pulled_object_reference.object_uuid, - referenced_uuid=pulled_object_reference.referenced_uuid, - timestamp=pulled_object_reference.timestamp, - relationship_type=pulled_object_reference.relationship_type, - comment=pulled_object_reference.comment, - ) + db: Session, pulled_object_reference: MISPObjectReference, event_uuid: UUID +) -> object_reference_schemas.ObjectReference: + client = get_opensearch_client() + ref_uuid = str(pulled_object_reference.uuid) + + ref_doc = { + "uuid": ref_uuid, + "event_uuid": str(event_uuid), + "source_uuid": str(pulled_object_reference.object_uuid), + "referenced_uuid": str(pulled_object_reference.referenced_uuid), + "timestamp": pulled_object_reference.timestamp, + "relationship_type": pulled_object_reference.relationship_type, + "comment": pulled_object_reference.comment or "", + "deleted": False, + } + + client.index(index="misp-object-references", id=ref_uuid, body=ref_doc, refresh=True) + + return object_reference_schemas.ObjectReference.model_validate(ref_doc) - return db_object_refence def get_object_reference_by_uuid( - db: Session, object_reference_uuid: int -) -> object_reference_models.ObjectReference: - return ( - db.query(object_reference_models.ObjectReference) - .filter(object_reference_models.ObjectReference.uuid == object_reference_uuid) - .first() - ) + db: Session, object_reference_uuid +) -> Optional[object_reference_schemas.ObjectReference]: + client = get_opensearch_client() + try: + doc = client.get(index="misp-object-references", id=str(object_reference_uuid)) + return object_reference_schemas.ObjectReference.model_validate(doc["_source"]) + except NotFoundError: + return None + def update_object_reference_from_pulled_object_reference( db: Session, - db_object_reference: object_reference_models.ObjectReference, + db_object_reference: object_reference_schemas.ObjectReference, pulled_object_reference: MISPObjectReference, - local_event_id: int, -): - db_object_reference.event_id = local_event_id - db_object_reference.source_uuid = pulled_object_reference.object_uuid - db_object_reference.referenced_uuid = pulled_object_reference.referenced_uuid - db_object_reference.timestamp = pulled_object_reference.timestamp - db_object_reference.relationship_type = pulled_object_reference.relationship_type - db_object_reference.comment = pulled_object_reference.comment - - return db_object_reference \ No newline at end of file + event_uuid: UUID, +) -> object_reference_schemas.ObjectReference: + client = get_opensearch_client() + patch = { + "event_uuid": str(event_uuid), + "source_uuid": str(pulled_object_reference.object_uuid), + "referenced_uuid": str(pulled_object_reference.referenced_uuid), + "timestamp": pulled_object_reference.timestamp, + "relationship_type": pulled_object_reference.relationship_type, + "comment": pulled_object_reference.comment or "", + } + client.update( + index="misp-object-references", + id=str(db_object_reference.uuid), + body={"doc": patch}, + refresh=True, + ) + return get_object_reference_by_uuid(db, db_object_reference.uuid) diff --git a/api/app/repositories/objects.py b/api/app/repositories/objects.py index ded32873..6fa34f2b 100644 --- a/api/app/repositories/objects.py +++ b/api/app/repositories/objects.py @@ -1,25 +1,23 @@ import logging +import math import time -from typing import Union +from typing import Optional from uuid import UUID, uuid4 -from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import feed as feed_models -from app.models import object as object_models from app.models import user as user_models -from app.models import object_reference as object_reference_models from app.repositories import attributes as attributes_repository from app.repositories import object_references as object_references_repository from app.repositories import events as events_repository from app.schemas import event as event_schemas from app.schemas import object as object_schemas from app.schemas import attribute as attribute_schemas +from app.schemas import object_reference as object_reference_schemas from app.schemas import user as user_schemas from app.worker import tasks from app.services.opensearch import get_opensearch_client from fastapi import HTTPException, status -from fastapi_pagination.ext.sqlalchemy import paginate +from fastapi_pagination import Page, Params from pymisp import MISPObject from sqlalchemy.orm import Session from collections import defaultdict @@ -63,447 +61,516 @@ def enrich_object_attributes_with_correlations( return attributes -def get_objects( - db: Session, + +def get_objects_from_opensearch( + params: Params, event_uuid: str = None, deleted: bool = False, - template_uuid: list[UUID] = None, -) -> list[object_models.Object]: - query = db.query(object_models.Object) + template_uuid: list = None, +) -> Page[object_schemas.Object]: + client = get_opensearch_client() + must_clauses = [{"term": {"deleted": bool(deleted)}}] if event_uuid is not None: - db_event = events_repository.get_event_by_uuid(event_uuid=event_uuid, db=db) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) + must_clauses.append({"term": {"event_uuid.keyword": event_uuid}}) + if template_uuid is not None: + must_clauses.append( + {"terms": {"template_uuid.keyword": [str(u) for u in template_uuid]}} + ) - query = query.filter(object_models.Object.event_id == db_event.id) + query_body = { + "query": {"bool": {"must": must_clauses}}, + "from": (params.page - 1) * params.size, + "size": params.size, + "sort": [{"timestamp": {"order": "desc"}}], + } - if template_uuid is not None: - query = query.filter(object_models.Object.template_uuid.in_(template_uuid)) + try: + response = client.search(index="misp-objects", body=query_body) + except NotFoundError: + return Page(items=[], total=0, page=params.page, size=params.size, pages=0) + + total = response["hits"]["total"]["value"] + hits = response["hits"]["hits"] + + if not hits: + return Page(items=[], total=total, page=params.page, size=params.size, pages=0) + + # Batch-fetch attributes for all returned objects + object_uuids = [str(hit["_source"]["uuid"]) for hit in hits] + attr_response = client.search( + index="misp-attributes", + body={ + "query": {"terms": {"object_uuid.keyword": object_uuids}}, + "size": 10000, + }, + ) - query = query.filter(object_models.Object.deleted.is_(bool(deleted))) + raw_by_uuid: dict = {} + for attr_hit in attr_response["hits"]["hits"]: + src = attr_hit["_source"] + key = src.get("object_uuid") + if key: + raw_by_uuid.setdefault(key, []).append(src) - objects_page = paginate(db, query) + # Build and correlate per-object attribute lists + attrs_by_uuid: dict = {} + for obj_uuid, raw_attrs in raw_by_uuid.items(): + built = [attribute_schemas.Attribute.model_validate(s) for s in raw_attrs] + attrs_by_uuid[obj_uuid] = enrich_object_attributes_with_correlations(built) - for obj in objects_page.items: - obj.attributes = enrich_object_attributes_with_correlations(obj.attributes) + items = [] + for hit in hits: + source = hit["_source"] + obj_uuid = str(source.get("uuid", "")) + source["attributes"] = attrs_by_uuid.get(obj_uuid, []) + source.setdefault("object_references", []) + items.append(object_schemas.Object.model_validate(source)) - return objects_page + pages = math.ceil(total / params.size) if params.size > 0 else 0 + return Page(items=items, total=total, page=params.page, size=params.size, pages=pages) -def get_object_by_id(db: Session, object_id: int): - return ( - db.query(object_models.Object) - .filter(object_models.Object.id == object_id) - .first() +def get_object_from_opensearch( + object_uuid: UUID, +) -> Optional[object_schemas.Object]: + client = get_opensearch_client() + + try: + doc = client.get(index="misp-objects", id=str(object_uuid)) + source = doc["_source"] + except NotFoundError: + return None + + obj_uuid = str(source.get("uuid", "")) + attr_response = client.search( + index="misp-attributes", + body={"query": {"term": {"object_uuid": obj_uuid}}, "size": 10000}, ) + attributes = [] + attributes = [ + attribute_schemas.Attribute.model_validate(h["_source"]) + for h in attr_response["hits"]["hits"] + ] + attributes = enrich_object_attributes_with_correlations(attributes) + source["attributes"] = attributes + source.setdefault("object_references", []) + + return object_schemas.Object.model_validate(source) + def get_object_by_uuid(db: Session, object_uuid: UUID): - return ( - db.query(object_models.Object) - .filter(object_models.Object.uuid == object_uuid) - .first() - ) + return get_object_from_opensearch(object_uuid) + + +def get_objects( + db: Session, + event_uuid=None, + deleted: bool = False, + template_uuid: list = None, +) -> Page[object_schemas.Object]: + """Return all matching objects using search_after pagination (no size cap).""" + client = get_opensearch_client() + + must_clauses = [{"term": {"deleted": bool(deleted)}}] + if event_uuid is not None: + must_clauses.append({"term": {"event_uuid.keyword": str(event_uuid)}}) + if template_uuid is not None: + must_clauses.append( + {"terms": {"template_uuid.keyword": [str(u) for u in template_uuid]}} + ) + + all_hits = [] + search_after = None + while True: + body = { + "query": {"bool": {"must": must_clauses}}, + "size": 500, + "sort": [{"timestamp": {"order": "desc"}}, {"uuid.keyword": "asc"}], + } + if search_after: + body["search_after"] = search_after + try: + response = client.search(index="misp-objects", body=body) + except NotFoundError: + break + hits = response["hits"]["hits"] + if not hits: + break + all_hits.extend(hits) + if len(hits) < 500: + break + search_after = hits[-1]["sort"] + + if not all_hits: + return Page(items=[], total=0, page=1, size=1, pages=0) + + object_uuids = [hit["_source"]["uuid"] for hit in all_hits] + try: + attr_response = client.search( + index="misp-attributes", + body={"query": {"terms": {"object_uuid.keyword": object_uuids}}, "size": 10000}, + ) + raw_by_uuid: dict = {} + for attr_hit in attr_response["hits"]["hits"]: + src = attr_hit["_source"] + key = src.get("object_uuid") + if key: + raw_by_uuid.setdefault(key, []).append(src) + except NotFoundError: + raw_by_uuid = {} + + attrs_by_uuid: dict = {} + for obj_uuid, raw_attrs in raw_by_uuid.items(): + built = [attribute_schemas.Attribute.model_validate(s) for s in raw_attrs] + attrs_by_uuid[obj_uuid] = enrich_object_attributes_with_correlations(built) + + items = [] + for hit in all_hits: + source = hit["_source"] + source["attributes"] = attrs_by_uuid.get(source["uuid"], []) + items.append(object_schemas.Object.model_validate(source)) + + total = len(items) + return Page(items=items, total=total, page=1, size=total or 1, pages=1) def create_object( db: Session, object: object_schemas.ObjectCreate -) -> object_models.Object: - # TODO: MispObject::beforeValidate() && MispObject::$validate - db_object = object_models.Object( - event_id=object.event_id, - name=object.name, - meta_category=object.meta_category, - description=object.description, - template_uuid=object.template_uuid, - template_version=object.template_version, - uuid=object.uuid, - timestamp=object.timestamp or time.time(), - distribution=( - event_schemas.DistributionLevel(object.distribution) - if object.distribution is not None - else event_schemas.DistributionLevel.INHERIT_EVENT - ), - sharing_group_id=object.sharing_group_id, - comment=object.comment, - deleted=object.deleted, - first_seen=object.first_seen, - last_seen=object.last_seen, - ) - - db.add(db_object) - db.commit() - db.refresh(db_object) +) -> object_schemas.Object: + from datetime import datetime as _datetime + + client = get_opensearch_client() + object_uuid = str(object.uuid or uuid4()) + + event_uuid = str(object.event_uuid) if object.event_uuid else None + + dist = object.distribution + dist_val = dist.value if hasattr(dist, "value") else (dist if dist is not None else 5) + + obj_doc = { + "uuid": object_uuid, + "event_uuid": event_uuid, + "name": object.name, + "meta_category": object.meta_category, + "description": object.description, + "template_uuid": object.template_uuid, + "template_version": object.template_version, + "timestamp": object.timestamp, + "distribution": dist_val, + "sharing_group_id": object.sharing_group_id, + "comment": object.comment or "", + "deleted": object.deleted or False, + "first_seen": object.first_seen, + "last_seen": object.last_seen, + "object_references": [], + "@timestamp": _datetime.fromtimestamp(object.timestamp).isoformat(), + } - for attribute in object.attributes: - attribute.object_id = db_object.id - attribute.event_id = object.event_id - attributes_repository.create_attribute(db, attribute) + client.index(index="misp-objects", id=object_uuid, body=obj_doc, refresh=True) + + built_attrs = [] + for attr in (object.attributes or []): + attr.event_uuid = event_uuid + attr_schema = attributes_repository.create_attribute(db, attr) + client.update( + index="misp-attributes", + id=str(attr_schema.uuid), + body={"doc": {"object_uuid": object_uuid}}, + refresh=True, + ) + built_attrs.append(attr_schema) - for object_reference in object.object_references: - object_reference.object_id = db_object.id - object_reference.event_id = db_object.event_id + for object_reference in (object.object_references or []): + object_reference.event_uuid = event_uuid object_references_repository.create_object_reference(db, object_reference) - db.refresh(db_object) + tasks.handle_created_object.delay(object_uuid, event_uuid) - tasks.handle_created_object.delay(db_object.id, db_object.event_id) - - return db_object + obj_doc["attributes"] = built_attrs + return object_schemas.Object.model_validate(obj_doc) def create_object_from_pulled_object( - db: Session, pulled_object: MISPObject, local_event_id: int, user: user_models.User -) -> object_models.Object: - # TODO: process sharing group // captureSG - # TODO: enforce warninglist - - db_object = object_models.Object( - event_id=local_event_id, - name=pulled_object.name, - meta_category=pulled_object["meta-category"], - description=pulled_object.description, - template_uuid=pulled_object.template_uuid, - template_version=pulled_object.template_version, - uuid=pulled_object.uuid, - timestamp=pulled_object.timestamp.timestamp(), - distribution=event_schemas.DistributionLevel(pulled_object.distribution), - sharing_group_id=None, - comment=pulled_object.comment, - deleted=pulled_object.deleted, - first_seen=( - pulled_object.first_seen.timestamp() - if hasattr(pulled_object, "first_seen") + db: Session, pulled_object: MISPObject, event_uuid: str, user: user_models.User +) -> object_schemas.Object: + from datetime import datetime as _datetime + + client = get_opensearch_client() + object_uuid = str(pulled_object.uuid) + + dist = pulled_object.distribution + dist_val = event_schemas.DistributionLevel(dist).value if dist is not None else 5 + ts_raw = pulled_object.timestamp + ts = int(ts_raw.timestamp()) if hasattr(ts_raw, "timestamp") else int(ts_raw or 0) + + obj_doc = { + "uuid": object_uuid, + "event_uuid": event_uuid, + "name": pulled_object.name, + "meta_category": pulled_object["meta-category"], + "description": pulled_object.description, + "template_uuid": str(pulled_object.template_uuid) if pulled_object.template_uuid else None, + "template_version": pulled_object.template_version, + "timestamp": ts, + "distribution": dist_val, + "sharing_group_id": None, + "comment": pulled_object.comment or "", + "deleted": pulled_object.deleted or False, + "first_seen": ( + int(pulled_object.first_seen.timestamp()) + if hasattr(pulled_object, "first_seen") and pulled_object.first_seen else None ), - last_seen=( - pulled_object.last_seen.timestamp() - if hasattr(pulled_object, "last_seen") + "last_seen": ( + int(pulled_object.last_seen.timestamp()) + if hasattr(pulled_object, "last_seen") and pulled_object.last_seen else None ), - ) + "object_references": [], + "@timestamp": _datetime.fromtimestamp(ts).isoformat(), + } + + client.index(index="misp-objects", id=object_uuid, body=obj_doc, refresh=True) for pulled_attribute in pulled_object.attributes: - local_object_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_attribute, local_event_id, user - ) + local_attribute = attributes_repository.create_attribute_from_pulled_attribute( + db, pulled_attribute, event_uuid, user ) - db_object.attributes.append(local_object_attribute) + if local_attribute: + client.update( + index="misp-attributes", + id=str(local_attribute.uuid), + body={"doc": {"object_uuid": object_uuid}}, + refresh=True, + ) for pulled_object_reference in pulled_object.ObjectReference: - local_object_reference = object_references_repository.create_object_reference_from_pulled_object_reference( - db, pulled_object_reference, local_event_id + object_references_repository.create_object_reference_from_pulled_object_reference( + db, pulled_object_reference, event_uuid ) - db_object.object_references.append(local_object_reference) - db.add(db_object) + tasks.handle_created_object.delay(object_uuid, event_uuid) - return db_object + return get_object_from_opensearch(UUID(object_uuid)) def update_object_from_pulled_object( db: Session, - local_object: object_models.Object, + local_object: object_schemas.Object, pulled_object: MISPObject, - local_event_id: int, + event_uuid: str, user: user_models.User, ): + ts_raw = pulled_object.timestamp + pulled_ts = ts_raw.timestamp() if hasattr(ts_raw, "timestamp") else float(ts_raw or 0) - if local_object.timestamp < pulled_object.timestamp.timestamp(): - # find object attributes to delete - local_object_attribute_uuids = [ - attribute.uuid for attribute in local_object.attributes - ] - pulled_object_attribute_uuids = [ - attribute.uuid for attribute in pulled_object.attributes - ] - delete_attributes = [ - str(uuid) - for uuid in local_object_attribute_uuids - if uuid not in pulled_object_attribute_uuids - ] - - for pulled_object_attribute in pulled_object.attributes: - pulled_object_attribute.object_id = local_object.id - local_attribute = attributes_repository.get_attribute_by_uuid( - db, pulled_object_attribute.uuid - ) + if local_object.timestamp < pulled_ts: + client = get_opensearch_client() + local_attr_uuids = {str(a.uuid) for a in local_object.attributes} + pulled_attr_uuids = {str(a.uuid) for a in pulled_object.attributes} + + for pulled_attr in pulled_object.attributes: + local_attribute = attributes_repository.get_attribute_by_uuid(db, pulled_attr.uuid) if local_attribute is None: - local_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_object_attribute, local_event_id, user - ) + new_attr = attributes_repository.create_attribute_from_pulled_attribute( + db, pulled_attr, event_uuid, user ) + if new_attr: + client.update( + index="misp-attributes", + id=str(new_attr.uuid), + body={"doc": {"object_uuid": str(local_object.uuid)}}, + refresh=True, + ) else: - pulled_object_attribute.id = local_attribute.id attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_object_attribute, local_event_id, user + db, local_attribute, pulled_attr, user ) + for uuid_to_delete in local_attr_uuids - pulled_attr_uuids: + attributes_repository.delete_attribute(db, uuid_to_delete) + object_patch = object_schemas.ObjectUpdate( - event_id=local_event_id, name=pulled_object.name, meta_category=pulled_object["meta-category"], description=pulled_object.description, template_uuid=pulled_object.template_uuid, template_version=pulled_object.template_version, - timestamp=pulled_object.timestamp.timestamp(), + timestamp=int(pulled_ts), distribution=event_schemas.DistributionLevel(pulled_object.distribution), sharing_group_id=None, comment=pulled_object.comment, deleted=pulled_object.deleted, first_seen=( - pulled_object.first_seen.timestamp() - if hasattr(pulled_object, "first_seen") + int(pulled_object.first_seen.timestamp()) + if hasattr(pulled_object, "first_seen") and pulled_object.first_seen else local_object.first_seen ), last_seen=( - pulled_object.last_seen.timestamp() - if hasattr(pulled_object, "last_seen") + int(pulled_object.last_seen.timestamp()) + if hasattr(pulled_object, "last_seen") and pulled_object.last_seen else local_object.last_seen ), - delete_attributes=delete_attributes, ) - for pulled_object_reference in pulled_object.ObjectReference: - local_object_reference = ( - object_references_repository.get_object_reference_by_uuid( - db, pulled_object_reference.uuid - ) + for pulled_ref in pulled_object.ObjectReference: + local_ref = object_references_repository.get_object_reference_by_uuid( + db, pulled_ref.uuid ) - - if local_object_reference is None: - local_object_reference = object_references_repository.create_object_reference_from_pulled_object_reference( - db, pulled_object_reference, local_event_id + if local_ref is None: + object_references_repository.create_object_reference_from_pulled_object_reference( + db, pulled_ref, event_uuid + ) + elif local_ref.timestamp < int(pulled_ts): + object_references_repository.update_object_reference_from_pulled_object_reference( + db, local_ref, pulled_ref, event_uuid ) - local_object.object_references.append(local_object_reference) - else: - if ( - local_object_reference.timestamp - < pulled_object.timestamp.timestamp() - ): - pulled_object_reference.id = local_object_reference.id - local_object_reference = object_references_repository.update_object_reference_from_pulled_object_reference( - db, - local_object_reference, - pulled_object_reference, - local_event_id, - ) - update_object(db, local_object.id, object_patch) + update_object(db, local_object.uuid, object_patch) def update_object( - db: Session, object_id: int, object: object_schemas.ObjectUpdate -) -> object_models.Object: - db_object = get_object_by_id(db, object_id=object_id) - - if db_object is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" + db: Session, object_uuid: UUID, object: object_schemas.ObjectUpdate +) -> object_schemas.Object: + client = get_opensearch_client() + os_obj = get_object_from_opensearch(object_uuid) + if os_obj is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Object not found") + + patch = object.model_dump( + exclude_unset=True, + exclude={"attributes", "new_attributes", "update_attributes", "delete_attributes"}, + ) + for k, v in list(patch.items()): + if hasattr(v, "value"): + patch[k] = v.value + + if patch: + client.update(index="misp-objects", id=str(os_obj.uuid), body={"doc": patch}, refresh=True) + + for attr in (object.new_attributes or []): + attr.event_uuid = os_obj.event_uuid + attr_schema = attributes_repository.create_attribute(db, attr) + client.update( + index="misp-attributes", + id=str(attr_schema.uuid), + body={"doc": {"object_uuid": str(os_obj.uuid)}}, + refresh=True, ) - object_patch = object.model_dump(exclude_unset=True) - for key, value in object_patch.items(): - if key == "attributes": - continue - setattr(db_object, key, value) - - # new attribute - for attribute in object.new_attributes: - attribute.object_id = db_object.id - attribute.event_id = db_object.event_id - attribute.uuid = str(uuid4()) - attributes_repository.create_attribute(db, attribute) - - # existing attribute - for attribute in object.update_attributes: - attributes_repository.update_attribute(db, attribute.id, attribute) + for attr in (object.update_attributes or []): + attributes_repository.update_attribute(db, attr.uuid, attr) - # delete attribute - for attribute_id in object.delete_attributes: - attributes_repository.delete_attribute(db, attribute_id) + for attr_id in (object.delete_attributes or []): + attributes_repository.delete_attribute(db, attr_id) - db.add(db_object) - db.commit() - db.refresh(db_object) + tasks.handle_updated_object.delay(str(os_obj.uuid), str(os_obj.event_uuid) if os_obj.event_uuid else None) - tasks.handle_updated_object.delay(db_object.id, db_object.event_id) + return get_object_from_opensearch(os_obj.uuid) - return db_object +def delete_object(db: Session, object_uuid: UUID) -> None: + client = get_opensearch_client() + os_obj = get_object_from_opensearch(object_uuid) + if os_obj is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Object not found") -def delete_object(db: Session, object_id: Union[int, UUID]) -> object_models.Object: - - if isinstance(object_id, int): - db_object = get_object_by_id(db, object_id=object_id) - else: - db_object = get_object_by_uuid(db, object_uuid=object_id) + client.update(index="misp-objects", id=str(os_obj.uuid), body={"doc": {"deleted": True}}, refresh=True) - if db_object is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" + for attr in os_obj.attributes: + client.update( + index="misp-attributes", + id=str(attr.uuid), + body={"doc": {"deleted": True}}, + refresh=True, ) - db_object.deleted = True - - # delete attributes - for attribute in db_object.attributes: - attributes_repository.delete_attribute(db, attribute.id) - - db.add(db_object) - db.commit() - db.refresh(db_object) - - tasks.handle_deleted_object.delay(db_object.id, db_object.event_id) - - return db_object + tasks.handle_deleted_object.delay(str(os_obj.uuid), str(os_obj.event_uuid) if os_obj.event_uuid else None) def create_objects_from_fetched_event( db: Session, - local_event: event_models.Event, + local_event: event_schemas.Event, objects: list[MISPObject], feed: feed_models.Feed, user: user_schemas.User, ): for object in objects: - create_object_from_pulled_object(db, object, local_event.id, user) + create_object_from_pulled_object(db, object, str(local_event.uuid), user) def update_objects_from_fetched_event( db: Session, - local_event: event_models.Event, + local_event: event_schemas.Event, event: event_schemas.Event, feed: feed_models.Feed, user: user_schemas.User, -) -> event_models.Event: - local_event_objects = ( - db.query(object_models.Object.uuid, object_models.Object.timestamp) - .filter(object_models.Object.event_id == local_event.id) - .all() +) -> event_schemas.Event: + client = get_opensearch_client() + resp = client.search( + index="misp-objects", + body={ + "query": {"term": {"event_uuid.keyword": str(local_event.uuid)}}, + "size": 10000, + }, ) - local_event_dict = {str(uuid): timestamp for uuid, timestamp in local_event_objects} - - new_objects = [ - object for object in event.objects if object.uuid not in local_event_dict.keys() - ] + local_event_dict = { + h["_source"]["uuid"]: h["_source"].get("timestamp", 0) + for h in resp["hits"]["hits"] + } + new_objects = [obj for obj in event.objects if str(obj.uuid) not in local_event_dict] updated_objects = [ - object - for object in event.objects - if object.uuid in local_event_dict - and object.timestamp.timestamp() > local_event_dict[object.uuid] + obj + for obj in event.objects + if str(obj.uuid) in local_event_dict + and obj.timestamp.timestamp() > local_event_dict[str(obj.uuid)] ] - # add new objects - local_event = create_objects_from_fetched_event( - db, local_event, new_objects, feed, user - ) - - # update existing attributes - batch_size = 100 # TODO: set the batch size via configuration - updated_uuids = [object.uuid for object in updated_objects] - - for batch_start in range(0, len(updated_uuids), batch_size): - batch_uuids = updated_uuids[batch_start : batch_start + batch_size] - - db_objects = ( - db.query(object_models.Object) - .filter(object_models.Object.uuid.in_(batch_uuids)) - .enable_eagerloads(False) - .yield_per(batch_size) - ) + create_objects_from_fetched_event(db, local_event, new_objects, feed, user) - updated_objects_dict = {object.uuid: object for object in updated_objects} - - for db_object in db_objects: - updated_object = updated_objects_dict[str(db_object.uuid)] - db_object.name = updated_object.name - db_object.meta_category = updated_object["meta-category"] - db_object.description = updated_object.description - db_object.template_uuid = updated_object.template_uuid - db_object.template_version = updated_object.template_version - db_object.timestamp = (updated_object.timestamp.timestamp(),) - db_object.comment = (updated_object.comment,) - db_object.deleted = updated_object.deleted - db_object.first_seen = ( - ( - updated_object.first_seen.timestamp() - if hasattr(updated_object, "first_seen") - else None - ), - ) - db_object.last_seen = ( - ( - updated_object.last_seen.timestamp() - if hasattr(updated_object, "last_seen") - else None - ), + for updated_object in updated_objects: + local_obj = get_object_from_opensearch(updated_object.uuid) + if local_obj is not None: + update_object_from_pulled_object( + db, local_obj, updated_object, str(local_event.uuid), user ) - for attribute in updated_object.attributes: - attribute.object_id = db_object.id - attribute.event_id = local_event.id - - # process attributes - local_event = attributes_repository.update_attributes_from_fetched_event( - db, local_event, updated_object.attributes, feed, user - ) - - # TODO: process galaxies - # TODO: process attribute sightings - # TODO: process analyst notes - - # process object references - for updated_object in updated_objects: - for reference in updated_object.references: - referenced = ( - db.query(object_models.Object) - .filter_by(uuid=reference.referenced_uuid) - .first() + for reference in updated_object.references: + referenced_type = None + referenced = get_object_from_opensearch(reference.referenced_uuid) + if referenced is None: + os_attr = attributes_repository.get_attribute_from_opensearch( + reference.referenced_uuid ) - if referenced is None: - referenced = ( - db.query(attribute_models.Attribute) - .filter_by(uuid=reference.referenced_uuid) - .first() - ) - if referenced is None: - logger.error( - f"Referenced entity not found, skipping object reference uuid: {reference.uuid}" - ) - break + if os_attr is not None: + referenced = os_attr + referenced_type = object_reference_schemas.ReferencedType.ATTRIBUTE + if referenced is None: + logger.error( + f"Referenced entity not found, skipping object reference uuid: {reference.uuid}" + ) + continue + if referenced_type is None: + referenced_type = object_reference_schemas.ReferencedType.OBJECT - db_object_reference = object_reference_models.ObjectReference( + object_references_repository.create_object_reference( + db, + object_reference_schemas.ObjectReferenceCreate( uuid=reference.uuid, - event_id=local_event.id, - object_id=db_object.id, - referenced_uuid=referenced.uuid, - referenced_id=referenced.id if referenced else None, + event_uuid=UUID(str(local_event.uuid)), + source_uuid=getattr(reference, "object_uuid", None), + referenced_uuid=reference.referenced_uuid, + referenced_id=None, relationship_type=reference.relationship_type, - timestamp=int(reference.timestamp), - referenced_type=( - object_reference_models.ReferencedType.ATTRIBUTE - if referenced.__class__.__name__ == "Attribute" - else object_reference_models.ReferencedType.OBJECT - ), - comment=reference.comment, + timestamp=int(reference.timestamp) if reference.timestamp else 0, + referenced_type=referenced_type, + comment=reference.comment or "", deleted=referenced.deleted, - ) - db.add(db_object_reference) - - db.commit() - - db.commit() - - # # TODO: process shadow_attributes - - # db.commit() + ), + ) return local_event diff --git a/api/app/repositories/reports.py b/api/app/repositories/reports.py index 0b7e5077..62472198 100644 --- a/api/app/repositories/reports.py +++ b/api/app/repositories/reports.py @@ -1,5 +1,6 @@ from app.services.opensearch import get_opensearch_client -from app.models import event as event_models +from app.schemas import event as event_schemas +from opensearchpy.exceptions import NotFoundError import logging import uuid import time @@ -20,12 +21,15 @@ def get_event_reports_by_event_uuid(event_uuid: uuid.UUID): } } - response = OpenSearchClient.search(index="misp-event-reports", body=search_body) + try: + response = OpenSearchClient.search(index="misp-event-reports", body=search_body) + except NotFoundError: + return [] return response["hits"]["hits"] -def create_event_report(event: event_models.Event, report: dict): +def create_event_report(event: event_schemas.Event, report: dict): OpenSearchClient = get_opensearch_client() report_uuid = str(uuid.uuid4()) @@ -38,12 +42,10 @@ def create_event_report(event: event_models.Event, report: dict): "sharing_group_id": event.sharing_group_id, "name": report["name"], "content": report["content"], - "id": None, - "event_id": event.id, + "event_uuid": str(event.uuid), "timestamp": int(time.time()), "@timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), "deleted": False, - "event_uuid": str(event.uuid), } response = OpenSearchClient.index( diff --git a/api/app/repositories/servers.py b/api/app/repositories/servers.py index 1db9efc4..abb3a09b 100644 --- a/api/app/repositories/servers.py +++ b/api/app/repositories/servers.py @@ -5,8 +5,8 @@ from app.models import server as server_models from app.models import user as user_models -from app.models import event as event_models from app.models.event import DistributionLevel +from app.schemas import event as event_schemas from app.repositories import sync as sync_repository from app.repositories import events as events_repository from app.repositories import sharing_groups as sharing_groups_repository @@ -137,7 +137,7 @@ def pull_server_by_id_full( user: user_models.User, ): - # get a list of the event_ids on the server + # get a list of the event_uuids on the server event_uuids = get_event_uuids_from_server(server, remote_misp) # TODO apply MISP.enableEventBlocklisting / removeBlockedEvents @@ -164,9 +164,9 @@ def get_event_uuids_from_server(server: server_schemas.Server, remote_misp: PyMI timestamp = server.pull_rules.get("timestamp", None) events = remote_misp.search_index(minimal=True, published=True, timestamp=timestamp) - event_ids = [event["uuid"] for event in events] + event_uuids = [event["uuid"] for event in events] - return event_ids + return event_uuids def pull_event_by_uuid( @@ -175,7 +175,7 @@ def pull_event_by_uuid( server: server_schemas.Server, user: user_models.User, settings: Settings, -) -> Union[event_models.Event, bool]: +) -> Union[event_schemas.Event, bool]: """ see: app/Model/Server.php::__pullEvent() """ @@ -236,8 +236,6 @@ def pull_event_by_uuid( # TODO: process tag collection, see app/Model/Event.php::_add() - tasks.index_event.delay(str(db_event.uuid), full_reindex=True) # TODO: optimize to index only changes - return db_event @@ -386,7 +384,7 @@ def create_or_update_pulled_event( else: event.sharing_group_id = None - created = events_repository.create_event_from_pulled_event(db, event) + created = events_repository.create_event_from_pulled_event(event) if created: sync_repository.create_pulled_event_tags(db, created, event.tags, user) @@ -394,10 +392,10 @@ def create_or_update_pulled_event( db, created.uuid, event.event_reports, user ) sync_repository.create_pulled_event_objects( - db, created.id, event.objects, user + db, str(created.uuid), event.objects, user ) sync_repository.create_pulled_event_attributes( - db, created.id, event.attributes, user + db, str(created.uuid), event.attributes, user ) logger.info(f"Event {event.uuid} created") @@ -441,10 +439,10 @@ def create_or_update_pulled_event( db, updated.uuid, event.event_reports, user ) sync_repository.update_pulled_event_objects( - db, updated.id, event.objects, user + db, str(updated.uuid), event.objects, user ) sync_repository.update_pulled_event_attributes( - db, updated.id, event.attributes, user + db, str(updated.uuid), event.attributes, user ) # TODO: publish event update to ZMQ @@ -701,7 +699,7 @@ def get_remote_event_objects( ) -def get_remote_event_reports(db: Session, server_id: int, event_id: int): +def get_remote_event_reports(db: Session, server_id: int, event_uuid: str): db_server = get_server_by_id(db, server_id=server_id) @@ -717,7 +715,7 @@ def get_remote_event_reports(db: Session, server_id: int, event_id: int): status_code=500, detail="Remote MISP instance not reachable: %s" % ex ) - return remote_misp.get_event_reports(event_id) + return remote_misp.get_event_reports(event_uuid) def push_event_by_uuid( @@ -853,13 +851,28 @@ def push_server_by_id_full( user: user_models.User, ): - # get a list of the event_ids eligible to be pushed to the server - event_uuids = db.query(event_models.Event.uuid).filter( - event_models.Event.published == True, - event_models.Event.distribution == DistributionLevel.CONNECTED_COMMUNITIES - or event_models.Event.distribution == DistributionLevel.ALL_COMMUNITIES, - ).all() - event_uuids = [event_uuid for (event_uuid,) in event_uuids] + # get a list of the event UUIDs eligible to be pushed to the server + from app.services.opensearch import get_opensearch_client as _get_os_client + _os = _get_os_client() + _resp = _os.search( + index="misp-events", + body={ + "query": { + "bool": { + "must": [ + {"term": {"published": True}}, + {"terms": {"distribution": [ + DistributionLevel.CONNECTED_COMMUNITIES.value, + DistributionLevel.ALL_COMMUNITIES.value, + ]}}, + ] + } + }, + "_source": ["uuid"], + "size": 10000, + }, + ) + event_uuids = [h["_source"]["uuid"] for h in _resp["hits"]["hits"]] # push each of the events in different tasks for event_uuid in event_uuids: diff --git a/api/app/repositories/sync.py b/api/app/repositories/sync.py index 711838bf..bb38ebba 100644 --- a/api/app/repositories/sync.py +++ b/api/app/repositories/sync.py @@ -3,11 +3,9 @@ from uuid import UUID from datetime import datetime -from app.models import event as event_models from app.models import tag as tag_models +from app.schemas import event as event_schemas from app.models import user as user_models -from app.models import attribute as attribute_models -from app.models import object as object_models from app.repositories import attributes as attributes_repository from app.repositories import objects as objects_repository from app.repositories import tags as tags_repository @@ -26,7 +24,7 @@ def create_pulled_tags( db: Session, - event: event_models.Event, + event: event_schemas.Event, pulled_tags: list[MISPTag], user: user_models.User, ) -> list[tag_models.Tag]: @@ -42,7 +40,7 @@ def create_pulled_tags( def create_pulled_event_tags( db: Session, - event: event_models.Event, + event: event_schemas.Event, pulled_tags: list[MISPTag], user: user_models.User, ) -> None: @@ -94,8 +92,8 @@ def create_pulled_event_reports( def create_pulled_event_attributes( db: Session, - local_event_id: int, - attributes: list[attribute_models.Attribute], + event_uuid: str, + attributes: list[MISPAttribute], user: user_models.User, ): hashes_dict = {} @@ -104,26 +102,23 @@ def create_pulled_event_attributes( (str(attribute.value) + attribute.type + attribute.category).encode("utf-8") ).hexdigest() if hash not in hashes_dict: - local_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, attribute, local_event_id, user - ) + attributes_repository.create_attribute_from_pulled_attribute( + db, attribute, event_uuid, user ) hashes_dict[hash] = True - db.add(local_attribute) db.commit() def create_pulled_event_objects( db: Session, - local_event_id: int, - objects: list[object_models.Object], + event_uuid: str, + objects: list[MISPObject], user: user_models.User, ): for object in objects: objects_repository.create_object_from_pulled_object( - db, object, local_event_id, user + db, object, event_uuid, user ) db.commit() @@ -131,7 +126,7 @@ def create_pulled_event_objects( def update_pulled_event_objects( db: Session, - local_event_id: int, + event_uuid: str, objects: list[MISPObject], user: user_models.User, ) -> None: @@ -140,17 +135,17 @@ def update_pulled_event_objects( if local_object is None: objects_repository.create_object_from_pulled_object( - db, object, local_event_id, user + db, object, event_uuid, user ) else: objects_repository.update_object_from_pulled_object( - db, local_object, object, local_event_id, user + db, local_object, object, event_uuid, user ) def update_pulled_event_attributes( db: Session, - local_event_id: int, + event_uuid: str, attributes: list[MISPAttribute], user: user_models.User, ) -> None: @@ -160,13 +155,10 @@ def update_pulled_event_attributes( ) if local_attribute is None: - local_attribute = ( - attributes_repository.create_attribute_from_pulled_attribute( - db, pulled_attribute, local_event_id, user - ) + attributes_repository.create_attribute_from_pulled_attribute( + db, pulled_attribute, event_uuid, user ) - db.add(local_attribute) else: attributes_repository.update_attribute_from_pulled_attribute( - db, local_attribute, pulled_attribute, local_event_id, user + db, local_attribute, pulled_attribute, user ) diff --git a/api/app/repositories/tags.py b/api/app/repositories/tags.py index fded9c7e..1141c479 100644 --- a/api/app/repositories/tags.py +++ b/api/app/repositories/tags.py @@ -1,10 +1,10 @@ -from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import tag as tag_models from app.models import user as user_models from app.schemas import tag as tag_schemas +from app.services.opensearch import get_opensearch_client from fastapi import HTTPException, Query, status from fastapi_pagination.ext.sqlalchemy import paginate +from opensearchpy.exceptions import NotFoundError from pymisp import MISPTag from sqlalchemy import func from sqlalchemy.orm import Session @@ -87,112 +87,125 @@ def delete_tag(db: Session, tag_id: int) -> None: db.commit() +def _tag_dict(tag: tag_models.Tag) -> dict: + return { + "id": tag.id, + "name": tag.name, + "colour": tag.colour, + "exportable": tag.exportable, + "hide_tag": tag.hide_tag, + "numerical_value": tag.numerical_value, + "is_galaxy": tag.is_galaxy, + "is_custom_galaxy": tag.is_custom_galaxy, + "local_only": tag.local_only, + } + + def tag_attribute( db: Session, - attribute: attribute_models.Attribute, + attribute, tag: tag_models.Tag, ): - - db_attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute.id, - tag_models.AttributeTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + attr_uuid = str(getattr(attribute, "uuid", None)) + + try: + doc = client.get(index="misp-attributes", id=attr_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") + + current_tags = doc["_source"].get("tags", []) + if any(t.get("name") == tag.name for t in current_tags): + return current_tags + + current_tags.append(_tag_dict(tag)) + client.update( + index="misp-attributes", + id=attr_uuid, + body={"doc": {"tags": current_tags}}, + refresh=True, ) - - if db_attribute_tag is not None: - return db_attribute_tag - - db_attribute_tag = tag_models.AttributeTag( - event_id=attribute.event_id, - attribute_id=attribute.id, - tag_id=tag.id, - ) - - db.add(db_attribute_tag) - db.commit() - db.refresh(db_attribute_tag) - - return db_attribute_tag + return current_tags def untag_attribute( db: Session, - attribute: attribute_models.Attribute, + attribute, tag: tag_models.Tag, ): - db_attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute.id, - tag_models.AttributeTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + attr_uuid = str(getattr(attribute, "uuid", None)) + + try: + doc = client.get(index="misp-attributes", id=attr_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found") + + current_tags = doc["_source"].get("tags", []) + new_tags = [t for t in current_tags if t.get("name") != tag.name] + if len(new_tags) == len(current_tags): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tag not found") + + client.update( + index="misp-attributes", + id=attr_uuid, + body={"doc": {"tags": new_tags}}, + refresh=True, ) - if db_attribute_tag is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="AttributeTag not found" - ) - - db.delete(db_attribute_tag) - db.commit() - def tag_event( db: Session, - event: event_models.Event, + event, tag: tag_models.Tag, ): - - db_event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event.id, - tag_models.EventTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + event_uuid = str(getattr(event, "uuid", None)) + + try: + doc = client.get(index="misp-events", id=event_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + current_tags = doc["_source"].get("tags", []) + if any(t.get("name") == tag.name for t in current_tags): + return current_tags + + current_tags.append(_tag_dict(tag)) + client.update( + index="misp-events", + id=event_uuid, + body={"doc": {"tags": current_tags}}, + refresh=True, ) - - if db_event_tag is not None: - return db_event_tag - - db_event_tag = tag_models.EventTag( - event_id=event.id, - tag_id=tag.id, - ) - - db.add(db_event_tag) - db.commit() - db.refresh(db_event_tag) - - return db_event_tag + return current_tags def untag_event( db: Session, - event: event_models.Event, + event, tag: tag_models.Tag, ): - db_event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event.id, - tag_models.EventTag.tag_id == tag.id, - ) - .first() + client = get_opensearch_client() + event_uuid = str(getattr(event, "uuid", None)) + + try: + doc = client.get(index="misp-events", id=event_uuid) + except NotFoundError: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Event not found") + + current_tags = doc["_source"].get("tags", []) + new_tags = [t for t in current_tags if t.get("name") != tag.name] + if len(new_tags) == len(current_tags): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Tag not found") + + client.update( + index="misp-events", + id=event_uuid, + body={"doc": {"tags": new_tags}}, + refresh=True, ) - if db_event_tag is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="EventTag not found" - ) - - db.delete(db_event_tag) - db.commit() - def capture_tag(db: Session, tag: MISPTag, user: user_models.User) -> tag_models.Tag: # see: app/Model/Tag.php::captureTag @@ -223,6 +236,9 @@ def capture_tag(db: Session, tag: MISPTag, user: user_models.User) -> tag_models ), ) + if db_tag is None: + return None + db.add(db_tag) db.commit() db.refresh(db_tag) diff --git a/api/app/routers/attributes.py b/api/app/routers/attributes.py index debde73d..88c7bd81 100644 --- a/api/app/routers/attributes.py +++ b/api/app/routers/attributes.py @@ -1,6 +1,4 @@ from typing import Optional - -from typing import Union from uuid import UUID from app.auth.security import get_current_active_user from app.db.session import get_db @@ -11,7 +9,7 @@ from app.schemas import user as user_schemas from app.worker import tasks from fastapi import APIRouter, Depends, HTTPException, Response, Security, status, Query -from fastapi_pagination import Page +from fastapi_pagination import Page, Params from fastapi.responses import JSONResponse from sqlalchemy.orm import Session @@ -21,13 +19,13 @@ async def get_attributes_parameters( event_uuid: Optional[str] = None, deleted: Optional[bool] = None, - object_id: Optional[int] = None, + object_uuid: Optional[UUID] = None, type: Optional[str] = None, ): return { "event_uuid": event_uuid, "deleted": deleted, - "object_id": object_id, + "object_uuid": object_uuid, "type": type, } @@ -35,13 +33,17 @@ async def get_attributes_parameters( @router.get("/attributes/", response_model=Page[attribute_schemas.Attribute]) def get_attributes( params: dict = Depends(get_attributes_parameters), - db: Session = Depends(get_db), + page_params: Params = Depends(), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:read"] ), ) -> Page[attribute_schemas.Attribute]: - return attributes_repository.get_attributes( - db, params["event_uuid"], params["deleted"], params["object_id"], params["type"] + return attributes_repository.get_attributes_from_opensearch( + page_params, + params["event_uuid"], + params["deleted"], + params["object_uuid"], + params["type"], ) @@ -76,33 +78,19 @@ async def export_attributes( status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid format specified" ) -@router.get("/attributes/{attribute_id}", response_model=attribute_schemas.Attribute) -def get_attribute_by_id( - attribute_id: Union[int, UUID], - db: Session = Depends(get_db), +@router.get("/attributes/{attribute_uuid}", response_model=attribute_schemas.Attribute) +def get_attribute_by_uuid( + attribute_uuid: UUID, user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:read"] ), ) -> attribute_schemas.Attribute: - - if isinstance(attribute_id, UUID): - db_attribute = attributes_repository.get_attribute_by_uuid( - db, attribute_uuid=attribute_id - ) - else: - db_attribute = attributes_repository.get_attribute_by_id( - db, attribute_id=attribute_id - ) - - if db_attribute is None: + os_attribute = attributes_repository.get_attribute_from_opensearch(attribute_uuid) + if os_attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" ) - - attribute = attribute_schemas.Attribute.from_orm(db_attribute) - attribute.event_uuid = str(db_attribute.event.uuid) - - return attribute + return os_attribute @router.post( @@ -117,76 +105,63 @@ def create_attribute( get_current_active_user, scopes=["attributes:create"] ), ) -> attribute_schemas.Attribute: - if attribute.event_id is None: - if attribute.event_uuid is None: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Event ID or UUID must be provided", - ) - event = events_repository.get_event_by_uuid( - db, event_uuid=str(attribute.event_uuid) + if attribute.event_uuid is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Event UUID must be provided", ) - else: - event = events_repository.get_event_by_id(db, event_id=attribute.event_id) + + event = events_repository.get_event_from_opensearch(attribute.event_uuid) if event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - attribute.event_id = event.id - db_attribute = attributes_repository.create_attribute(db=db, attribute=attribute) - - return db_attribute + attribute.event_uuid = event.uuid + return attributes_repository.create_attribute(db=db, attribute=attribute) -@router.patch("/attributes/{attribute_id}", response_model=attribute_schemas.Attribute) +@router.patch("/attributes/{attribute_uuid}", response_model=attribute_schemas.Attribute) def update_attribute( - attribute_id: int, + attribute_uuid: UUID, attribute: attribute_schemas.AttributeUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:update"] ), ) -> attribute_schemas.Attribute: - attribute_db = attributes_repository.get_attribute_by_id( - db, attribute_id=attribute_id + return attributes_repository.update_attribute( + db=db, attribute_uuid=attribute_uuid, attribute=attribute ) - attribute_db = attributes_repository.update_attribute( - db=db, attribute_id=attribute_id, attribute=attribute - ) - event = events_repository.get_event_by_id(db, event_id=attribute_db.event_id) - - return attribute_db - -@router.delete("/attributes/{attribute_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/attributes/{attribute_uuid}", status_code=status.HTTP_204_NO_CONTENT) def delete_attribute( - attribute_id: int, + attribute_uuid: UUID, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:delete"] ), ): - attributes_repository.delete_attribute(db=db, attribute_id=attribute_id) + attributes_repository.delete_attribute(db=db, attribute_uuid=attribute_uuid) return Response(status_code=status.HTTP_204_NO_CONTENT) @router.post( - "/attributes/{attribute_id}/tag/{tag}", + "/attributes/{attribute_uuid}/tag/{tag}", status_code=status.HTTP_201_CREATED, ) def tag_attribute( - attribute_id: int, + attribute_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:update"] ), ): - attribute = attributes_repository.get_attribute_by_id(db, attribute_id=attribute_id) + attribute = attributes_repository.get_attribute_from_opensearch(attribute_uuid) if attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" @@ -204,18 +179,18 @@ def tag_attribute( @router.delete( - "/attributes/{attribute_id}/tag/{tag}", + "/attributes/{attribute_uuid}/tag/{tag}", status_code=status.HTTP_204_NO_CONTENT, ) def untag_attribute( - attribute_id: int, + attribute_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["attributes:update"] ), ): - attribute = attributes_repository.get_attribute_by_id(db, attribute_id=attribute_id) + attribute = attributes_repository.get_attribute_from_opensearch(attribute_uuid) if attribute is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Attribute not found" diff --git a/api/app/routers/events.py b/api/app/routers/events.py index cef8df1b..f3a55e8c 100644 --- a/api/app/routers/events.py +++ b/api/app/routers/events.py @@ -1,8 +1,6 @@ import json -from typing import Optional, Annotated - -from typing import Union +from typing import Annotated, Optional from uuid import UUID from app.auth.security import get_current_active_user from app.db.session import get_db @@ -14,7 +12,6 @@ from app.schemas import user as user_schemas from app.schemas import object as object_schemas from app.schemas import vulnerability as vulnerability_schemas -from app.worker import tasks from fastapi import ( APIRouter, Depends, @@ -25,7 +22,7 @@ Form, Query, ) -from fastapi_pagination import Page +from fastapi_pagination import Page, Params from sqlalchemy.orm import Session from starlette import status from fastapi.responses import JSONResponse @@ -50,15 +47,14 @@ async def get_events_parameters( @router.get("/events/", response_model=Page[event_schemas.Event]) async def get_events( params: dict = Depends(get_events_parameters), - db: Session = Depends(get_db), + page_params: Params = Depends(), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ) -> Page[event_schemas.Event]: - return events_repository.get_events( - db, + return events_repository.get_events_from_opensearch( + page_params, params["info"], params["deleted"], params["uuid"], - params["include_attributes"], ) @@ -93,23 +89,48 @@ async def export_events( ) -@router.get("/events/{event_id}", response_model=event_schemas.Event) -def get_event_by_id( - event_id: Union[int, UUID], +@router.post("/events/force-index", status_code=status.HTTP_202_ACCEPTED) +def force_index( + uuid: Optional[UUID] = Query(None), db: Session = Depends(get_db), - user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), -) -> event_schemas.Event: + user: user_schemas.User = Security( + get_current_active_user, scopes=["events:update"] + ), +): + from app.worker import tasks as _tasks - if isinstance(event_id, int): - db_event = events_repository.get_event_by_id(db, event_id=event_id) - else: - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_id) + if uuid is not None: + os_event = events_repository.get_event_from_opensearch(uuid) + if os_event is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" + ) + _tasks.handle_updated_event(str(os_event.uuid)) + return JSONResponse( + content={"message": f"Event {uuid} has been re-indexed."}, + status_code=status.HTTP_202_ACCEPTED, + ) - if db_event is None: + uuids = events_repository.get_event_uuids_from_opensearch() + for event_uuid in uuids: + _tasks.handle_updated_event(str(event_uuid)) + return JSONResponse( + content={"message": "Indexing job dispatched for all events."}, + status_code=status.HTTP_202_ACCEPTED, + ) + + +@router.get("/events/{event_uuid}", response_model=event_schemas.Event) +def get_event_by_uuid( + event_uuid: UUID, + user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), +) -> event_schemas.Event: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - return db_event + return os_event @router.post( @@ -122,8 +143,8 @@ def create_event( get_current_active_user, scopes=["events:create"] ), ) -> event_schemas.Event: - db_event = events_repository.get_user_by_info(db, info=event_create_request.info) - if db_event: + existing = events_repository.get_event_by_info(event_create_request.info) + if existing: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="An event with this info already exists", @@ -131,49 +152,46 @@ def create_event( event_create_request.user_id = user.id event_create_request.org_id = user.org_id - db_event = events_repository.create_event(db=db, event=event_create_request) - tasks.index_event.delay(str(db_event.uuid), full_reindex=True) + return events_repository.create_event(db=db, event=event_create_request) - return db_event - -@router.patch("/events/{event_id}", response_model=event_schemas.Event) +@router.patch("/events/{event_uuid}", response_model=event_schemas.Event) def update_event( - event_id: int, + event_uuid: UUID, event: event_schemas.EventUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:update"] ), ) -> event_schemas.Event: - return events_repository.update_event(db=db, event_id=event_id, event=event) + return events_repository.update_event(db=db, event_uuid=event_uuid, event=event) -@router.delete("/events/{event_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/events/{event_uuid}", status_code=status.HTTP_204_NO_CONTENT) def delete_event( - event_id: Union[int, UUID], + event_uuid: UUID, force: Optional[bool] = Query(False), db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:delete"] ), ): - return events_repository.delete_event(db=db, event_id=event_id, force=force) + return events_repository.delete_event(db=db, event_uuid=event_uuid, force=force) @router.post( - "/events/{event_id}/tag/{tag}", + "/events/{event_uuid}/tag/{tag}", status_code=status.HTTP_201_CREATED, ) def tag_event( - event_id: int, + event_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:update"] ), ): - event = events_repository.get_event_by_id(db, event_id=event_id) + event = events_repository.get_event_from_opensearch(event_uuid) if event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -191,18 +209,18 @@ def tag_event( @router.delete( - "/events/{event_id}/tag/{tag}", + "/events/{event_uuid}/tag/{tag}", status_code=status.HTTP_204_NO_CONTENT, ) def untag_event( - event_id: int, + event_uuid: UUID, tag: str, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["events:update"] ), ): - event = events_repository.get_event_by_id(db, event_id=event_id) + event = events_repository.get_event_from_opensearch(event_uuid) if event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -220,12 +238,12 @@ def untag_event( @router.post( - "/events/{event_id}/upload_attachments/", + "/events/{event_uuid}/upload_attachments/", status_code=status.HTTP_200_OK, response_model=list[object_schemas.Object], ) async def upload_attachments( - event_id: Union[int, UUID], + event_uuid: UUID, attachments: list[UploadFile], attachments_meta: Annotated[str, Form()], db: Session = Depends(get_db), @@ -233,10 +251,7 @@ async def upload_attachments( get_current_active_user, scopes=["events:update"] ), ): - if isinstance(event_id, int): - db_event = events_repository.get_event_by_id(db, event_id=event_id) - else: - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_id) + db_event = events_repository.get_event_from_opensearch(event_uuid) if db_event is None: raise HTTPException( @@ -264,7 +279,7 @@ def get_event_attachments( db: Session = Depends(get_db), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) + db_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) if db_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" @@ -282,47 +297,6 @@ def get_event_attachments( return objects -@router.post( - "/events/force-index", - status_code=status.HTTP_201_CREATED, -) -async def force_index( - event_uuid: Optional[UUID] = Query(None, alias="uuid"), - event_id: Optional[int] = Query(None, alias="id"), - db: Session = Depends(get_db), - user: user_schemas.User = Security( - get_current_active_user, scopes=["events:update"] - ), -): - - if event_uuid: - tasks.index_event.delay(str(event_uuid), full_reindex=True) - return JSONResponse( - content={"message": f"Indexing started for event {event_uuid}."}, - status_code=status.HTTP_202_ACCEPTED, - ) - - if event_id: - db_event = events_repository.get_event_by_id(db, event_id=event_id) - if db_event is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" - ) - tasks.index_event.delay(str(db_event.uuid), full_reindex=True) - return JSONResponse( - content={"message": f"Indexing started for event {db_event.uuid}."}, - status_code=status.HTTP_202_ACCEPTED, - ) - - uuids = events_repository.get_event_uuids(db) - for uuid in uuids: - tasks.index_event.delay(str(uuid[0]), full_reindex=True) - - return JSONResponse( - content={"message": "Indexing started for all events."}, - status_code=status.HTTP_202_ACCEPTED, - ) - @router.post("/events/{event_uuid}/publish") def publish( @@ -332,13 +306,13 @@ def publish( get_current_active_user, scopes=["events:publish"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - events_repository.publish_event(db, db_event) + events_repository.publish_event(os_event) return JSONResponse( content={"message": f"Event {event_uuid} has been published."}, @@ -354,13 +328,13 @@ def unpublish( get_current_active_user, scopes=["events:publish"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - events_repository.unpublish_event(db, db_event) + events_repository.unpublish_event(os_event) return JSONResponse( content={"message": f"Event {event_uuid} has been unpublished."}, @@ -376,13 +350,13 @@ def toggle_correlation( get_current_active_user, scopes=["events:update"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - events_repository.toggle_event_correlation(db, db_event) + events_repository.toggle_event_correlation(os_event) return JSONResponse( content={ @@ -401,14 +375,14 @@ def import_data( get_current_active_user, scopes=["events:import"] ), ): - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(event_uuid) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) try: - result = events_repository.import_data(db, event=db_event, data=data) + result = events_repository.import_data(db, event=os_event, data=data) return JSONResponse( content=result, status_code=status.HTTP_202_ACCEPTED, @@ -429,13 +403,13 @@ def get_event_vulnerabilities( db: Session = Depends(get_db), user: user_schemas.User = Security(get_current_active_user, scopes=["events:read"]), ) -> list[vulnerability_schemas.Vulnerability]: - db_event = events_repository.get_event_by_uuid(db, event_uuid=event_uuid) - if db_event is None: + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) return events_repository.get_event_vulnerabilities( db, - event_uuid=db_event.uuid, + event_uuid=str(os_event.uuid), ) diff --git a/api/app/routers/mcp.py b/api/app/routers/mcp.py index e4fd2fdf..7ecdc61a 100644 --- a/api/app/routers/mcp.py +++ b/api/app/routers/mcp.py @@ -379,7 +379,6 @@ def search_attributes(query: str, page: int = 1, size: int = 10) -> dict: - category (str): e.g. "Network activity", "Payload delivery", "External analysis", "Persistence" - event_uuid (str): UUID of parent event - - event_id (int): ID of parent event - to_ids (bool): whether this is an IDS-exportable indicator - comment (str): analyst comment - tags.name (str): associated tag names @@ -1094,9 +1093,13 @@ def search_event_reports( "sort": [{"@timestamp": {"order": "desc"}}], } - response = client.search(index="misp-event-reports", body=query_body) - hits = response["hits"]["hits"] - total = response["hits"]["total"]["value"] + from opensearchpy.exceptions import NotFoundError as _NotFoundError + try: + response = client.search(index="misp-event-reports", body=query_body) + hits = response["hits"]["hits"] + total = response["hits"]["total"]["value"] + except _NotFoundError: + hits, total = [], 0 results = [h["_source"] for h in hits] logger.debug( f"Event reports search returned {total} total, {len(results)} in page {page}" @@ -1478,7 +1481,7 @@ def query_syntax() -> str: "- attribute_count, object_count, tags.name\n\n" "## Attribute fields\n" "- value (default), type, category, uuid\n" - "- event_uuid, event_id, to_ids (bool)\n" + "- event_uuid, to_ids (bool)\n" "- comment, tags.name, deleted, disable_correlation\n" "- expanded.ip2geo.country_iso_code (GeoIP enrichment)\n" ) diff --git a/api/app/routers/objects.py b/api/app/routers/objects.py index 796f0059..d37f63c6 100644 --- a/api/app/routers/objects.py +++ b/api/app/routers/objects.py @@ -1,5 +1,5 @@ from uuid import UUID -from typing import Optional, Union +from typing import Optional from app.auth.security import get_current_active_user from app.db.session import get_db @@ -10,7 +10,7 @@ from app.worker import tasks from fastapi import APIRouter, Depends, HTTPException, Security, status, Response from sqlalchemy.orm import Session -from fastapi_pagination import Page +from fastapi_pagination import Page, Params router = APIRouter() @@ -26,39 +26,32 @@ async def get_objects_parameters( @router.get("/objects/", response_model=Page[object_schemas.Object]) def get_objects( params: dict = Depends(get_objects_parameters), - db: Session = Depends(get_db), + page_params: Params = Depends(), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:read"] ), ) -> Page[object_schemas.Object]: - return objects_repository.get_objects( - db, params["event_uuid"], params["deleted"], params["template_uuid"] + return objects_repository.get_objects_from_opensearch( + page_params, + params["event_uuid"], + params["deleted"], + params["template_uuid"], ) -@router.get("/objects/{object_id}", response_model=object_schemas.Object) +@router.get("/objects/{object_uuid}", response_model=object_schemas.Object) def get_object_by_id( - object_id: Union[int, UUID], - db: Session = Depends(get_db), + object_uuid: UUID, user: user_schemas.User = Security( get_current_active_user, scopes=["objects:read"] ), ): - - if isinstance(object_id, int): - db_object = objects_repository.get_object_by_id(db, object_id=object_id) - else: - db_object = objects_repository.get_object_by_uuid(db, object_uuid=object_id) - - if db_object is None: + os_object = objects_repository.get_object_from_opensearch(object_uuid) + if os_object is None: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="Object not found" ) - - object = object_schemas.Object.from_orm(db_object) - object.event_uuid = str(db_object.event.uuid) - - return object + return os_object @router.post( "/objects/", @@ -73,10 +66,8 @@ def create_object( ), ): - if object.event_id: - event = events_repository.get_event_by_id(db, event_id=object.event_id) - elif object.event_uuid: - event = events_repository.get_event_by_uuid(db, event_uuid=object.event_uuid) + if object.event_uuid: + event = events_repository.get_event_from_opensearch(object.event_uuid) else: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Event UUID is required" @@ -87,32 +78,30 @@ def create_object( status_code=status.HTTP_404_NOT_FOUND, detail="Event not found" ) - object.event_id = event.id - object_db = objects_repository.create_object(db=db, object=object) - - return object_db + object.event_uuid = event.uuid + return objects_repository.create_object(db=db, object=object) -@router.patch("/objects/{object_id}", response_model=object_schemas.Object) +@router.patch("/objects/{object_uuid}", response_model=object_schemas.Object) def update_object( - object_id: int, + object_uuid: UUID, object: object_schemas.ObjectUpdate, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:update"] ), ): - return objects_repository.update_object(db=db, object_id=object_id, object=object) + return objects_repository.update_object(db=db, object_uuid=object_uuid, object=object) -@router.delete("/objects/{object_id}", status_code=status.HTTP_204_NO_CONTENT) +@router.delete("/objects/{object_uuid}", status_code=status.HTTP_204_NO_CONTENT) def delete_object( - object_id: Union[int, UUID], + object_uuid: UUID, db: Session = Depends(get_db), user: user_schemas.User = Security( get_current_active_user, scopes=["objects:delete"] ), ): - objects_repository.delete_object(db=db, object_id=object_id) + objects_repository.delete_object(db=db, object_uuid=object_uuid) return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/api/app/routers/servers.py b/api/app/routers/servers.py index a71e7cea..442a8925 100644 --- a/api/app/routers/servers.py +++ b/api/app/routers/servers.py @@ -204,10 +204,10 @@ def get_remote_event_objects( ) -@router.get("/servers/{server_id}/events/{event_id}/reports") +@router.get("/servers/{server_id}/events/{event_uuid}/reports") def get_remote_event_reports( server_id: int, - event_id: int, + event_uuid: str, limit: int = 10, page: int = 0, db: Session = Depends(get_db), @@ -218,7 +218,7 @@ def get_remote_event_reports( return servers_repository.get_remote_event_reports( db=db, server_id=server_id, - event_id=event_id, + event_uuid=event_uuid, ) diff --git a/api/app/schemas/attribute.py b/api/app/schemas/attribute.py index 4aa59f71..23eca85d 100644 --- a/api/app/schemas/attribute.py +++ b/api/app/schemas/attribute.py @@ -1,3 +1,4 @@ +import logging from enum import Enum from typing import Optional from uuid import UUID @@ -6,10 +7,11 @@ from app.schemas.tag import Tag from pydantic import BaseModel, ConfigDict +logger = logging.getLogger(__name__) + class AttributeBase(BaseModel): - event_id: Optional[int] = None - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None event_uuid: Optional[UUID] = None object_relation: Optional[str] = None category: str @@ -29,18 +31,50 @@ class AttributeBase(BaseModel): class Attribute(AttributeBase): - id: int tags: list[Tag] = [] - correlations: list[dict] = None + correlations: Optional[list[dict]] = None + expanded: Optional[dict] = None model_config = ConfigDict(from_attributes=True) + def to_misp_format(self) -> dict: + from app.services.attachments import get_b64_attachment + from app.settings import get_settings + + attr_json = { + "id": None, + "object_uuid": str(self.object_uuid) if self.object_uuid else None, + "object_relation": self.object_relation, + "category": self.category, + "type": self.type, + "value": self.value, + "to_ids": self.to_ids, + "uuid": str(self.uuid), + "timestamp": self.timestamp, + "distribution": self.distribution, + "sharing_group_id": self.sharing_group_id, + "comment": self.comment, + "deleted": self.deleted, + "disable_correlation": self.disable_correlation, + "first_seen": self.first_seen, + "last_seen": self.last_seen, + "Tag": [tag.model_dump() for tag in self.tags], + } + + if self.type in ["malware-sample", "attachment"]: + try: + attr_json["data"] = get_b64_attachment(str(self.uuid), get_settings()) + except Exception as e: + logger.error(f"Error fetching attachment: {e}") + + return attr_json + class AttributeCreate(AttributeBase): pass class AttributeUpdate(BaseModel): - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None object_relation: Optional[str] = None category: Optional[str] = None type: Optional[str] = None diff --git a/api/app/schemas/event.py b/api/app/schemas/event.py index b08a21e1..ddd994a7 100644 --- a/api/app/schemas/event.py +++ b/api/app/schemas/event.py @@ -38,12 +38,11 @@ class EventBase(BaseModel): class Event(EventBase): - id: int attributes: list[Attribute] = [] objects: list[Object] = [] sharing_group: Optional[SharingGroup] = None tags: list[Tag] = [] - organisation: Organisation + organisation: Optional[Organisation] = None model_config = ConfigDict(from_attributes=True) diff --git a/api/app/schemas/feed.py b/api/app/schemas/feed.py index 883e59c3..52d31ec0 100644 --- a/api/app/schemas/feed.py +++ b/api/app/schemas/feed.py @@ -17,7 +17,7 @@ class FeedBase(BaseModel): source_format: str fixed_event: Optional[bool] = False delta_merge: Optional[bool] = False - event_id: Optional[int] = None + event_uuid: Optional[str] = None publish: Optional[bool] = False override_ids: Optional[bool] = False settings: Optional[dict] = None @@ -55,7 +55,7 @@ class FeedUpdate(FeedBase): source_format: Optional[str] = None fixed_event: Optional[bool] = None delta_merge: Optional[bool] = None - event_id: Optional[int] = None + event_uuid: Optional[str] = None publish: Optional[bool] = None override_ids: Optional[bool] = None settings: Optional[dict] = None diff --git a/api/app/schemas/object.py b/api/app/schemas/object.py index 59e1d173..952e9fdf 100644 --- a/api/app/schemas/object.py +++ b/api/app/schemas/object.py @@ -13,7 +13,6 @@ class ObjectBase(BaseModel): description: Optional[str] = None template_uuid: Optional[str] = None template_version: int - event_id: Optional[int] = None event_uuid: Optional[UUID] = None uuid: Optional[UUID] = None timestamp: int @@ -27,11 +26,30 @@ class ObjectBase(BaseModel): class Object(ObjectBase): - id: int attributes: list[Attribute] = [] object_references: list[ObjectReference] = [] model_config = ConfigDict(from_attributes=True) + def to_misp_format(self) -> dict: + return { + "id": None, + "name": self.name, + "meta-category": self.meta_category, + "description": self.description if self.description else self.name, + "template_uuid": self.template_uuid, + "template_version": self.template_version, + "uuid": str(self.uuid), + "timestamp": self.timestamp, + "distribution": self.distribution, + "sharing_group_id": self.sharing_group_id, + "comment": self.comment, + "deleted": self.deleted, + "first_seen": self.first_seen, + "last_seen": self.last_seen, + "Attribute": [attr.to_misp_format() for attr in self.attributes], + "ObjectReference": [ref.to_misp_format() for ref in self.object_references], + } + class ObjectCreate(ObjectBase): event_uuid: Optional[UUID] = None diff --git a/api/app/schemas/object_reference.py b/api/app/schemas/object_reference.py index dc704319..eb0c51ba 100644 --- a/api/app/schemas/object_reference.py +++ b/api/app/schemas/object_reference.py @@ -1,38 +1,68 @@ +import enum from typing import Optional from uuid import UUID -from app.models.object_reference import ReferencedType from pydantic import BaseModel, ConfigDict +class ReferencedType(enum.Enum): + ATTRIBUTE = 0 + OBJECT = 1 + + class ObjectReferenceBase(BaseModel): uuid: UUID - object_id: int - event_id: int + object_uuid: Optional[UUID] = None + event_uuid: Optional[UUID] = None source_uuid: Optional[UUID] = None referenced_uuid: Optional[UUID] = None - timestamp: int + timestamp: Optional[int] = None referenced_id: Optional[int] = None referenced_type: Optional[ReferencedType] = None relationship_type: Optional[str] = None comment: Optional[str] = None - deleted: bool + deleted: bool = False model_config = ConfigDict(use_enum_values=True) class ObjectReference(ObjectReferenceBase): - id: int model_config = ConfigDict(from_attributes=True) + def to_misp_format(self) -> dict: + ref_type = self.referenced_type + if isinstance(ref_type, ReferencedType): + ref_type_name = ref_type.name + elif ref_type is not None: + try: + ref_type_name = ReferencedType(ref_type).name + except ValueError: + ref_type_name = str(ref_type) + else: + ref_type_name = None + + return { + "id": None, + "uuid": str(self.uuid), + "timestamp": self.timestamp, + "object_uuid": str(self.object_uuid) if self.object_uuid else None, + "event_uuid": str(self.event_uuid) if self.event_uuid else None, + "source_uuid": str(self.source_uuid) if self.source_uuid else None, + "referenced_uuid": str(self.referenced_uuid) if self.referenced_uuid else None, + "referenced_id": self.referenced_id, + "referenced_type": ref_type_name, + "relationship_type": self.relationship_type, + "comment": self.comment, + "deleted": self.deleted, + } + class ObjectReferenceCreate(ObjectReferenceBase): - object_id: Optional[int] = None referenced_type: Optional[ReferencedType] = None comment: Optional[str] = "" class ObjectReferenceUpdate(ObjectReferenceBase): - object_id: Optional[int] = None + object_uuid: Optional[UUID] = None source_uuid: Optional[UUID] = None referenced_uuid: Optional[UUID] = None timestamp: Optional[int] = None diff --git a/api/app/schemas/tag.py b/api/app/schemas/tag.py index c73c833f..17e3ef2c 100644 --- a/api/app/schemas/tag.py +++ b/api/app/schemas/tag.py @@ -38,24 +38,3 @@ class TagUpdate(TagBase): local_only: Optional[bool] = None -class AttributeTagBase(BaseModel): - attribute_id: int - event_id: int - tag_id: int - local: bool - - -class AttributeTag(AttributeTagBase): - id: int - model_config = ConfigDict(from_attributes=True) - - -class EventTagBase(BaseModel): - event_id: int - tag_id: int - local: bool - - -class EventTag(EventTagBase): - id: int - model_config = ConfigDict(from_attributes=True) diff --git a/api/app/tests/api/test_attributes.py b/api/app/tests/api/test_attributes.py index 8f79b251..bc9c929f 100644 --- a/api/app/tests/api/test_attributes.py +++ b/api/app/tests/api/test_attributes.py @@ -1,7 +1,5 @@ import pytest from app.auth import auth -from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import tag as tag_models from app.tests.api_tester import ApiTester from fastapi import status @@ -14,7 +12,7 @@ class TestAttributesResource(ApiTester): def test_get_attributes( self, client: TestClient, - attribute_1: attribute_models.Attribute, + attribute_1, auth_token: auth.Token, ): response = client.get( @@ -25,7 +23,7 @@ def test_get_attributes( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == attribute_1.id + assert data[0]["uuid"] == str(attribute_1.uuid) assert data[0]["category"] == attribute_1.category assert data[0]["type"] == attribute_1.type assert data[0]["value"] == attribute_1.value @@ -42,12 +40,12 @@ def test_get_attributes_unauthorized( @pytest.mark.parametrize("scopes", [["attributes:create"]]) def test_create_attribute( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/attributes/", json={ - "event_id": event_1.id, + "event_uuid": str(event_1.uuid), "category": "Network activity", "type": "ip-dst", "value": "127.0.0.1", @@ -57,20 +55,20 @@ def test_create_attribute( data = response.json() assert response.status_code == status.HTTP_201_CREATED - assert data["id"] is not None - assert data["event_id"] == event_1.id + assert data["uuid"] is not None + assert data["event_uuid"] == str(event_1.uuid) assert data["category"] == "Network activity" assert data["type"] == "ip-dst" assert data["value"] == "127.0.0.1" @pytest.mark.parametrize("scopes", [["attributes:read"]]) def test_create_attribute_unauthorized( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/attributes/", json={ - "event_id": event_1.id, + "event_uuid": str(event_1.uuid), "category": "Network activity", "type": "ip-dst", "value": "127.0.0.1", @@ -81,13 +79,13 @@ def test_create_attribute_unauthorized( @pytest.mark.parametrize("scopes", [["attributes:create"]]) def test_create_attribute_incomplete( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): # missing value response = client.post( "/attributes/", json={ - "event_id": event_1.id, + "event_uuid": str(event_1.uuid), "category": "Network activity", "type": "ip-dst", }, @@ -99,11 +97,11 @@ def test_create_attribute_incomplete( def test_update_attribute( self, client: TestClient, - attribute_1: attribute_models.Attribute, + attribute_1, auth_token: auth.Token, ): response = client.patch( - f"/attributes/{attribute_1.id}", + f"/attributes/{attribute_1.uuid}", json={ "type": "ip-src", "value": "8.8.8.8", @@ -120,11 +118,11 @@ def test_update_attribute( def test_delete_attribute( self, client: TestClient, - attribute_1: attribute_models.Attribute, + attribute_1, auth_token: auth.Token, ): response = client.delete( - f"/attributes/{attribute_1.id}", + f"/attributes/{attribute_1.uuid}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -134,54 +132,44 @@ def test_delete_attribute( def test_tag_attribute( self, client: TestClient, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, + event_1: object, + attribute_1, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.post( - f"/attributes/{attribute_1.id}/tag/{tlp_white_tag.name}", + f"/attributes/{attribute_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_201_CREATED - attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute_1.id, - tag_models.AttributeTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - - assert attribute_tag is not None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_attr = os_client.get(index="misp-attributes", id=str(attribute_1.uuid)) + tag_names = [t.get("name") for t in os_attr["_source"].get("tags", [])] + assert tlp_white_tag.name in tag_names @pytest.mark.parametrize("scopes", [["attributes:update"]]) def test_untag_event( self, client: TestClient, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, + event_1: object, + attribute_1, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.delete( - f"/attributes/{attribute_1.id}/tag/{tlp_white_tag.name}", + f"/attributes/{attribute_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_204_NO_CONTENT - attribute_tag = ( - db.query(tag_models.AttributeTag) - .filter( - tag_models.AttributeTag.attribute_id == attribute_1.id, - tag_models.AttributeTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - - assert attribute_tag is None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_attr = os_client.get(index="misp-attributes", id=str(attribute_1.uuid)) + tag_names = [t.get("name") for t in os_attr["_source"].get("tags", [])] + assert tlp_white_tag.name not in tag_names diff --git a/api/app/tests/api/test_events.py b/api/app/tests/api/test_events.py index 37d4e445..19947fe9 100644 --- a/api/app/tests/api/test_events.py +++ b/api/app/tests/api/test_events.py @@ -1,7 +1,5 @@ import pytest from app.auth import auth -from app.models import attribute as attribute_models -from app.models import event as event_models from app.models import organisation as organisation_models from app.models import tag as tag_models from app.models import user as user_models @@ -17,28 +15,21 @@ def test_get_events( self, client: TestClient, user_1: user_models.User, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, + event_1: object, auth_token: auth.Token, ): response = client.get( - "/events/", headers={"Authorization": "Bearer " + auth_token}, - params={"include_attributes": True} + "/events/", headers={"Authorization": "Bearer " + auth_token} ) data = response.json()["items"] assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == event_1.id assert data[0]["info"] == event_1.info assert data[0]["org_id"] == event_1.org_id assert data[0]["orgc_id"] == event_1.orgc_id assert data[0]["user_id"] == user_1.id - assert data[0]["attributes"][0]["event_id"] == attribute_1.event_id - assert data[0]["attributes"][0]["value"] == attribute_1.value - assert data[0]["attributes"][0]["category"] == attribute_1.category - assert data[0]["attributes"][0]["type"] == attribute_1.type @pytest.mark.parametrize("scopes", [[]]) def test_get_events_unauthorized( @@ -71,7 +62,7 @@ def test_create_event( data = response.json() assert response.status_code == status.HTTP_201_CREATED - assert data["id"] is not None + assert data["uuid"] is not None assert data["info"] == "test create event" assert data["user_id"] == api_tester_user.id assert data["org_id"] == api_tester_user.org_id @@ -109,7 +100,7 @@ def test_create_event_incomplete(self, client: TestClient, auth_token: auth.Toke @pytest.mark.parametrize("scopes", [["events:create"]]) def test_create_event_invalid_exists( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): # event with duplicated info response = client.post( @@ -132,11 +123,11 @@ def test_create_event_invalid_exists( def test_update_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.patch( - f"/events/{event_1.id}", + f"/events/{event_1.uuid}", json={ "info": "updated via API", "published": False, @@ -153,11 +144,11 @@ def test_update_event( def test_delete_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.delete( - f"/events/{event_1.id}", + f"/events/{event_1.uuid}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -167,53 +158,45 @@ def test_delete_event( def test_tag_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.post( - f"/events/{event_1.id}/tag/{tlp_white_tag.name}", + f"/events/{event_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_201_CREATED - event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event_1.id, - tag_models.EventTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - assert event_tag is not None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_event = os_client.get(index="misp-events", id=str(event_1.uuid)) + tag_names = [t.get("name") for t in os_event["_source"].get("tags", [])] + assert tlp_white_tag.name in tag_names @pytest.mark.parametrize("scopes", [["events:update"]]) def test_untag_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, tlp_white_tag: tag_models.Tag, auth_token: auth.Token, db: Session, ): response = client.delete( - f"/events/{event_1.id}/tag/{tlp_white_tag.name}", + f"/events/{event_1.uuid}/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_204_NO_CONTENT - event_tag = ( - db.query(tag_models.EventTag) - .filter( - tag_models.EventTag.event_id == event_1.id, - tag_models.EventTag.tag_id == tlp_white_tag.id, - ) - .first() - ) - assert event_tag is None + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + os_event = os_client.get(index="misp-events", id=str(event_1.uuid)) + tag_names = [t.get("name") for t in os_event["_source"].get("tags", [])] + assert tlp_white_tag.name not in tag_names @pytest.fixture(scope="class") def event_2( @@ -223,19 +206,21 @@ def event_2( user_1: user_models.User, ): """A second event used for force-delete and other destructive tests.""" - event_2 = event_models.Event( + from datetime import datetime + from uuid import UUID + from app.repositories import events as events_repository + from app.schemas import event as event_schemas + + event_create = event_schemas.EventCreate( info="test event 2 for force delete", user_id=user_1.id, orgc_id=1, org_id=organisation_1.id, - date="2020-01-01", - uuid="d8a2b0c1-aaaa-bbbb-cccc-ef1234567890", + date=datetime(2020, 1, 1), + uuid=UUID("d8a2b0c1-aaaa-bbbb-cccc-ef1234567890"), timestamp=1577836800, ) - db.add(event_2) - db.commit() - db.refresh(event_2) - yield event_2 + yield events_repository.create_event(db=db, event=event_create) @pytest.fixture(scope="class") def event_for_filter( @@ -245,43 +230,29 @@ def event_for_filter( user_1: user_models.User, ): """A stable event used for filter/search tests that won't be mutated.""" - ev = event_models.Event( + from datetime import datetime + from uuid import UUID + from app.repositories import events as events_repository + from app.schemas import event as event_schemas + + event_create = event_schemas.EventCreate( info="stable filter test event", user_id=user_1.id, orgc_id=1, org_id=organisation_1.id, - date="2020-01-01", - uuid="f1f7e2a3-0000-0000-0000-000000000001", + date=datetime(2020, 1, 1), + uuid=UUID("f1f7e2a3-0000-0000-0000-000000000001"), timestamp=1577836800, ) - db.add(ev) - db.commit() - db.refresh(ev) - yield ev + yield events_repository.create_event(db=db, event=event_create) - # ---- GET /events/{event_id} ---- - - @pytest.mark.parametrize("scopes", [["events:read"]]) - def test_get_event_by_id( - self, - client: TestClient, - event_1: event_models.Event, - auth_token: auth.Token, - ): - response = client.get( - f"/events/{event_1.id}", - headers={"Authorization": "Bearer " + auth_token}, - ) - data = response.json() - - assert response.status_code == status.HTTP_200_OK - assert data["id"] == event_1.id + # ---- GET /events/{event_uuid} ---- @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_event_by_uuid( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.get( @@ -291,7 +262,7 @@ def test_get_event_by_uuid( data = response.json() assert response.status_code == status.HTTP_200_OK - assert data["id"] == event_1.id + assert data["uuid"] == str(event_1.uuid) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_event_not_found( @@ -300,7 +271,7 @@ def test_get_event_not_found( auth_token: auth.Token, ): response = client.get( - "/events/999999", + "/events/00000000-0000-0000-0000-000000000000", headers={"Authorization": "Bearer " + auth_token}, ) @@ -313,7 +284,7 @@ def test_get_event_not_found( def test_get_events_filter_by_info( self, client: TestClient, - event_for_filter: event_models.Event, + event_for_filter: object, auth_token: auth.Token, ): response = client.get( @@ -325,7 +296,7 @@ def test_get_events_filter_by_info( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == event_for_filter.id + assert data[0]["uuid"] == str(event_for_filter.uuid) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_events_filter_by_info_no_results( @@ -347,7 +318,7 @@ def test_get_events_filter_by_info_no_results( def test_get_events_filter_by_uuid( self, client: TestClient, - event_for_filter: event_models.Event, + event_for_filter: object, auth_token: auth.Token, ): response = client.get( @@ -359,13 +330,13 @@ def test_get_events_filter_by_uuid( assert response.status_code == status.HTTP_200_OK assert len(data) == 1 - assert data[0]["id"] == event_for_filter.id + assert data[0]["uuid"] == str(event_for_filter.uuid) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_events_filter_by_deleted_true( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): # event_1 was soft-deleted by test_delete_event @@ -377,14 +348,14 @@ def test_get_events_filter_by_deleted_true( data = response.json()["items"] assert response.status_code == status.HTTP_200_OK - assert any(item["id"] == event_1.id for item in data) + assert any(item["uuid"] == str(event_1.uuid) for item in data) @pytest.mark.parametrize("scopes", [["events:read"]]) def test_get_events_filter_by_deleted_false( self, client: TestClient, - event_1: event_models.Event, - event_for_filter: event_models.Event, + event_1: object, + event_for_filter: object, auth_token: auth.Token, ): # event_1 was soft-deleted; event_for_filter was not @@ -396,9 +367,9 @@ def test_get_events_filter_by_deleted_false( data = response.json()["items"] assert response.status_code == status.HTTP_200_OK - ids = [item["id"] for item in data] - assert event_1.id not in ids - assert event_for_filter.id in ids + uuids = [item["uuid"] for item in data] + assert str(event_1.uuid) not in uuids + assert str(event_for_filter.uuid) in uuids # ---- DELETE /events/{event_id} force ---- @@ -406,25 +377,25 @@ def test_get_events_filter_by_deleted_false( def test_delete_event_force( self, client: TestClient, - event_2: event_models.Event, + event_2: object, db: Session, auth_token: auth.Token, ): response = client.delete( - f"/events/{event_2.id}", + f"/events/{event_2.uuid}", params={"force": True}, headers={"Authorization": "Bearer " + auth_token}, ) assert response.status_code == status.HTTP_204_NO_CONTENT - db.expire_all() - deleted = ( - db.query(event_models.Event) - .filter(event_models.Event.id == event_2.id) - .first() + from app.services.opensearch import get_opensearch_client + os_client = get_opensearch_client() + response_os = os_client.search( + index="misp-events", + body={"query": {"term": {"uuid.keyword": str(event_2.uuid)}}}, ) - assert deleted is None + assert response_os["hits"]["total"]["value"] == 0 # ---- Tag/untag 404 cases ---- @@ -436,7 +407,7 @@ def test_tag_event_event_not_found( auth_token: auth.Token, ): response = client.post( - f"/events/999999/tag/{tlp_white_tag.name}", + f"/events/00000000-0000-0000-0000-000000000000/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -447,11 +418,11 @@ def test_tag_event_event_not_found( def test_tag_event_tag_not_found( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( - f"/events/{event_1.id}/tag/nonexistent:tag", + f"/events/{event_1.uuid}/tag/nonexistent:tag", headers={"Authorization": "Bearer " + auth_token}, ) @@ -466,7 +437,7 @@ def test_untag_event_event_not_found( auth_token: auth.Token, ): response = client.delete( - f"/events/999999/tag/{tlp_white_tag.name}", + f"/events/00000000-0000-0000-0000-000000000000/tag/{tlp_white_tag.name}", headers={"Authorization": "Bearer " + auth_token}, ) @@ -477,11 +448,11 @@ def test_untag_event_event_not_found( def test_untag_event_tag_not_found( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.delete( - f"/events/{event_1.id}/tag/nonexistent:tag", + f"/events/{event_1.uuid}/tag/nonexistent:tag", headers={"Authorization": "Bearer " + auth_token}, ) @@ -494,7 +465,7 @@ def test_untag_event_tag_not_found( def test_publish_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): # event_1 has published=False after test_update_event @@ -527,7 +498,7 @@ def test_publish_event_not_found( def test_unpublish_event( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): # event_1 was published by test_publish_event @@ -560,7 +531,7 @@ def test_unpublish_event_not_found( def test_toggle_correlation( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -592,7 +563,7 @@ def test_toggle_correlation_not_found( def test_import_data( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -642,7 +613,7 @@ def test_import_data_not_found( def test_force_index_by_uuid( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -655,43 +626,11 @@ def test_force_index_by_uuid( assert response.status_code == status.HTTP_202_ACCEPTED assert str(event_1.uuid) in data["message"] - @pytest.mark.parametrize("scopes", [["events:update"]]) - def test_force_index_by_id( - self, - client: TestClient, - event_1: event_models.Event, - auth_token: auth.Token, - ): - response = client.post( - "/events/force-index", - params={"id": event_1.id}, - headers={"Authorization": "Bearer " + auth_token}, - ) - data = response.json() - - assert response.status_code == status.HTTP_202_ACCEPTED - assert str(event_1.uuid) in data["message"] - - @pytest.mark.parametrize("scopes", [["events:update"]]) - def test_force_index_by_id_not_found( - self, - client: TestClient, - auth_token: auth.Token, - ): - response = client.post( - "/events/force-index", - params={"id": 999999}, - headers={"Authorization": "Bearer " + auth_token}, - ) - - assert response.status_code == status.HTTP_404_NOT_FOUND - assert response.json()["detail"] == "Event not found" - @pytest.mark.parametrize("scopes", [["events:update"]]) def test_force_index_all_events( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.post( @@ -709,7 +648,7 @@ def test_force_index_all_events( def test_get_event_attachments( self, client: TestClient, - event_1: event_models.Event, + event_1: object, auth_token: auth.Token, ): response = client.get( diff --git a/api/app/tests/api/test_feeds.py b/api/app/tests/api/test_feeds.py index f2c08023..9f267c2f 100644 --- a/api/app/tests/api/test_feeds.py +++ b/api/app/tests/api/test_feeds.py @@ -40,7 +40,7 @@ def test_get_feeds( assert data[0]["default"] == feed_1.default assert data[0]["fixed_event"] == feed_1.fixed_event assert data[0]["delta_merge"] == feed_1.delta_merge - assert data[0]["event_id"] == feed_1.event_id + assert data[0]["event_uuid"] == feed_1.event_uuid assert data[0]["publish"] == feed_1.publish assert data[0]["override_ids"] == feed_1.override_ids assert data[0]["settings"] == feed_1.settings diff --git a/api/app/tests/api/test_objects.py b/api/app/tests/api/test_objects.py index 765d671d..2b64216d 100644 --- a/api/app/tests/api/test_objects.py +++ b/api/app/tests/api/test_objects.py @@ -1,7 +1,5 @@ import pytest from app.auth import auth -from app.models import event as event_models -from app.models import object as object_models from app.tests.api_tester import ApiTester from fastapi import status from fastapi.testclient import TestClient @@ -12,7 +10,7 @@ class TestObjectsResource(ApiTester): def test_get_objects( self, client: TestClient, - object_1: object_models.Object, + object_1, auth_token: auth.Token, ): response = client.get( @@ -23,7 +21,7 @@ def test_get_objects( assert response.status_code == status.HTTP_200_OK assert len(data['items']) == 1 - assert data['items'][0]["id"] == object_1.id + assert data['items'][0]["uuid"] == str(object_1.uuid) assert data['items'][0]["name"] == object_1.name assert data['items'][0]["template_version"] == object_1.template_version assert data['items'][0]["timestamp"] == object_1.timestamp @@ -39,7 +37,7 @@ def test_get_objects_unauthorized(self, client: TestClient, auth_token: auth.Tok @pytest.mark.parametrize("scopes", [["objects:create"]]) def test_create_object( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/objects/", @@ -55,15 +53,15 @@ def test_create_object( data = response.json() assert response.status_code == status.HTTP_201_CREATED - assert data["id"] is not None - assert data["event_id"] == event_1.id + assert data["uuid"] is not None + assert data["event_uuid"] == str(event_1.uuid) assert data["template_version"] == 0 assert data["timestamp"] == 1655283899 assert data["deleted"] is False @pytest.mark.parametrize("scopes", [["objects:read"]]) def test_create_object_unauthorized( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): response = client.post( "/objects/", @@ -80,7 +78,7 @@ def test_create_object_unauthorized( @pytest.mark.parametrize("scopes", [["objects:create"]]) def test_create_object_incomplete( - self, client: TestClient, event_1: event_models.Event, auth_token: auth.Token + self, client: TestClient, event_1: object, auth_token: auth.Token ): # missing value response = client.post( @@ -96,11 +94,11 @@ def test_create_object_incomplete( def test_update_object( self, client: TestClient, - object_1: object_models.Object, + object_1, auth_token: auth.Token, ): response = client.patch( - f"/objects/{object_1.id}", + f"/objects/{object_1.uuid}", json={ "name": "updated via API", "comment": "test comment", @@ -117,11 +115,11 @@ def test_update_object( def test_delete_object( self, client: TestClient, - object_1: object_models.Object, + object_1, auth_token: auth.Token, ): response = client.delete( - f"/objects/{object_1.id}", + f"/objects/{object_1.uuid}", headers={"Authorization": "Bearer " + auth_token}, ) diff --git a/api/app/tests/api_tester.py b/api/app/tests/api_tester.py index c74efc5c..b4f6f612 100644 --- a/api/app/tests/api_tester.py +++ b/api/app/tests/api_tester.py @@ -5,14 +5,11 @@ from app.auth import auth from app.db.session import get_db from app.main import app -from app.models import attribute as attribute_models from app.models import event as event_models from app.models import feed as feed_models from app.models import galaxy as galaxy_models from app.models import hunt as hunt_models from app.models import module as module_models -from app.models import object as object_models -from app.models import object_reference as object_reference_models from app.models import organisation as organisation_models from app.models import server as server_models from app.models import sharing_groups as sharing_groups_models @@ -24,6 +21,7 @@ from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.orm import Session, declarative_base, sessionmaker +import sys class ApiTester: @@ -69,13 +67,7 @@ def teardown_db(self, db: Session): db.query(galaxy_models.GalaxyCluster).delete(synchronize_session=False) db.query(galaxy_models.Galaxy).delete(synchronize_session=False) db.query(feed_models.Feed).delete(synchronize_session=False) - db.query(tag_models.AttributeTag).delete(synchronize_session=False) - db.query(tag_models.EventTag).delete(synchronize_session=False) db.query(tag_models.Tag).delete(synchronize_session=False) - db.query(attribute_models.Attribute).delete(synchronize_session=False) - db.query(object_reference_models.ObjectReference).delete(synchronize_session=False) - db.query(object_models.Object).delete(synchronize_session=False) - db.query(event_models.Event).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupOrganisation).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroupServer).delete(synchronize_session=False) db.query(sharing_groups_models.SharingGroup).delete(synchronize_session=False) @@ -96,7 +88,29 @@ def cleanup(self, db: Session): try: pass finally: - # teardown + # clean OpenSearch docs left over from previous test classes + try: + from app.services.opensearch import get_opensearch_client + + os_client = get_opensearch_client() + for index in ("misp-events", "misp-attributes", "misp-objects"): + try: + os_client.delete_by_query( + index=index, + body={"query": {"match_all": {}}}, + refresh=True, + ignore=[404], + ) + except Exception as exc: + print( + f"Warning: failed to delete OpenSearch documents for index '{index}': {exc}", + file=sys.stderr, + ) + except Exception as exc: + print( + f"Warning: OpenSearch cleanup skipped due to error: {exc}", + file=sys.stderr, + ) self.teardown_db(db) # MISP data model fixtures @@ -157,69 +171,100 @@ def event_1( organisation_1: organisation_models.Organisation, user_1: user_models.User, ): - event_1 = event_models.Event( + from datetime import datetime + from uuid import UUID + from app.repositories import events as events_repository + from app.schemas import event as event_schemas + + event_create = event_schemas.EventCreate( info="test event", user_id=user_1.id, orgc_id=1, org_id=organisation_1.id, - date="2020-01-01", - uuid="ba4b11b6-dcce-4315-8fd0-67b69160ea76", + date=datetime(2020, 1, 1), + uuid=UUID("ba4b11b6-dcce-4315-8fd0-67b69160ea76"), timestamp=1577836800, ) - db.add(event_1) - db.commit() - db.refresh(event_1) + event_1 = events_repository.create_event(db=db, event=event_create) yield event_1 @pytest.fixture(scope="class") - def attribute_1(self, db: Session, event_1: event_models.Event): - attribute_1 = attribute_models.Attribute( - event_id=event_1.id, + def attribute_1(self, db: Session, event_1): + from uuid import UUID + from app.repositories import attributes as attributes_repository + from app.schemas import attribute as attribute_schemas + + attr_create = attribute_schemas.AttributeCreate( category="Network activity", type="ip-src", value="127.0.0.1", - uuid="7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0", + uuid=UUID("7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0"), timestamp=157783680, - ) - db.add(attribute_1) - db.commit() - db.refresh(attribute_1) - - yield attribute_1 - - @pytest.fixture(scope="class") - def object_1(self, db: Session, event_1: event_models.Event): - object_1 = object_models.Object( - event_id=event_1.id, - uuid="90e06ef6-26f8-40dd-9fb7-75897445e2a0", - name="test object", - template_version=0, - timestamp=1577836800, + event_uuid=event_1.uuid, deleted=False, + to_ids=False, + disable_correlation=False, ) - db.add(object_1) - db.commit() - db.refresh(object_1) + yield attributes_repository.create_attribute(db, attr_create) - yield object_1 + @pytest.fixture(scope="class") + def object_1(self, db: Session, event_1): + from datetime import datetime + from uuid import UUID as _UUID + from app.services.opensearch import get_opensearch_client + + obj_uuid = "90e06ef6-26f8-40dd-9fb7-75897445e2a0" + obj_doc = { + "uuid": obj_uuid, + "event_uuid": str(event_1.uuid), + "name": "test object", + "meta_category": None, + "template_uuid": None, + "template_version": 0, + "timestamp": 1577836800, + "@timestamp": datetime.fromtimestamp(1577836800).isoformat(), + "deleted": False, + "distribution": 0, + "sharing_group_id": None, + "first_seen": None, + "last_seen": None, + "comment": "", + "object_references": [], + } + + client = get_opensearch_client() + client.index(index="misp-objects", id=obj_uuid, body=obj_doc, refresh=True) + + from app.schemas import object as object_schemas + yield object_schemas.Object.model_validate(obj_doc) @pytest.fixture(scope="class") def object_attribute_1( - self, db: Session, event_1: event_models.Event, object_1: object_models.Object + self, db: Session, event_1, object_1 ): - object_attribute_1 = attribute_models.Attribute( - event_id=event_1.id, - object_id=object_1.id, + from uuid import UUID + from app.repositories import attributes as attributes_repository + from app.schemas import attribute as attribute_schemas + from app.services.opensearch import get_opensearch_client + + attr_create = attribute_schemas.AttributeCreate( category="Network activity", type="ip-src", value="127.0.0.2", - uuid="1355e435-aa0f-4f06-acd3-b44498131e82", + uuid=UUID("1355e435-aa0f-4f06-acd3-b44498131e82"), timestamp=1577836800, + event_uuid=event_1.uuid, + deleted=False, + ) + object_attribute_1 = attributes_repository.create_attribute(db, attr_create) + + get_opensearch_client().update( + index="misp-attributes", + id=str(object_attribute_1.uuid), + body={"doc": {"object_uuid": str(object_1.uuid)}}, + refresh=True, ) - db.add(object_attribute_1) - db.commit() - db.refresh(object_attribute_1) yield object_attribute_1 @@ -365,7 +410,7 @@ def feed_1(self, db: Session, organisation_1: organisation_models.Organisation): source_format="misp", fixed_event=False, delta_merge=False, - event_id=None, + event_uuid=None, publish=False, override_ids=False, settings=None, diff --git a/api/app/tests/repositories/test_feeds.py b/api/app/tests/repositories/test_feeds.py index 65c79655..62afe169 100644 --- a/api/app/tests/repositories/test_feeds.py +++ b/api/app/tests/repositories/test_feeds.py @@ -1,13 +1,15 @@ from unittest.mock import MagicMock, patch -from app.models import attribute as attribute_models -from app.models import event as event_models +from uuid import UUID + from app.models import feed as feed_models -from app.models import object as object_models -from app.models import object_reference as object_reference_models from app.models import tag as tag_models from app.models import user as user_models +from app.repositories import attributes as attributes_repository +from app.repositories import events as events_repository from app.repositories import feeds as feeds_repository +from app.repositories import object_references as object_references_repository +from app.repositories import objects as objects_repository from app.tests.api_tester import ApiTester from app.tests.scenarios import feed_fetch_scenarios from sqlalchemy.orm import Session @@ -43,86 +45,64 @@ def test_fetch_feed_by_id_new_event( ) # check that the events were created - events = ( - db.query(event_models.Event) - .filter( - event_models.Event.uuid - == feed_fetch_scenarios.feed_new_event["Event"]["uuid"] - ) - .all() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_new_event["Event"]["uuid"]) ) - assert len(events) == 1 + assert os_event is not None # check that the attributes were created - attributes = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid.in_( - [ - "317e63e6-b95d-4dd1-b4fd-de2f64f33fd8", - "8be7a04d-c10b-4ef6-854f-2072e67f6cd5", - ] - ) - ) - .all() - ) - assert len(attributes) == 2 + attributes = [ + attributes_repository.get_attribute_from_opensearch(UUID(u)) + for u in [ + "317e63e6-b95d-4dd1-b4fd-de2f64f33fd8", + "8be7a04d-c10b-4ef6-854f-2072e67f6cd5", + ] + ] + assert len([a for a in attributes if a is not None]) == 2 # check the objects were created - objects = ( - db.query(object_models.Object) - .filter( - object_models.Object.uuid == "df23d3be-1179-4824-ac03-471f0bc6d92d" - ) - .all() + obj = objects_repository.get_object_from_opensearch( + UUID("df23d3be-1179-4824-ac03-471f0bc6d92d") ) - assert len(objects) == 1 + assert obj is not None # check the object references were created - object_references = ( - db.query(object_reference_models.ObjectReference) - .filter( - object_reference_models.ObjectReference.uuid - == "d7e57f39-4dd5-4b87-b040-75561fa8289e" - ) - .all() + object_reference = object_references_repository.get_object_reference_by_uuid( + db, UUID("d7e57f39-4dd5-4b87-b040-75561fa8289e") ) - assert len(object_references) == 1 + assert object_reference is not None # check the tags were created tags = db.query(tag_models.Tag).all() assert len(tags) == 4 # check the event tags were created - event_tags = ( - db.query(tag_models.Tag) - .join(tag_models.EventTag) - .filter( - tag_models.Tag.name.in_(["type:OSINT", "tlp:clear", "tlp:white"]), - ) - .all() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_new_event["Event"]["uuid"]) ) - assert len(event_tags) == 3 + event_tag_names = {t.name for t in (os_event.tags or [])} + assert {"type:OSINT", "tlp:clear", "tlp:white"}.issubset(event_tag_names) # check the attribute tags were created - attribute_tags = ( - db.query(tag_models.Tag) - .join(tag_models.AttributeTag) - .filter( - tag_models.Tag.name.in_(["tlp:red"]), - ) - .all() - ) - assert len(attribute_tags) == 1 + all_attribute_tag_names = set() + for uuid in [ + "317e63e6-b95d-4dd1-b4fd-de2f64f33fd8", + "8be7a04d-c10b-4ef6-854f-2072e67f6cd5", + ]: + attr = attributes_repository.get_attribute_from_opensearch(UUID(uuid)) + if attr: + for t in (attr.tags or []): + all_attribute_tag_names.add(t.name) + assert "tlp:red" in all_attribute_tag_names def test_fetch_feed_by_id_existing_event( self, db: Session, feed_1: feed_models.Feed, - event_1: event_models.Event, - attribute_1: attribute_models.Attribute, - object_1: object_models.Object, - object_attribute_1: attribute_models.Attribute, + event_1, + attribute_1, + object_1, + object_attribute_1, user_1: user_models.User, ): # mock remote Feed API calls @@ -149,60 +129,39 @@ def test_fetch_feed_by_id_existing_event( ) # check that the events was updated - event = ( - db.query(event_models.Event) - .filter( - event_models.Event.uuid - == feed_fetch_scenarios.feed_update_event["Event"]["uuid"] - ) - .first() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_update_event["Event"]["uuid"]) ) - assert event.info == "Updated by Feed fetch" - assert event.timestamp == 1577836801 + assert os_event is not None + assert os_event.info == "Updated by Feed fetch" + assert os_event.timestamp == 1577836801 # check that the attribute was updated - attribute = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid - == "7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0" - ) - .first() + attribute = attributes_repository.get_attribute_from_opensearch( + UUID("7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0") ) + assert attribute is not None assert attribute.value == "7edc546f741eff3e13590a62ce2856bb39d8f71d" assert attribute.timestamp == 1577836801 # check the object was updated - object = ( - db.query(object_models.Object) - .filter( - object_models.Object.uuid == "90e06ef6-26f8-40dd-9fb7-75897445e2a0" - ) - .first() + object = objects_repository.get_object_from_opensearch( + UUID("90e06ef6-26f8-40dd-9fb7-75897445e2a0") ) assert object.comment == "Object comment updated by Feed fetch" assert object.timestamp == 1577836801 # check the object attribute was added - object_attribute = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid - == "011aca4f-eaf0-4a06-8133-b69f3806cbe8" - ) - .first() + object_attribute = attributes_repository.get_attribute_from_opensearch( + UUID("011aca4f-eaf0-4a06-8133-b69f3806cbe8") ) + assert object_attribute is not None assert object_attribute.value == "Foobar12345" assert object_attribute.timestamp == 1577836801 # check the object references were created - object_reference = ( - db.query(object_reference_models.ObjectReference) - .filter( - object_reference_models.ObjectReference.uuid - == "4d4c12b9-e514-496e-a8a6-06d5c6815b97" - ) - .first() + object_reference = object_references_repository.get_object_reference_by_uuid( + db, UUID("4d4c12b9-e514-496e-a8a6-06d5c6815b97") ) assert ( str(object_reference.referenced_uuid) @@ -210,28 +169,21 @@ def test_fetch_feed_by_id_existing_event( ) # check the event tags were created - event_tags = ( - db.query(tag_models.Tag) - .join(tag_models.EventTag) - .filter( - tag_models.Tag.name.in_(["EVENT_FEED_ADDED_TAG"]), - ) - .all() + os_event = events_repository.get_event_from_opensearch( + UUID(feed_fetch_scenarios.feed_update_event["Event"]["uuid"]) ) - assert len(event_tags) == 1 + event_tag_names = {t.name for t in (os_event.tags or [])} + assert "EVENT_FEED_ADDED_TAG" in event_tag_names # check the attribute tags were created - attribute_tags = ( - db.query(tag_models.Tag) - .join(tag_models.AttributeTag) - .filter( - tag_models.Tag.name.in_( - [ - "ATTRIBUTE_EVENT_FEED_ADDED_TAG", - "OBJECT_ATTRIBUTE_EVENT_FEED_ADDED_TAG", - ] - ), - ) - .all() - ) - assert len(attribute_tags) == 2 + all_attribute_tag_names = set() + for uuid in [ + "7f2fd15d-3c63-47ba-8a39-2c4b0b3314b0", + "011aca4f-eaf0-4a06-8133-b69f3806cbe8", + ]: + attr = attributes_repository.get_attribute_from_opensearch(UUID(uuid)) + if attr: + for t in (attr.tags or []): + all_attribute_tag_names.add(t.name) + assert "ATTRIBUTE_EVENT_FEED_ADDED_TAG" in all_attribute_tag_names + assert "OBJECT_ATTRIBUTE_EVENT_FEED_ADDED_TAG" in all_attribute_tag_names diff --git a/api/app/tests/repositories/test_servers.py b/api/app/tests/repositories/test_servers.py index ae198da7..c1f4ff7f 100644 --- a/api/app/tests/repositories/test_servers.py +++ b/api/app/tests/repositories/test_servers.py @@ -1,20 +1,21 @@ from unittest.mock import MagicMock, patch import pytest -from app.models import attribute as attribute_models -from app.models import event as event_models -from app.models import object as object_models -from app.models import object_reference as object_reference_models +from uuid import UUID + +from app.repositories import attributes as attributes_repository +from app.repositories import object_references as object_references_repository +from app.repositories import objects as objects_repository from app.models import organisation as organisations_models from app.models import server as server_models from app.models import sharing_groups as sharing_groups_models from app.models import tag as tag_models from app.models import user as user_models +from app.repositories import events as events_repository from app.repositories import servers as servers_repository from app.settings import Settings from app.tests.api_tester import ApiTester from app.tests.scenarios import server_pull_scenarios -from sqlalchemy import and_ from sqlalchemy.orm import Session @@ -34,8 +35,6 @@ def test_pull_server_by_id( scenario: dict, ): # clear the database - db.query(tag_models.AttributeTag).delete() - db.query(tag_models.EventTag).delete() db.query(tag_models.Tag).delete() # mock remote MISP API calls @@ -69,53 +68,37 @@ def test_pull_server_by_id( ) # check that the events were created - events = ( - db.query(event_models.Event) - .filter( - event_models.Event.uuid.in_( - scenario["expected_result"]["event_uuids"] - ) - ) - .all() - ) - assert len(events) == len(scenario["expected_result"]["event_uuids"]) + os_events = [ + events_repository.get_event_from_opensearch(UUID(uuid)) + for uuid in scenario["expected_result"]["event_uuids"] + ] + os_events = [e for e in os_events if e is not None] + assert len(os_events) == len(scenario["expected_result"]["event_uuids"]) # check that the attributes were created - attributes = ( - db.query(attribute_models.Attribute) - .filter( - attribute_models.Attribute.uuid.in_( - scenario["expected_result"]["attribute_uuids"] - ) - ) - .all() - ) + attributes = [ + attributes_repository.get_attribute_from_opensearch(UUID(u)) + for u in scenario["expected_result"]["attribute_uuids"] + ] + attributes = [a for a in attributes if a is not None] assert len(attributes) == len( scenario["expected_result"]["attribute_uuids"] ) # check the objects were created - objects = ( - db.query(object_models.Object) - .filter( - object_models.Object.uuid.in_( - scenario["expected_result"]["object_uuids"] - ) - ) - .all() - ) + objects = [ + objects_repository.get_object_from_opensearch(UUID(uuid)) + for uuid in scenario["expected_result"]["object_uuids"] + ] + objects = [o for o in objects if o is not None] assert len(objects) == len(scenario["expected_result"]["object_uuids"]) # check the object references were created - object_references = ( - db.query(object_reference_models.ObjectReference) - .filter( - object_reference_models.ObjectReference.uuid.in_( - scenario["expected_result"]["object_reference_uuids"] - ) - ) - .all() - ) + object_references = [ + object_references_repository.get_object_reference_by_uuid(db, UUID(uuid)) + for uuid in scenario["expected_result"]["object_reference_uuids"] + ] + object_references = [r for r in object_references if r is not None] assert len(object_references) == len( scenario["expected_result"]["object_reference_uuids"] ) @@ -154,30 +137,18 @@ def test_pull_server_by_id( assert len(tags) == len(scenario["expected_result"]["tags"]) # check the event tags were created - event_tags = ( - db.query(tag_models.Tag) - .join(tag_models.EventTag) - .filter( - tag_models.Tag.name.in_(scenario["expected_result"]["event_tags"]) - ) - .all() - ) - assert len(event_tags) == len(scenario["expected_result"]["event_tags"]) + event_tag_names = set() + for event in os_events: + for t in (event.tags or []): + event_tag_names.add(t.name) + for tag_name in scenario["expected_result"]["event_tags"]: + assert tag_name in event_tag_names # check the attribute tags were created + all_attribute_tag_names = set() + for attr in attributes: + for t in (attr.tags or []): + all_attribute_tag_names.add(t.name) for attribute_tag in scenario["expected_result"]["attribute_tags"]: - attribute_tags = ( - db.query(tag_models.Tag) - .join(tag_models.AttributeTag) - .filter( - and_( - tag_models.Tag.name.in_(attribute_tag["tags"]), - attribute_models.Attribute.uuid - == attribute_tag["attribute_uuid"], - attribute_models.Attribute.id - == tag_models.AttributeTag.attribute_id, - ) - ) - .all() - ) - assert len(attribute_tags) == len(attribute_tag["tags"]) + for tag_name in attribute_tag["tags"]: + assert tag_name in all_attribute_tag_names diff --git a/api/app/worker/tasks.py b/api/app/worker/tasks.py index b927253d..2e4940ea 100644 --- a/api/app/worker/tasks.py +++ b/api/app/worker/tasks.py @@ -3,6 +3,7 @@ import smtplib from email.message import EmailMessage from datetime import datetime +from uuid import UUID from app.database import SQLALCHEMY_DATABASE_URL from app.services.opensearch import get_opensearch_client @@ -21,13 +22,10 @@ from app.repositories import galaxies as galaxies_repository from app.repositories import hunts as hunts_repository from app.repositories import taxonomies as taxonomies_repository -from app.schemas import event as event_schemas from app.schemas import attribute as attribute_schemas -from app.models import event as event_models from celery import Celery from sqlalchemy import create_engine from sqlalchemy.orm import Session -from opensearchpy import helpers as opensearch_helpers # Celery configuration celery_app = Celery("misp-workbench") @@ -142,14 +140,10 @@ def push_event_by_uuid(event_uuid: str, server_id: int, user_id: int): def handle_created_event(event_uuid: str): logger.info("handling created event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - notifications_repository.create_event_notifications( - db, "created", event=db_event - ) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "created", event=os_event) return True @@ -158,21 +152,16 @@ def handle_created_event(event_uuid: str): def handle_updated_event(event_uuid: str): logger.info("handling updated event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - db_event.timestamp = datetime.now().timestamp() - db.commit() - db.refresh(db_event) - - notifications_repository.create_event_notifications( - db, "updated", event=db_event + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + get_opensearch_client().update( + index="misp-events", + id=event_uuid, + body={"doc": {"timestamp": int(datetime.now().timestamp())}}, + refresh=True, ) - - index_event.delay(str(db_event.uuid), full_reindex=False) + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "updated", event=os_event) return True @@ -181,113 +170,93 @@ def handle_updated_event(event_uuid: str): def handle_deleted_event(event_uuid: str): logger.info("handling deleted event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "deleted", event=os_event) - notifications_repository.create_event_notifications( - db, "deleted", event=db_event - ) - - delete_indexed_event.delay(str(event_uuid)) + delete_indexed_event(event_uuid) return True @celery_app.task -def handle_created_attribute(attribute_id: int, object_id: int | None, event_id: int): - logger.info("handling created attribute id=%s job started", attribute_id) +def handle_created_attribute(attribute_uuid: str, object_uuid, event_uuid: str | None): + logger.info("handling created attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - if object_id is None: - events_repository.increment_attribute_count(db, event_id) - - db_attribute = attributes_repository.get_attribute_by_id(db, attribute_id) - notifications_repository.create_attribute_notifications( - db, "created", attribute=db_attribute - ) + if object_uuid is None and event_uuid: + events_repository.increment_attribute_count(db, event_uuid) - index_attribute.delay(str(db_attribute.uuid)) + os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) + if os_attr is not None: + notifications_repository.create_attribute_notifications(db, "created", attribute=os_attr) return True @celery_app.task -def handle_updated_attribute(attribute_id: int, object_id: int | None, event_id: int): - logger.info("handling updated attribute id=%s job started", attribute_id) +def handle_updated_attribute(attribute_uuid: str, object_uuid, event_uuid: str | None): + logger.info("handling updated attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - db_attribute = attributes_repository.get_attribute_by_id(db, attribute_id) - notifications_repository.create_attribute_notifications( - db, "updated", attribute=db_attribute - ) - - index_attribute.delay(str(db_attribute.uuid)) + os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) + if os_attr is not None: + notifications_repository.create_attribute_notifications(db, "updated", attribute=os_attr) return True @celery_app.task -def handle_deleted_attribute(attribute_id: int, object_id: int | None, event_id: int): - logger.info("handling deleted attribute id=%s job started", attribute_id) +def handle_deleted_attribute(attribute_uuid: str, object_uuid, event_uuid: str | None): + logger.info("handling deleted attribute uuid=%s job started", attribute_uuid) with Session(engine) as db: - db_attribute = attributes_repository.get_attribute_by_id(db, attribute_id) - if object_id is None: - events_repository.decrement_attribute_count(db, event_id) - - notifications_repository.create_attribute_notifications( - db, "deleted", attribute=db_attribute - ) + if object_uuid is None and event_uuid: + events_repository.decrement_attribute_count(db, event_uuid) - delete_indexed_attribute.delay(str(db_attribute.uuid)) + os_attr = attributes_repository.get_attribute_from_opensearch(UUID(attribute_uuid)) + if os_attr is not None: + notifications_repository.create_attribute_notifications(db, "deleted", attribute=os_attr) return True @celery_app.task -def handle_created_object(object_id: int, event_id: int): - logger.info("handling created object id=%s job started", object_id) +def handle_created_object(object_uuid: str, event_uuid: str | None): + logger.info("handling created object uuid=%s job started", object_uuid) with Session(engine) as db: - events_repository.increment_object_count(db, event_id) - - db_object = objects_repository.get_object_by_id(db, object_id) - notifications_repository.create_object_notifications( - db, "created", object=db_object - ) + if event_uuid: + events_repository.increment_object_count(db, event_uuid) - index_object.delay(str(db_object.uuid)) + os_obj = objects_repository.get_object_from_opensearch(UUID(object_uuid)) + if os_obj is not None: + notifications_repository.create_object_notifications(db, "created", object=os_obj) return True @celery_app.task -def handle_updated_object(object_id: int, event_id: int): - logger.info("handling updated object id=%s job started", object_id) +def handle_updated_object(object_uuid: str, event_uuid: str | None): + logger.info("handling updated object uuid=%s job started", object_uuid) with Session(engine) as db: - db_object = objects_repository.get_object_by_id(db, object_id) - notifications_repository.create_object_notifications( - db, "updated", object=db_object - ) - - index_object.delay(str(db_object.uuid)) + os_obj = objects_repository.get_object_from_opensearch(UUID(object_uuid)) + if os_obj is not None: + notifications_repository.create_object_notifications(db, "updated", object=os_obj) return True @celery_app.task -def handle_deleted_object(object_id: int, event_id: int): - logger.info("handling deleted object id=%s job started", object_id) +def handle_deleted_object(object_uuid: str, event_uuid: str | None): + logger.info("handling deleted object uuid=%s job started", object_uuid) with Session(engine) as db: - events_repository.decrement_object_count(db, event_id) - - db_object = objects_repository.get_object_by_id(db, object_id) - notifications_repository.create_object_notifications( - db, "deleted", object=db_object - ) + if event_uuid: + events_repository.decrement_object_count(db, event_uuid) - delete_indexed_object.delay(str(db_object.uuid)) + os_obj = objects_repository.get_object_from_opensearch(UUID(object_uuid)) + if os_obj is not None: + notifications_repository.create_object_notifications(db, "deleted", object=os_obj) return True @@ -322,163 +291,6 @@ def send_email(email: dict): return True -@celery_app.task -def index_event(event_uuid: str, full_reindex: bool = False): - logger.info("index event uuid=%s job started", event_uuid) - - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - event = event_schemas.Event.model_validate(db_event) - - OpenSearchClient = get_opensearch_client() - - event_raw = event.model_dump() - attributes = event_raw.pop("attributes") - objects = event_raw.pop("objects") - - # convert timestamp to datetime so it can be indexed - event_raw["@timestamp"] = datetime.fromtimestamp(event_raw["timestamp"]).isoformat() - - if event_raw["publish_timestamp"]: - event_raw["@publish_timestamp"] = datetime.fromtimestamp( - event_raw["publish_timestamp"] - ).isoformat() - - response = OpenSearchClient.index( - index="misp-events", id=event.uuid, body=event_raw, refresh=True - ) - - if response["result"] not in ["created", "updated"]: - logger.error( - "Failed to index event uuid=%s. Response: %s", event_uuid, response - ) - raise Exception("Failed to index event.") - - logger.info("indexed event uuid=%s", event_uuid) - - if not full_reindex: - return True - - # delete existing indexed attributes and objects - query = {"query": {"bool": {"must": [{"term": {"event_uuid": str(event.uuid)}}]}}} - - response = OpenSearchClient.delete_by_query( - index="misp-attributes", body=query, refresh=True - ) - logger.info( - "deleted %s previously indexed attributes for event uuid=%s", - response["deleted"], - event_uuid, - ) - - response = OpenSearchClient.delete_by_query( - index="misp-objects", body=query, refresh=True - ) - logger.info( - "deleted %s previously indexed objects for event uuid=%s", - response["deleted"], - event_uuid, - ) - - # index attributes - attributes_docs = [] - for attribute in attributes: - attribute["@timestamp"] = datetime.fromtimestamp( - attribute["timestamp"] - ).isoformat() - attribute["event_uuid"] = event.uuid - attribute["data"] = "" # do not index file contents - - attributes_docs.append( - { - "_id": attribute["uuid"], - "_index": "misp-attributes", - "_source": attribute, - } - ) - - success, failed = opensearch_helpers.bulk( - OpenSearchClient, attributes_docs, refresh=True - ) - if failed: - logger.error("Failed to index attributes. Failed: %s", failed) - raise Exception("Failed to index attributes.") - logger.info( - "indexed attributes of event uuid=%s, indexed %s attributes", - event_uuid, - len(attributes_docs), - ) - - # index objects - objects_docs = [] - for object in objects: - object["@timestamp"] = datetime.fromtimestamp(object["timestamp"]).isoformat() - - object_attributes = object.pop("attributes") - - object_attributes_docs = [] - for attribute in object_attributes: - attribute["@timestamp"] = datetime.fromtimestamp( - attribute["timestamp"] - ).isoformat() - attribute["event_uuid"] = event.uuid - attribute["object_uuid"] = object["uuid"] - attribute["data"] = "" # do not index file contents - object_attributes_docs.append( - { - "_id": attribute["uuid"], - "_index": "misp-attributes", - "_source": attribute, - } - ) - - success, failed = opensearch_helpers.bulk( - OpenSearchClient, object_attributes_docs, refresh=True - ) - if failed: - logger.error("Failed to index object attributes. Failed: %s", failed) - raise Exception("Failed to index object attributes.") - logger.info( - "indexed attributes of event uuid=%s, object uuid=%s, indexed %s attributes", - event_uuid, - object["uuid"], - len(object_attributes_docs), - ) - - for reference in object["object_references"]: - reference["@timestamp"] = datetime.fromtimestamp( - reference["timestamp"] - ).isoformat() - - object["event_uuid"] = event.uuid - - objects_docs.append( - { - "_id": object["uuid"], - "_index": "misp-objects", - "_source": object, - } - ) - - success, failed = opensearch_helpers.bulk( - OpenSearchClient, objects_docs, refresh=True - ) - if failed: - logger.error("Failed to index objects. Failed: %s", failed) - raise Exception("Failed to index objects.") - logger.info( - "indexed objects of event uuid=%s, indexed %s objects", - event_uuid, - len(objects_docs), - ) - - logger.info("index event uuid=%s job finished", event_uuid) - - return True - @celery_app.task def fetch_feed(feed_id: int, user_id: int): logger.info("fetch feed id=%s job started", feed_id) @@ -537,7 +349,7 @@ def fetch_csv_feed(feed_id: int, user_id: int): attribute = feeds_repository.process_csv_feed_row(row, db_feed.settings) db_attribute = attribute_schemas.AttributeCreate( - event_id=db_event.id, + event_uuid=db_event.uuid, type=attribute["type"], value=attribute["value"], category=attribute.get("category", "External analysis"), @@ -561,8 +373,6 @@ def fetch_csv_feed(feed_id: int, user_id: int): index += 1 - index_event.delay(str(db_event.uuid), full_reindex=True) - logger.info("fetch csv feed id=%s job finished", feed_id) return { @@ -608,7 +418,7 @@ def fetch_freetext_feed(feed_id: int, user_id: int): attr_type = freetext_repository.detect_type(value) db_attribute = attribute_schemas.AttributeCreate( - event_id=db_event.id, + event_uuid=db_event.uuid, type=attr_type, value=value, category="External analysis", @@ -620,8 +430,6 @@ def fetch_freetext_feed(feed_id: int, user_id: int): failed_rows += 1 logger.error("Error processing freetext feed line: %s", e) - index_event.delay(str(db_event.uuid), full_reindex=True) - logger.info("fetch freetext feed id=%s job finished", feed_id) return { @@ -663,7 +471,7 @@ def fetch_json_feed(feed_id: int, user_id: int): continue db_attribute = attribute_schemas.AttributeCreate( - event_id=db_event.id, + event_uuid=db_event.uuid, type=attribute["type"], value=attribute["value"], category=attribute.get("category", "External analysis"), @@ -682,8 +490,6 @@ def fetch_json_feed(feed_id: int, user_id: int): failed_items += 1 logger.error("Error processing JSON feed item: %s", e) - index_event.delay(str(db_event.uuid), full_reindex=True) - logger.info("fetch json feed id=%s job finished", feed_id) return { @@ -767,7 +573,7 @@ def handle_created_sighting( "value": value, "type": sighting_type, "observer": {"organisation": organisation}, - "timestamp": timestamp or datetime.datetime.now().timestamp(), + "timestamp": timestamp or datetime.now().timestamp(), } with Session(engine) as db: @@ -809,17 +615,12 @@ def handle_created_correlation( def handle_published_event(event_uuid: str): logger.info("handling published event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - notifications_repository.create_event_notifications( - db, "published", event=db_event - ) - - logger.info("handling published event uuid=%s job finished", event_uuid) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "published", event=os_event) + logger.info("handling published event uuid=%s job finished", event_uuid) return True @@ -827,17 +628,12 @@ def handle_published_event(event_uuid: str): def handle_unpublished_event(event_uuid: str): logger.info("handling unpublished event uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - notifications_repository.create_event_notifications( - db, "unpublished", event=db_event - ) - - logger.info("handling unpublished event uuid=%s job finished", event_uuid) + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is not None: + with Session(engine) as db: + notifications_repository.create_event_notifications(db, "unpublished", event=os_event) + logger.info("handling unpublished event uuid=%s job finished", event_uuid) return True @@ -845,27 +641,20 @@ def handle_unpublished_event(event_uuid: str): def handle_toggled_event_correlation(event_uuid: str, disable_correlation: bool): logger.info("handling toggled event correlation uuid=%s job started", event_uuid) - with Session(engine) as db: - db_event = events_repository.get_event_by_uuid(db, event_uuid) - if db_event is None: - raise Exception("Event with uuid=%s not found", event_uuid) - - if disable_correlation: - correlations_repository.delete_event_correlations(event_uuid) - else: - with Session(engine) as db: - runtimeSettings = get_runtime_settings(db) - - correlations_repository.correlate_event( - runtimeSettings, str(event_uuid) - ) - - run_correlation_hunts.delay() + os_event = events_repository.get_event_from_opensearch(UUID(event_uuid)) + if os_event is None: + logger.warning("handle_toggled_event_correlation: event %s not found", event_uuid) + return True - logger.info( - "handling toggled event correlation uuid=%s job finished", event_uuid - ) + if disable_correlation: + correlations_repository.delete_event_correlations(event_uuid) + else: + with Session(engine) as db: + runtimeSettings = get_runtime_settings(db) + correlations_repository.correlate_event(runtimeSettings, str(event_uuid)) + run_correlation_hunts.delay() + logger.info("handling toggled event correlation uuid=%s job finished", event_uuid) return True @@ -888,21 +677,21 @@ def delete_indexed_event(event_uuid: str): query = {"query": {"bool": {"must": [{"term": {"event_uuid": str(event_uuid)}}]}}} response = OpenSearchClient.delete_by_query( - index="misp-attributes", body=query, refresh=True + index="misp-attributes", body=query, refresh=True, ignore=[404] ) logger.info( "deleted %s indexed attributes for event uuid=%s", - response["deleted"], + response.get("deleted", 0), event_uuid, ) # delete indexed objects response = OpenSearchClient.delete_by_query( - index="misp-objects", body=query, refresh=True + index="misp-objects", body=query, refresh=True, ignore=[404] ) logger.info( "deleted %s indexed objects for event uuid=%s", - response["deleted"], + response.get("deleted", 0), event_uuid, ) @@ -911,55 +700,6 @@ def delete_indexed_event(event_uuid: str): return True -@celery_app.task -def index_attribute(attribute_uuid: str): - logger.info("indexing attribute uuid=%s job started", attribute_uuid) - - with Session(engine) as db: - db_attribute = attributes_repository.get_attribute_by_uuid(db, attribute_uuid) - if db_attribute is None: - raise Exception("Attribute with uuid=%s not found", attribute_uuid) - - if db_attribute.event and db_attribute.event.deleted: - logger.info( - "skipping indexing attribute uuid=%s, event is deleted", attribute_uuid - ) - return True - - attribute = event_schemas.Attribute.model_validate(db_attribute) - event_uuid = db_attribute.event.uuid if db_attribute.event else None - - OpenSearchClient = get_opensearch_client() - - attribute_raw = attribute.model_dump() - attribute_raw["event_uuid"] = str(event_uuid) if event_uuid else None - - # convert timestamp to datetime so it can be indexed - attribute_raw["@timestamp"] = datetime.fromtimestamp( - attribute_raw["timestamp"] - ).isoformat() - attribute_raw["data"] = "" # do not index file contents - - response = OpenSearchClient.index( - index="misp-attributes", - id=attribute.uuid, - body=attribute_raw, - refresh=True, - ) - - if response["result"] not in ["created", "updated"]: - logger.error( - "Failed to index attribute uuid=%s. Response: %s", - attribute_uuid, - response, - ) - raise Exception("Failed to index attribute.") - - logger.info("indexed attribute uuid=%s job finished", attribute_uuid) - - return True - - @celery_app.task def delete_indexed_attribute(attribute_uuid: str): logger.info("deleting indexed attribute uuid=%s job started", attribute_uuid) @@ -992,8 +732,12 @@ def load_galaxies(user_id: int): logger.info("load_galaxies job started") with Session(engine) as db: + user = None if user_id is not None: user = users_repository.get_user_by_id(db, user_id) + if user is None: + first = users_repository.get_users(db, skip=0, limit=1) + user = first[0] if first else None if user is None: logger.error("No user found to run load_galaxies; aborting") @@ -1027,45 +771,6 @@ def load_taxonomies(): return False -@celery_app.task -def index_object(object_uuid: str): - logger.info("indexing object uuid=%s job started", object_uuid) - - with Session(engine) as db: - db_object = objects_repository.get_object_by_uuid(db, object_uuid) - if db_object is None: - raise Exception("Object with uuid=%s not found", object_uuid) - - object = event_schemas.Object.model_validate(db_object) - - OpenSearchClient = get_opensearch_client() - - object_raw = object.model_dump() - - # convert timestamp to datetime so it can be indexed - object_raw["@timestamp"] = datetime.fromtimestamp( - object_raw["timestamp"] - ).isoformat() - object_raw["event_uuid"] = str(db_object.event.uuid) - - response = OpenSearchClient.index( - index="misp-objects", - id=object.uuid, - body=object_raw, - refresh=True, - ) - - if response["result"] not in ["created", "updated"]: - logger.error( - "Failed to index object uuid=%s. Response: %s", object_uuid, response - ) - raise Exception("Failed to index object.") - - logger.info("indexed object uuid=%s job finished", object_uuid) - - return True - - @celery_app.task def delete_indexed_object(object_uuid: str): logger.info("deleting indexed object uuid=%s job started", object_uuid) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d90799ec..6807ca0e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -32,6 +32,7 @@ services: retries: 5 api: + mem_limit: 4g build: context: ./api dockerfile: Dockerfile diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 91c8b7b8..592ffaad 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,8 +1,7 @@ services: api: - mem_limit: 2g - memswap_limit: 2g + mem_limit: 4g volumes: - ./api:/code ports: diff --git a/docs/features/enrichments.md b/docs/features/enrichments.md index 677fb633..72c8ce2c 100644 --- a/docs/features/enrichments.md +++ b/docs/features/enrichments.md @@ -82,7 +82,7 @@ Click **query** on an enabled module to send a test query directly from the sett 6. Use the checkboxes to select which results to keep, or click **Select All Enrichments**. 7. Click **Add** to create the selected attributes and objects in the event. -New attributes inherit the source attribute's `event_id`, `distribution`, and `sharing_group_id`. +New attributes inherit the source attribute's `event_uuid`, `distribution`, and `sharing_group_id`. ## API reference diff --git a/frontend/src/components/attachments/AttachmentIcon.vue b/frontend/src/components/attachments/AttachmentIcon.vue index c989a487..4e522b17 100644 --- a/frontend/src/components/attachments/AttachmentIcon.vue +++ b/frontend/src/components/attachments/AttachmentIcon.vue @@ -206,7 +206,7 @@ async function downloadAttachment() { :id="`deleteObjectModal_${attachment.id}`" @object-deleted="handleObjectDeleted" :modal="deleteObjectModal" - :object_id="attachment.id" + :object_uuid="attachment.uuid" /> diff --git a/frontend/src/components/attachments/UploadAttachmentsWidget.vue b/frontend/src/components/attachments/UploadAttachmentsWidget.vue index b538ff15..1d2c5008 100644 --- a/frontend/src/components/attachments/UploadAttachmentsWidget.vue +++ b/frontend/src/components/attachments/UploadAttachmentsWidget.vue @@ -86,8 +86,8 @@ function handleAttachmentDeleted(attachment_id) { ); } -function handleObjectDeleted(object_id) { - emit("object-deleted", object_id); +function handleObjectDeleted(object_uuid) { + emit("object-deleted", object_uuid); } diff --git a/frontend/src/components/attributes/AttributeActions.vue b/frontend/src/components/attributes/AttributeActions.vue index fcbcb8b9..31bacc61 100644 --- a/frontend/src/components/attributes/AttributeActions.vue +++ b/frontend/src/components/attributes/AttributeActions.vue @@ -65,13 +65,13 @@ const { modulesResponses } = storeToRefs(modulesStore); onMounted(() => { deleteAttributeModal.value = new Modal( - document.getElementById(`deleteAttributeModal_${props.attribute.id}`), + document.getElementById(`deleteAttributeModal_${props.attribute.uuid}`), ); enrichAttributeModal.value = new Modal( - document.getElementById(`enrichAttributeModal_${props.attribute.id}`), + document.getElementById(`enrichAttributeModal_${props.attribute.uuid}`), ); correlatedAttributesModal.value = new Modal( - document.getElementById(`correlatedAttributesModal${props.attribute.id}`), + document.getElementById(`correlatedAttributesModal${props.attribute.uuid}`), ); followed.value = isFollowingEntity("attributes", props.attribute.uuid); }); @@ -90,11 +90,11 @@ function openCorrelationsModal() { } function handleAttributeDeleted() { - emit("attribute-deleted", props.attribute.id); + emit("attribute-deleted", props.attribute.uuid); } function handleAttributeEnriched() { - emit("attribute-enriched", props.attribute.id); + emit("attribute-enriched", props.attribute.uuid); } function handleObjectCreated(object) { @@ -190,7 +190,7 @@ function followAttribute() { @@ -228,7 +228,7 @@ function followAttribute() {
  • - + View @@ -237,7 +237,7 @@ function followAttribute() {
  • Update @@ -279,16 +279,16 @@ function followAttribute() { diff --git a/frontend/src/components/attributes/AttributeUpdate.vue b/frontend/src/components/attributes/AttributeUpdate.vue index 370a5c9c..97de18f5 100644 --- a/frontend/src/components/attributes/AttributeUpdate.vue +++ b/frontend/src/components/attributes/AttributeUpdate.vue @@ -15,7 +15,7 @@ function onSubmit(values, { setErrors }) { return attributesStore .update(values.attribute) .then(() => { - router.push(`/attributes/${values.attribute.id}`); + router.push(`/attributes/${values.attribute.uuid}`); }) .catch((error) => setErrors({ apiError: error })); } @@ -48,19 +48,6 @@ function handleDistributionLevelUpdated(distributionLevelId) { :validation-schema="AttributeSchema" v-slot="{ errors }" > -
    - - - -
    {{ errors["attribute.id"] }}
    -
    - - - - - +
    id{{ attribute.id }}
    uuid @@ -166,7 +162,7 @@ div.row h3 { diff --git a/frontend/src/components/attributes/AttributesIndex.vue b/frontend/src/components/attributes/AttributesIndex.vue index 8ef7e9b0..f2e5c648 100644 --- a/frontend/src/components/attributes/AttributesIndex.vue +++ b/frontend/src/components/attributes/AttributesIndex.vue @@ -87,7 +87,7 @@ function handleObjectCreated(object) {
    {{ attribute.value }} diff --git a/frontend/src/components/attributes/DeleteAttributeModal.vue b/frontend/src/components/attributes/DeleteAttributeModal.vue index 0e20ebb4..72ba8682 100644 --- a/frontend/src/components/attributes/DeleteAttributeModal.vue +++ b/frontend/src/components/attributes/DeleteAttributeModal.vue @@ -5,14 +5,14 @@ import { storeToRefs } from "pinia"; const attributesStore = useAttributesStore(); const { status } = storeToRefs(attributesStore); -const props = defineProps(["attribute_id", "modal"]); +const props = defineProps(["attribute_uuid", "modal"]); const emit = defineEmits(["attribute-deleted"]); function deleteAttribute() { return attributesStore - .delete(props.attribute_id) + .delete(props.attribute_uuid) .then(() => { - emit("attribute-deleted", { attribute_id: props.attribute_id }); + emit("attribute-deleted", { attribute_uuid: props.attribute_uuid }); props.modal.hide(); }) .catch((error) => (status.error = error)); @@ -20,11 +20,11 @@ function deleteAttribute() {