From 87e5041ccaa277c1aeee6bdbf9f45875f02a0c52 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Thu, 18 Jun 2026 17:50:23 +0200 Subject: [PATCH 1/5] feat(api): add triggers domain skeleton, events catalog, and Composio adapter Stand up the inbound triggers domain mirroring tools (action -> event): the read-only events catalog and ComposioTriggersAdapter behind TriggersGatewayInterface (list/get events, create/set-status/delete subscription on ti_* instances). - New core/triggers, apis/fastapi/triggers, dbs/postgres/triggers (skeleton). - Verified Composio v3 REST paths (triggers_types, trigger_instances/...). - Dedicated VIEW_TRIGGERS permission (own triad, viewer default); not VIEW_TOOLS. Co-Authored-By: Claude Opus 4.8 (1M context) --- api/ee/src/core/access/permissions/types.py | 8 + .../pytest/acceptance/triggers/__init__.py | 0 .../triggers/test_triggers_catalog.py | 160 +++++++++ api/entrypoints/routers.py | 45 +++ api/oss/src/apis/fastapi/triggers/__init__.py | 0 api/oss/src/apis/fastapi/triggers/models.py | 36 ++ api/oss/src/apis/fastapi/triggers/router.py | 319 ++++++++++++++++++ api/oss/src/core/triggers/__init__.py | 0 api/oss/src/core/triggers/dtos.py | 49 +++ api/oss/src/core/triggers/exceptions.py | 36 ++ api/oss/src/core/triggers/interfaces.py | 75 ++++ .../src/core/triggers/providers/__init__.py | 0 .../triggers/providers/composio/__init__.py | 18 + .../triggers/providers/composio/adapter.py | 187 ++++++++++ .../triggers/providers/composio/catalog.py | 188 +++++++++++ api/oss/src/core/triggers/registry.py | 27 ++ api/oss/src/core/triggers/service.py | 86 +++++ api/oss/src/dbs/postgres/triggers/__init__.py | 0 .../pytest/acceptance/triggers/__init__.py | 0 .../triggers/test_triggers_catalog.py | 78 +++++ 20 files changed, 1312 insertions(+) create mode 100644 api/ee/tests/pytest/acceptance/triggers/__init__.py create mode 100644 api/ee/tests/pytest/acceptance/triggers/test_triggers_catalog.py create mode 100644 api/oss/src/apis/fastapi/triggers/__init__.py create mode 100644 api/oss/src/apis/fastapi/triggers/models.py create mode 100644 api/oss/src/apis/fastapi/triggers/router.py create mode 100644 api/oss/src/core/triggers/__init__.py create mode 100644 api/oss/src/core/triggers/dtos.py create mode 100644 api/oss/src/core/triggers/exceptions.py create mode 100644 api/oss/src/core/triggers/interfaces.py create mode 100644 api/oss/src/core/triggers/providers/__init__.py create mode 100644 api/oss/src/core/triggers/providers/composio/__init__.py create mode 100644 api/oss/src/core/triggers/providers/composio/adapter.py create mode 100644 api/oss/src/core/triggers/providers/composio/catalog.py create mode 100644 api/oss/src/core/triggers/registry.py create mode 100644 api/oss/src/core/triggers/service.py create mode 100644 api/oss/src/dbs/postgres/triggers/__init__.py create mode 100644 api/oss/tests/pytest/acceptance/triggers/__init__.py create mode 100644 api/oss/tests/pytest/acceptance/triggers/test_triggers_catalog.py diff --git a/api/ee/src/core/access/permissions/types.py b/api/ee/src/core/access/permissions/types.py index c3ab36b719..6cf7ee1647 100644 --- a/api/ee/src/core/access/permissions/types.py +++ b/api/ee/src/core/access/permissions/types.py @@ -190,6 +190,11 @@ class Permission(str, Enum): EDIT_TOOLS = "edit_tools" RUN_TOOLS = "run_tools" + # Triggers + VIEW_TRIGGERS = "view_triggers" + EDIT_TRIGGERS = "edit_triggers" + RUN_TRIGGERS = "run_triggers" + @classmethod def default_permissions(cls, role): VIEWER_PERMISSIONS = [ @@ -217,6 +222,7 @@ def default_permissions(cls, role): cls.VIEW_EVALUATION_METRICS, cls.VIEW_EVALUATION_QUEUES, cls.VIEW_TOOLS, + cls.VIEW_TRIGGERS, ] ANNOTATOR_PERMISSIONS = VIEWER_PERMISSIONS + [ cls.CREATE_EVALUATION, @@ -230,6 +236,7 @@ def default_permissions(cls, role): cls.EDIT_EVALUATION_QUEUES, cls.EDIT_SPANS, cls.RUN_TOOLS, + cls.RUN_TRIGGERS, ] EDITOR_PERMISSIONS = ANNOTATOR_PERMISSIONS + [ cls.EDIT_APPLICATIONS, @@ -251,6 +258,7 @@ def default_permissions(cls, role): cls.EDIT_TESTSETS, cls.EDIT_INVOCATIONS, cls.EDIT_TOOLS, + cls.EDIT_TRIGGERS, ] DEVELOPER_PERMISSIONS = EDITOR_PERMISSIONS + [ cls.VIEW_API_KEYS, diff --git a/api/ee/tests/pytest/acceptance/triggers/__init__.py b/api/ee/tests/pytest/acceptance/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/ee/tests/pytest/acceptance/triggers/test_triggers_catalog.py b/api/ee/tests/pytest/acceptance/triggers/test_triggers_catalog.py new file mode 100644 index 0000000000..005bdc367f --- /dev/null +++ b/api/ee/tests/pytest/acceptance/triggers/test_triggers_catalog.py @@ -0,0 +1,160 @@ +"""EE acceptance tests for the triggers events catalog. + +Mirrors the OSS suite (oss/tests/pytest/acceptance/triggers/test_triggers_catalog.py) +but exercises /triggers/catalog/* as a business-plan, developer-role account. +Under EE the catalog is gated on the VIEW_TOOLS permission (the triggers domain +shares the gateway permission surface with tools); a developer role carries +VIEW_TOOLS, so this verifies the endpoint behaves once the gate is satisfied. + +Provider-catalog reads need no Composio credentials (empty catalog is valid). +Event browse / config-schema fetch make real Composio calls and are gated on +COMPOSIO_API_KEY being present in the runner's environment. + +Requires a running API. +""" + +import os +from uuid import uuid4 + +import pytest +import requests + +from utils.constants import BASE_TIMEOUT + + +_COMPOSIO_ENABLED = bool(os.getenv("COMPOSIO_API_KEY")) +_requires_composio = pytest.mark.skipif( + not _COMPOSIO_ENABLED, + reason="needs live Composio credentials (COMPOSIO_API_KEY)", +) + + +def _create_developer_business_account(admin_api): + uid = uuid4().hex[:12] + email = f"triggers-dev-{uid}@test.agenta.ai" + resp = admin_api( + "POST", + "/admin/simple/accounts/", + json={ + "accounts": { + "u": { + "user": {"email": email}, + "options": { + "create_api_keys": True, + "return_api_keys": True, + "seed_defaults": False, + }, + "subscription": {"plan": "cloud_v0_business"}, + "organization_memberships": [ + { + "organization_ref": {"ref": "org"}, + "user_ref": {"ref": "user"}, + "role": "developer", + } + ], + "workspace_memberships": [ + { + "workspace_ref": {"ref": "wrk"}, + "user_ref": {"ref": "user"}, + "role": "developer", + } + ], + "project_memberships": [ + { + "project_ref": {"ref": "prj"}, + "user_ref": {"ref": "user"}, + "role": "developer", + } + ], + } + } + }, + ) + assert resp.status_code == 200, resp.text + account = resp.json()["accounts"]["u"] + return { + "email": email, + "credentials": f"ApiKey {account['api_keys']['key']}", + } + + +def _delete_account_by_email(admin_api, *, email): + resp = admin_api( + "DELETE", + "/admin/simple/accounts/", + json={"accounts": {"u": {"user": {"email": email}}}, "confirm": "delete"}, + ) + assert resp.status_code == 204, resp.text + + +@pytest.fixture(scope="class") +def triggers_api(admin_api, ag_env): + account = _create_developer_business_account(admin_api) + + def _request(method: str, endpoint: str, **kwargs): + headers = kwargs.pop("headers", {}) + headers.setdefault("Authorization", account["credentials"]) + return requests.request( + method=method, + url=f"{ag_env['api_url']}{endpoint}", + headers=headers, + timeout=BASE_TIMEOUT, + **kwargs, + ) + + yield _request + + _delete_account_by_email(admin_api, email=account["email"]) + + +class TestTriggersCatalogProviders: + def test_list_providers_returns_200(self, triggers_api): + response = triggers_api("GET", "/triggers/catalog/providers/") + assert response.status_code == 200 + + def test_list_providers_response_shape(self, triggers_api): + body = triggers_api("GET", "/triggers/catalog/providers/").json() + assert "count" in body + assert "providers" in body + assert isinstance(body["providers"], list) + assert body["count"] == len(body["providers"]) + + @pytest.mark.skipif( + _COMPOSIO_ENABLED, + reason="catalog is non-empty when Composio is enabled", + ) + def test_list_providers_empty_when_composio_disabled(self, triggers_api): + body = triggers_api("GET", "/triggers/catalog/providers/").json() + assert body["count"] == 0 + assert body["providers"] == [] + + +@_requires_composio +class TestTriggersCatalogEvents: + def test_browse_events_returns_200(self, triggers_api): + response = triggers_api( + "GET", + "/triggers/catalog/providers/composio/integrations/github/events/", + ) + assert response.status_code == 200 + body = response.json() + assert "events" in body + assert isinstance(body["events"], list) + + def test_fetch_event_config_schema(self, triggers_api): + listing = triggers_api( + "GET", + "/triggers/catalog/providers/composio/integrations/github/events/", + ).json() + if not listing["events"]: + pytest.skip("no github events available from Composio") + + event_key = listing["events"][0]["key"] + response = triggers_api( + "GET", + f"/triggers/catalog/providers/composio/integrations/github/events/{event_key}", + ) + assert response.status_code == 200 + event = response.json()["event"] + assert event["key"] == event_key + assert "trigger_config" in event diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index c235e817fd..96c11eb9a8 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -142,6 +142,10 @@ from oss.src.core.tools.registry import ToolsGatewayRegistry from oss.src.core.tools.service import ToolsService from oss.src.apis.fastapi.tools.router import ToolsRouter +from oss.src.core.triggers.providers.composio import ComposioTriggersAdapter +from oss.src.core.triggers.registry import TriggersGatewayRegistry +from oss.src.core.triggers.service import TriggersService +from oss.src.apis.fastapi.triggers.router import TriggersRouter from oss.src.apis.fastapi.shared.utils import SupportHeadersMiddleware @@ -215,6 +219,9 @@ async def lifespan(*args, **kwargs): for adapter in _composio_connections_adapters.values(): await adapter.close() + for adapter in _composio_triggers_adapters.values(): + await adapter.close() + await _transactions_engine.close() await _analytics_engine.close() await _streams_engine.close() @@ -308,6 +315,11 @@ async def lifespan(*args, **kwargs): "description": "External tool connections and OAuth integrations available to applications.", }, # -- + { + "name": "Triggers", + "description": "Inbound provider event triggers and their watchable event catalog.", + }, + # -- { "name": "Folders", "description": "Organize applications and other resources into folder hierarchies.", @@ -616,6 +628,22 @@ async def lifespan(*args, **kwargs): adapter_registry=tools_adapter_registry, ) +# Triggers adapter + service +_composio_triggers_adapters = {} +if env.composio.enabled: + _composio_triggers_adapters["composio"] = ComposioTriggersAdapter( + api_key=env.composio.api_key, # type: ignore[arg-type] # guarded by .enabled + api_url=env.composio.api_url, + ) + +triggers_adapter_registry = TriggersGatewayRegistry( + adapters=_composio_triggers_adapters, +) + +triggers_service = TriggersService( + adapter_registry=triggers_adapter_registry, +) + _t_services_done = time.perf_counter() - _t_services print(f"[STARTUP] Service initialization completed (+{_t_services_done:.3f}s)") _t_routers = time.perf_counter() @@ -730,6 +758,10 @@ async def lifespan(*args, **kwargs): tools_service=tools_service, ) +triggers = TriggersRouter( + triggers_service=triggers_service, +) + simple_traces = SimpleTracesRouter( simple_traces_service=simple_traces_service, ) @@ -1097,6 +1129,19 @@ async def lifespan(*args, **kwargs): include_in_schema=False, ) +app.include_router( + router=triggers.router, + prefix="/triggers", + tags=["Triggers"], +) + +app.include_router( + router=triggers.router, + prefix="/preview/triggers", + tags=["Triggers"], + include_in_schema=False, +) + app.include_router( router=evaluations.admin_router, prefix="/admin/evaluations", diff --git a/api/oss/src/apis/fastapi/triggers/__init__.py b/api/oss/src/apis/fastapi/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/apis/fastapi/triggers/models.py b/api/oss/src/apis/fastapi/triggers/models.py new file mode 100644 index 0000000000..93e9d5ab4e --- /dev/null +++ b/api/oss/src/apis/fastapi/triggers/models.py @@ -0,0 +1,36 @@ +from typing import List, Optional + +from pydantic import BaseModel + +from oss.src.core.triggers.dtos import ( + TriggerCatalogEvent, + TriggerCatalogEventDetails, + TriggerCatalogProvider, +) + + +# --------------------------------------------------------------------------- +# Trigger Catalog +# --------------------------------------------------------------------------- + + +class TriggerCatalogProviderResponse(BaseModel): + count: int = 0 + provider: Optional[TriggerCatalogProvider] = None + + +class TriggerCatalogProvidersResponse(BaseModel): + count: int = 0 + providers: List[TriggerCatalogProvider] = [] + + +class TriggerCatalogEventResponse(BaseModel): + count: int = 0 + event: Optional[TriggerCatalogEventDetails] = None + + +class TriggerCatalogEventsResponse(BaseModel): + count: int = 0 + total: int = 0 + cursor: Optional[str] = None + events: List[TriggerCatalogEvent] = [] diff --git a/api/oss/src/apis/fastapi/triggers/router.py b/api/oss/src/apis/fastapi/triggers/router.py new file mode 100644 index 0000000000..5270682dc4 --- /dev/null +++ b/api/oss/src/apis/fastapi/triggers/router.py @@ -0,0 +1,319 @@ +from functools import wraps +from typing import Optional + +import httpx +from fastapi import APIRouter, HTTPException, Query, Request, status +from fastapi.responses import JSONResponse + +from oss.src.utils.exceptions import intercept_exceptions +from oss.src.utils.logging import get_module_logger +from oss.src.utils.caching import get_cache, set_cache +from oss.src.utils.common import is_ee + +from oss.src.apis.fastapi.triggers.models import ( + TriggerCatalogEventResponse, + TriggerCatalogEventsResponse, + TriggerCatalogProviderResponse, + TriggerCatalogProvidersResponse, +) +from oss.src.core.triggers.exceptions import AdapterError +from oss.src.core.triggers.service import TriggersService + + +if is_ee(): + from ee.src.core.access.permissions.types import Permission + from ee.src.core.access.permissions.service import ( + check_action_access, + FORBIDDEN_EXCEPTION, + ) + +log = get_module_logger(__name__) + + +def handle_adapter_exceptions(): + """Convert only upstream 401 AdapterError failures to 424 Failed Dependency.""" + + def decorator(func): + @wraps(func) + async def wrapper(*args, **kwargs): + try: + return await func(*args, **kwargs) + except AdapterError as e: + cause = e.__cause__ + if not ( + isinstance(cause, httpx.HTTPStatusError) + and cause.response is not None + and cause.response.status_code == status.HTTP_401_UNAUTHORIZED + ): + raise + + raise HTTPException( + status_code=status.HTTP_424_FAILED_DEPENDENCY, + detail=e.message, + ) from e + + return wrapper + + return decorator + + +class TriggersRouter: + def __init__( + self, + *, + triggers_service: TriggersService, + ): + self.triggers_service = triggers_service + + self.router = APIRouter() + + # --- Trigger Catalog --- + self.router.add_api_route( + "/catalog/providers/", + self.list_providers, + methods=["GET"], + operation_id="list_trigger_providers", + response_model=TriggerCatalogProvidersResponse, + response_model_exclude_none=True, + ) + self.router.add_api_route( + "/catalog/providers/{provider_key}", + self.get_provider, + methods=["GET"], + operation_id="fetch_trigger_provider", + response_model=TriggerCatalogProviderResponse, + response_model_exclude_none=True, + ) + self.router.add_api_route( + "/catalog/providers/{provider_key}/integrations/{integration_key}/events/", + self.list_events, + methods=["GET"], + operation_id="list_trigger_events", + response_model=TriggerCatalogEventsResponse, + response_model_exclude_none=True, + ) + self.router.add_api_route( + "/catalog/providers/{provider_key}/integrations/{integration_key}/events/{event_key}", + self.get_event, + methods=["GET"], + operation_id="fetch_trigger_event", + response_model=TriggerCatalogEventResponse, + response_model_exclude_none=True, + ) + + # ----------------------------------------------------------------------- + # Trigger Catalog + # ----------------------------------------------------------------------- + + @intercept_exceptions() + @handle_adapter_exceptions() + async def list_providers( + self, + request: Request, + ) -> TriggerCatalogProvidersResponse: + if is_ee(): + has_permission = await check_action_access( + project_id=request.state.project_id, + user_uid=request.state.user_id, + permission=Permission.VIEW_TRIGGERS, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + cached = await get_cache( + project_id=None, # catalog is global; not per-project + namespace="triggers:catalog:providers", + key={}, + model=TriggerCatalogProvidersResponse, + ) + if cached: + return cached + + providers = await self.triggers_service.list_providers() + items = list(providers) + + response = TriggerCatalogProvidersResponse( + count=len(items), + providers=items, + ) + + await set_cache( + project_id=None, + namespace="triggers:catalog:providers", + key={}, + value=response, + ttl=5 * 60, + ) + + return response + + @intercept_exceptions() + @handle_adapter_exceptions() + async def get_provider( + self, + request: Request, + provider_key: str, + ) -> TriggerCatalogProviderResponse: + if is_ee(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_TRIGGERS, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + cache_key = {"provider_key": provider_key} + cached = await get_cache( + project_id=None, + namespace="triggers:catalog:provider", + key=cache_key, + model=TriggerCatalogProviderResponse, + ) + if cached: + return cached + + provider = await self.triggers_service.get_provider( + provider_key=provider_key, + ) + if not provider: + return JSONResponse( + status_code=404, + content={"detail": "Provider not found"}, + ) + + response = TriggerCatalogProviderResponse( + count=1, + provider=provider, + ) + + await set_cache( + project_id=None, + namespace="triggers:catalog:provider", + key=cache_key, + value=response, + ttl=5 * 60, + ) + + return response + + @intercept_exceptions() + @handle_adapter_exceptions() + async def list_events( + self, + request: Request, + provider_key: str, + integration_key: str, + *, + query: Optional[str] = Query(default=None), + limit: Optional[int] = Query(default=None), + cursor: Optional[str] = Query(default=None), + ) -> TriggerCatalogEventsResponse: + if is_ee(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_TRIGGERS, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + cache_key = { + "provider_key": provider_key, + "integration_key": integration_key, + "query": query, + "limit": limit, + "cursor": cursor, + } + cached = await get_cache( + project_id=None, + namespace="triggers:catalog:events", + key=cache_key, + model=TriggerCatalogEventsResponse, + ) + if cached: + return cached + + events, next_cursor, total = await self.triggers_service.list_events( + provider_key=provider_key, + integration_key=integration_key, + query=query, + limit=limit, + cursor=cursor, + ) + items = list(events) + + response = TriggerCatalogEventsResponse( + count=len(items), + total=total, + cursor=next_cursor, + events=items, + ) + + await set_cache( + project_id=None, + namespace="triggers:catalog:events", + key=cache_key, + value=response, + ttl=5 * 60, + ) + + return response + + @intercept_exceptions() + @handle_adapter_exceptions() + async def get_event( + self, + request: Request, + provider_key: str, + integration_key: str, + event_key: str, + ) -> TriggerCatalogEventResponse: + if is_ee(): + has_permission = await check_action_access( + user_uid=request.state.user_id, + project_id=request.state.project_id, + permission=Permission.VIEW_TRIGGERS, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + cache_key = { + "provider_key": provider_key, + "integration_key": integration_key, + "event_key": event_key, + } + cached = await get_cache( + project_id=None, + namespace="triggers:catalog:event", + key=cache_key, + model=TriggerCatalogEventResponse, + ) + if cached: + return cached + + event = await self.triggers_service.get_event( + provider_key=provider_key, + integration_key=integration_key, + event_key=event_key, + ) + if not event: + return JSONResponse( + status_code=404, + content={"detail": "Event not found"}, + ) + + response = TriggerCatalogEventResponse( + count=1, + event=event, + ) + + await set_cache( + project_id=None, + namespace="triggers:catalog:event", + key=cache_key, + value=response, + ttl=5 * 60, + ) + + return response diff --git a/api/oss/src/core/triggers/__init__.py b/api/oss/src/core/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/core/triggers/dtos.py b/api/oss/src/core/triggers/dtos.py new file mode 100644 index 0000000000..656a7ce56b --- /dev/null +++ b/api/oss/src/core/triggers/dtos.py @@ -0,0 +1,49 @@ +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + + +# --------------------------------------------------------------------------- +# Trigger Enums +# --------------------------------------------------------------------------- + + +class TriggerProviderKind(str, Enum): + COMPOSIO = "composio" + + +# --------------------------------------------------------------------------- +# Trigger Catalog +# +# The catalog leaf is an **event** (Composio "trigger type"), the analogue of a +# tools **action**. An event carries a ``trigger_config`` JSON Schema, the +# analogue of an action's ``input_parameters``. +# --------------------------------------------------------------------------- + + +class TriggerCatalogEvent(BaseModel): + key: str + # + name: str + description: Optional[str] = None + # + provider: Optional[str] = None + integration: Optional[str] = None + # + categories: List[str] = [] + logo: Optional[str] = None + + +class TriggerCatalogEventDetails(TriggerCatalogEvent): + # FROZEN (WS-PRE): the Event DTO carries the event's trigger_config JSON Schema + # — the inbound analogue of an action's input_parameters. + trigger_config: Optional[Dict[str, Any]] = None + payload: Optional[Dict[str, Any]] = None + + +class TriggerCatalogProvider(BaseModel): + key: TriggerProviderKind + # + name: str + description: Optional[str] = None diff --git a/api/oss/src/core/triggers/exceptions.py b/api/oss/src/core/triggers/exceptions.py new file mode 100644 index 0000000000..473b4094a4 --- /dev/null +++ b/api/oss/src/core/triggers/exceptions.py @@ -0,0 +1,36 @@ +from typing import Optional + + +class TriggersError(Exception): + """Base exception for the triggers domain.""" + + def __init__(self, message: str = "Triggers error"): + self.message = message + super().__init__(self.message) + + +class ProviderNotFoundError(TriggersError): + """Raised when the requested provider_key has no registered adapter.""" + + def __init__(self, provider_key: str): + self.provider_key = provider_key + super().__init__(f"Provider not found: {provider_key}") + + +class AdapterError(TriggersError): + """Raised when an adapter operation fails.""" + + def __init__( + self, + *, + provider_key: str, + operation: str, + detail: Optional[str] = None, + ): + self.provider_key = provider_key + self.operation = operation + self.detail = detail + msg = f"Adapter error ({provider_key}.{operation})" + if detail: + msg += f": {detail}" + super().__init__(msg) diff --git a/api/oss/src/core/triggers/interfaces.py b/api/oss/src/core/triggers/interfaces.py new file mode 100644 index 0000000000..2b07ca835f --- /dev/null +++ b/api/oss/src/core/triggers/interfaces.py @@ -0,0 +1,75 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Tuple +from uuid import UUID + +from oss.src.core.triggers.dtos import ( + TriggerCatalogEvent, + TriggerCatalogEventDetails, + TriggerCatalogProvider, +) + + +class TriggersGatewayInterface(ABC): + """Port for external trigger providers (Composio, ...). + + FROZEN (WS-PRE) — consumed by WS3 (subscriptions) and WS5 (web catalog). + The catalog reads (``list_events``/``get_event``) back the events catalog; + the subscription verbs build/manage the provider-side trigger instance + (``ti_*``) that WP3 stores on a local subscription row. + """ + + @abstractmethod + async def list_providers(self) -> List[TriggerCatalogProvider]: ... + + @abstractmethod + async def list_events( + self, + *, + integration_key: str, + query: Optional[str] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[TriggerCatalogEvent], Optional[str], int]: + """Returns (items, next_cursor, total_items).""" + ... + + @abstractmethod + async def get_event( + self, + *, + integration_key: str, + event_key: str, + ) -> Optional[TriggerCatalogEventDetails]: + """Return one event's detail, carrying its trigger_config JSON Schema.""" + ... + + @abstractmethod + async def create_subscription( + self, + *, + project_id: UUID, + event_key: str, + connected_account_id: str, + trigger_config: Dict[str, Any], + ) -> str: + """Create the provider-side trigger instance; returns its id (``ti_*``).""" + ... + + @abstractmethod + async def set_subscription_status( + self, + *, + trigger_id: str, + enabled: bool, + ) -> None: + """Enable or disable the provider-side trigger instance.""" + ... + + @abstractmethod + async def delete_subscription( + self, + *, + trigger_id: str, + ) -> None: + """Permanently delete the provider-side trigger instance.""" + ... diff --git a/api/oss/src/core/triggers/providers/__init__.py b/api/oss/src/core/triggers/providers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/core/triggers/providers/composio/__init__.py b/api/oss/src/core/triggers/providers/composio/__init__.py new file mode 100644 index 0000000000..9841fc07c1 --- /dev/null +++ b/api/oss/src/core/triggers/providers/composio/__init__.py @@ -0,0 +1,18 @@ +# Avoid importing adapter here to prevent SDK dependency issues in standalone scripts. +# Import directly when needed: +# from oss.src.core.triggers.providers.composio.adapter import ComposioTriggersAdapter + +__all__ = [ + "ComposioTriggersAdapter", +] + + +def __getattr__(name): + """Lazy import to avoid SDK dependency on module import.""" + if name == "ComposioTriggersAdapter": + from oss.src.core.triggers.providers.composio.adapter import ( + ComposioTriggersAdapter, + ) + + return ComposioTriggersAdapter + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/api/oss/src/core/triggers/providers/composio/adapter.py b/api/oss/src/core/triggers/providers/composio/adapter.py new file mode 100644 index 0000000000..20fd9dd212 --- /dev/null +++ b/api/oss/src/core/triggers/providers/composio/adapter.py @@ -0,0 +1,187 @@ +from typing import Any, Dict, List, Optional +from uuid import UUID + +import httpx + +from oss.src.utils.logging import get_module_logger + +from oss.src.core.triggers.dtos import ( + TriggerCatalogProvider, + TriggerProviderKind, +) +from oss.src.core.triggers.interfaces import TriggersGatewayInterface +from oss.src.core.triggers.exceptions import AdapterError +from oss.src.core.triggers.providers.composio.catalog import ( + ComposioTriggersCatalogClient, +) + + +log = get_module_logger(__name__) + +COMPOSIO_DEFAULT_API_URL = "https://backend.composio.dev/api/v3" + + +class ComposioTriggersAdapter(ComposioTriggersCatalogClient, TriggersGatewayInterface): + """Composio V3 triggers adapter — uses httpx directly (no SDK). + + Modeled on ``ComposioToolsAdapter``: own httpx client, ``_get/_post/_delete`` + helpers, slug passthrough. Catalog operations (list/get events) come from + ``ComposioTriggersCatalogClient``; subscription (trigger-instance) management + is implemented here and consumed by WP3. + + REST paths (E5 — verified vs the live Composio API reference): + list events GET /triggers_types?toolkit_slugs={i} + get event GET /triggers_types/{slug} + create/upsert POST /trigger_instances/{slug}/upsert + enable/disable PATCH /trigger_instances/manage/{trigger_id} + delete DELETE /trigger_instances/manage/{trigger_id} + """ + + def __init__( + self, + *, + api_key: str, + api_url: str = COMPOSIO_DEFAULT_API_URL, + ): + self.api_key = api_key + self.api_url = api_url.rstrip("/") + # Shared client — one connection pool for the adapter's lifetime. + # Call close() on shutdown (wired in entrypoints/routers.py lifespan). + self._client = httpx.AsyncClient(timeout=30.0) + + async def close(self) -> None: + """Close the shared HTTP client and release connection pool resources.""" + await self._client.aclose() + + def _headers(self) -> Dict[str, str]: + return { + "x-api-key": self.api_key, + "Content-Type": "application/json", + } + + async def _post( + self, + path: str, + *, + json: Optional[Dict[str, Any]] = None, + ) -> Any: + resp = await self._client.post( + f"{self.api_url}{path}", + headers=self._headers(), + json=json or {}, + ) + if not resp.is_success: + log.error("Composio POST %s → %s: %s", path, resp.status_code, resp.text) + resp.raise_for_status() + return resp.json() + + async def _patch( + self, + path: str, + *, + json: Optional[Dict[str, Any]] = None, + ) -> Any: + resp = await self._client.patch( + f"{self.api_url}{path}", + headers=self._headers(), + json=json or {}, + ) + if not resp.is_success: + log.error("Composio PATCH %s → %s: %s", path, resp.status_code, resp.text) + resp.raise_for_status() + return resp.json() + + async def _delete(self, path: str) -> bool: + resp = await self._client.delete( + f"{self.api_url}{path}", + headers=self._headers(), + ) + resp.raise_for_status() + return True + + # ----------------------------------------------------------------------- + # Catalog — provider listing + # ----------------------------------------------------------------------- + + async def list_providers(self) -> List[TriggerCatalogProvider]: + return [ + TriggerCatalogProvider( + key=TriggerProviderKind.COMPOSIO, + name="Composio", + description="Third-party event triggers via Composio", + ) + ] + + # list_events and get_event are inherited from ComposioTriggersCatalogClient + # and satisfy the TriggersGatewayInterface catalog contract. + + # ----------------------------------------------------------------------- + # Subscriptions (provider-side trigger instances — ti_*) — consumed by WP3 + # ----------------------------------------------------------------------- + + async def create_subscription( + self, + *, + project_id: UUID, + event_key: str, + connected_account_id: str, + trigger_config: Dict[str, Any], + ) -> str: + """Create/upsert the provider-side trigger instance; return its id (ti_*).""" + payload: Dict[str, Any] = { + "connected_account_id": connected_account_id, + "trigger_config": trigger_config or {}, + } + try: + result = await self._post( + f"/trigger_instances/{event_key}/upsert", + json=payload, + ) + except httpx.HTTPError as e: + raise AdapterError( + provider_key="composio", + operation="create_subscription", + detail=str(e), + ) from e + + trigger_id = result.get("trigger_id") or result.get("id") + if not trigger_id: + raise AdapterError( + provider_key="composio", + operation="create_subscription", + detail=f"No trigger_id in upsert response for event '{event_key}'", + ) + return trigger_id + + async def set_subscription_status( + self, + *, + trigger_id: str, + enabled: bool, + ) -> None: + status = "enable" if enabled else "disable" + try: + await self._patch( + f"/trigger_instances/manage/{trigger_id}", + json={"status": status}, + ) + except httpx.HTTPError as e: + raise AdapterError( + provider_key="composio", + operation="set_subscription_status", + detail=str(e), + ) from e + + async def delete_subscription( + self, + *, + trigger_id: str, + ) -> None: + try: + await self._delete(f"/trigger_instances/manage/{trigger_id}") + except httpx.HTTPError as e: + raise AdapterError( + provider_key="composio", + operation="delete_subscription", + detail=str(e), + ) from e diff --git a/api/oss/src/core/triggers/providers/composio/catalog.py b/api/oss/src/core/triggers/providers/composio/catalog.py new file mode 100644 index 0000000000..f773fab8ec --- /dev/null +++ b/api/oss/src/core/triggers/providers/composio/catalog.py @@ -0,0 +1,188 @@ +"""Composio triggers catalog operations — mixin for ComposioTriggersAdapter. + +Provides catalog HTTP calls (list events, get one event) backed by +``self._client``, ``self.api_key``, and ``self.api_url`` which must be supplied +by the concrete subclass (ComposioTriggersAdapter). + +Mirrors ``core/tools/providers/composio/catalog.py`` with ``action → event``: +the tools "action" leaf becomes the triggers "event" leaf (a Composio *trigger +type*), and an action's ``input_parameters`` schema becomes an event's +``trigger_config`` schema. The ``cursor`` value is Composio's native +``next_cursor`` string, passed through as-is. +""" + +from typing import Any, Dict, List, Optional, Tuple + +import httpx + +from oss.src.utils.logging import get_module_logger +from oss.src.core.triggers.dtos import ( + TriggerCatalogEvent, + TriggerCatalogEventDetails, +) +from oss.src.core.triggers.exceptions import AdapterError + + +log = get_module_logger(__name__) + +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 1000 + + +class ComposioTriggersCatalogClient: + """Catalog mixin for ComposioTriggersAdapter — cursor-based pagination. + + Subclass must set ``self.api_key``, ``self.api_url``, and ``self._client`` + (an ``httpx.AsyncClient``) before calling any method. + """ + + # Annotated for type-checkers; filled in by ComposioTriggersAdapter.__init__ + api_key: str + api_url: str + _client: httpx.AsyncClient + + async def list_events( + self, + *, + integration_key: str, + query: Optional[str] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[TriggerCatalogEvent], Optional[str], int]: + """Fetch one page of events (Composio trigger types) for an integration. + + E5 (verified vs live Composio API reference): GET /triggers_types, + filtered by ``toolkit_slugs``. + """ + page_limit = min(limit, MAX_PAGE_SIZE) if limit else DEFAULT_PAGE_SIZE + + params: Dict[str, Any] = { + "toolkit_slugs": integration_key, + "limit": page_limit, + } + if query: + params["query"] = query + if cursor: + params["cursor"] = cursor + + try: + resp = await self._client.get( + f"{self.api_url}/triggers_types", + headers={"x-api-key": self.api_key, "Content-Type": "application/json"}, + params=params, + timeout=30.0, + ) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as e: + raise AdapterError( + provider_key="composio", + operation="list_events", + detail=str(e), + ) from e + + items_raw: List[Dict[str, Any]] = ( + data.get("items", []) if isinstance(data, dict) else data + ) + next_cursor: Optional[str] = ( + data.get("next_cursor") if isinstance(data, dict) else None + ) + total_items: int = ( + data.get("total_items", len(items_raw)) + if isinstance(data, dict) + else len(items_raw) + ) + + items = [_parse_event(item, integration_key) for item in items_raw] + + log.debug( + "[composio] list_events(%s) cursor=%s items=%d total=%d next=%s", + integration_key, + cursor, + len(items), + total_items, + next_cursor, + ) + + return items, next_cursor, total_items + + async def get_event( + self, + *, + integration_key: str, + event_key: str, + ) -> Optional[TriggerCatalogEventDetails]: + """Fetch one event (trigger type) by slug, with its trigger_config schema. + + E5 (verified vs live Composio API reference): GET /triggers_types/{slug}. + Returns None when the event does not exist (404). + """ + try: + resp = await self._client.get( + f"{self.api_url}/triggers_types/{event_key}", + headers={"x-api-key": self.api_key, "Content-Type": "application/json"}, + timeout=15.0, + ) + if resp.status_code == 404: + return None + resp.raise_for_status() + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + return None + raise AdapterError( + provider_key="composio", + operation="get_event", + detail=str(e), + ) from e + except httpx.HTTPError as e: + raise AdapterError( + provider_key="composio", + operation="get_event", + detail=str(e), + ) from e + + return _parse_event_detail(resp.json(), integration_key) + + +# --------------------------------------------------------------------------- +# Parsers (module-level — no instance state needed) +# --------------------------------------------------------------------------- + + +def _toolkit_slug(item: Dict[str, Any], fallback: str) -> str: + toolkit = item.get("toolkit") + if isinstance(toolkit, dict): + return toolkit.get("slug") or toolkit.get("name") or fallback + if isinstance(toolkit, str): + return toolkit + return fallback + + +def _parse_event(item: Dict[str, Any], integration_key: str) -> TriggerCatalogEvent: + return TriggerCatalogEvent( + key=item.get("slug", ""), + name=item.get("name", ""), + description=item.get("description"), + provider="composio", + integration=_toolkit_slug(item, integration_key), + ) + + +def _parse_event_detail( + item: Dict[str, Any], + integration_key: str, +) -> TriggerCatalogEventDetails: + # The event's required config is the JSON Schema under "config" — the inbound + # analogue of an action's "input_parameters". + trigger_config = item.get("config") or item.get("trigger_config") + payload = item.get("payload") + + return TriggerCatalogEventDetails( + key=item.get("slug", ""), + name=item.get("name", ""), + description=item.get("description"), + provider="composio", + integration=_toolkit_slug(item, integration_key), + trigger_config=trigger_config, + payload=payload, + ) diff --git a/api/oss/src/core/triggers/registry.py b/api/oss/src/core/triggers/registry.py new file mode 100644 index 0000000000..4e641f6202 --- /dev/null +++ b/api/oss/src/core/triggers/registry.py @@ -0,0 +1,27 @@ +from typing import Dict, ItemsView + +from oss.src.core.triggers.interfaces import TriggersGatewayInterface +from oss.src.core.triggers.exceptions import ProviderNotFoundError + + +class TriggersGatewayRegistry: + """Dispatches to the correct adapter based on provider_key.""" + + def __init__( + self, + *, + adapters: Dict[str, TriggersGatewayInterface], + ): + self._adapters = adapters + + def get(self, provider_key: str) -> TriggersGatewayInterface: + adapter = self._adapters.get(provider_key) + if not adapter: + raise ProviderNotFoundError(provider_key) + return adapter + + def keys(self) -> list[str]: + return list(self._adapters.keys()) + + def items(self) -> ItemsView[str, TriggersGatewayInterface]: + return self._adapters.items() diff --git a/api/oss/src/core/triggers/service.py b/api/oss/src/core/triggers/service.py new file mode 100644 index 0000000000..bc08263c2f --- /dev/null +++ b/api/oss/src/core/triggers/service.py @@ -0,0 +1,86 @@ +from typing import List, Optional, Tuple + +from oss.src.utils.logging import get_module_logger + +from oss.src.core.triggers.dtos import ( + TriggerCatalogEvent, + TriggerCatalogEventDetails, + TriggerCatalogProvider, +) +from oss.src.core.triggers.registry import TriggersGatewayRegistry + + +log = get_module_logger(__name__) + + +class TriggersService: + """Triggers domain orchestration. + + WP1 scope is the read-only events catalog. Subscriptions/deliveries CRUD and + ingress/dispatch land in later WPs (WP3/WP4) and will extend this service. + """ + + def __init__( + self, + *, + adapter_registry: TriggersGatewayRegistry, + ): + self.adapter_registry = adapter_registry + + # ----------------------------------------------------------------------- + # Catalog browse + # ----------------------------------------------------------------------- + + async def list_providers(self) -> List[TriggerCatalogProvider]: + """Return all providers across registered adapters.""" + results: List[TriggerCatalogProvider] = [] + for _key, adapter in self.adapter_registry.items(): + providers = await adapter.list_providers() + results.extend(providers) + return results + + async def get_provider( + self, + *, + provider_key: str, + ) -> Optional[TriggerCatalogProvider]: + """Return a single provider by key, or None if not found.""" + adapter = self.adapter_registry.get(provider_key) + providers = await adapter.list_providers() + for p in providers: + if p.key == provider_key: + return p + return None + + async def list_events( + self, + *, + provider_key: str, + integration_key: str, + # + query: Optional[str] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[TriggerCatalogEvent], Optional[str], int]: + """List events for an integration with optional search and pagination.""" + adapter = self.adapter_registry.get(provider_key) + return await adapter.list_events( + integration_key=integration_key, + query=query, + limit=limit, + cursor=cursor, + ) + + async def get_event( + self, + *, + provider_key: str, + integration_key: str, + event_key: str, + ) -> Optional[TriggerCatalogEventDetails]: + """Return full event detail including its trigger_config schema, or None.""" + adapter = self.adapter_registry.get(provider_key) + return await adapter.get_event( + integration_key=integration_key, + event_key=event_key, + ) diff --git a/api/oss/src/dbs/postgres/triggers/__init__.py b/api/oss/src/dbs/postgres/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/tests/pytest/acceptance/triggers/__init__.py b/api/oss/tests/pytest/acceptance/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/tests/pytest/acceptance/triggers/test_triggers_catalog.py b/api/oss/tests/pytest/acceptance/triggers/test_triggers_catalog.py new file mode 100644 index 0000000000..bdfa94897c --- /dev/null +++ b/api/oss/tests/pytest/acceptance/triggers/test_triggers_catalog.py @@ -0,0 +1,78 @@ +"""Acceptance tests for GET /triggers/catalog/* endpoints (events catalog). + +The provider-catalog endpoints are reachable without any external API key: an +empty catalog is a valid response (no Composio adapter is registered when +``env.composio`` is unset). The event-browse / config-schema fetch make real +Composio calls, so those tests are gated on COMPOSIO_API_KEY being present in +the runner's environment (the same env the API reads). +""" + +import os + +import pytest + + +_COMPOSIO_ENABLED = bool(os.getenv("COMPOSIO_API_KEY")) +_requires_composio = pytest.mark.skipif( + not _COMPOSIO_ENABLED, + reason="needs live Composio credentials (COMPOSIO_API_KEY)", +) + + +class TestTriggersCatalogProviders: + def test_list_providers_returns_200(self, authed_api): + response = authed_api("GET", "/triggers/catalog/providers/") + assert response.status_code == 200 + + def test_list_providers_response_shape(self, authed_api): + body = authed_api("GET", "/triggers/catalog/providers/").json() + assert "count" in body + assert "providers" in body + assert isinstance(body["providers"], list) + + def test_list_providers_count_matches_list(self, authed_api): + body = authed_api("GET", "/triggers/catalog/providers/").json() + assert body["count"] == len(body["providers"]) + + @pytest.mark.skipif( + _COMPOSIO_ENABLED, + reason="catalog is non-empty when Composio is enabled", + ) + def test_list_providers_empty_when_composio_disabled(self, authed_api): + """With env.composio unset, no adapter is registered → empty catalog.""" + body = authed_api("GET", "/triggers/catalog/providers/").json() + assert body["count"] == 0 + assert body["providers"] == [] + + +@_requires_composio +class TestTriggersCatalogEvents: + def test_browse_events_returns_200(self, authed_api): + response = authed_api( + "GET", + "/triggers/catalog/providers/composio/integrations/github/events/", + ) + assert response.status_code == 200 + body = response.json() + assert "events" in body + assert isinstance(body["events"], list) + + def test_fetch_event_config_schema(self, authed_api): + """A single event carries its trigger_config JSON Schema.""" + listing = authed_api( + "GET", + "/triggers/catalog/providers/composio/integrations/github/events/", + ).json() + if not listing["events"]: + pytest.skip("no github events available from Composio") + + event_key = listing["events"][0]["key"] + response = authed_api( + "GET", + f"/triggers/catalog/providers/composio/integrations/github/events/{event_key}", + ) + assert response.status_code == 200 + event = response.json()["event"] + assert event["key"] == event_key + # trigger_config is the inbound analogue of an action's input_parameters + assert "trigger_config" in event From b3787e3427e2cb96ce87aa3438bf8cf4c619e1c0 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Thu, 18 Jun 2026 17:47:33 +0200 Subject: [PATCH 2/5] refactor(sdk): promote mapping resolver to agenta.sdk.utils.resolvers Move resolve_payload_fields out of core/webhooks/delivery.py into the SDK as resolve_target_fields (next to resolve_json_selector) so triggers and webhooks share it without a cross-domain import. Pure move + rename, no behavior change; its live consumer today is webhooks. Co-Authored-By: Claude Opus 4.8 (1M context) --- api/oss/src/core/tools/interfaces.py | 88 +++--- api/oss/src/core/tools/service.py | 270 +++++++++--------- api/oss/src/core/webhooks/delivery.py | 29 +- .../unit/webhooks/test_webhooks_tasks.py | 29 +- sdks/python/agenta/sdk/utils/resolvers.py | 32 +++ 5 files changed, 229 insertions(+), 219 deletions(-) diff --git a/api/oss/src/core/tools/interfaces.py b/api/oss/src/core/tools/interfaces.py index 0e459619e6..0a61d59ee9 100644 --- a/api/oss/src/core/tools/interfaces.py +++ b/api/oss/src/core/tools/interfaces.py @@ -20,53 +20,53 @@ class ToolsGatewayInterface(ABC): @abstractmethod async def list_providers(self) -> List[ToolCatalogProvider]: ... - - @abstractmethod - async def list_integrations( - self, - *, - search: Optional[str] = None, - sort_by: Optional[str] = None, - limit: Optional[int] = None, - cursor: Optional[str] = None, - ) -> Tuple[List[ToolCatalogIntegration], Optional[str], int]: - """Returns (items, next_cursor, total_items).""" - ... - - @abstractmethod - async def get_integration( - self, - *, - integration_key: str, - ) -> Optional[ToolCatalogIntegration]: ... - - @abstractmethod - async def list_actions( - self, - *, - integration_key: str, - query: Optional[str] = None, - categories: Optional[List[str]] = None, - important: Optional[bool] = None, - limit: Optional[int] = None, - cursor: Optional[str] = None, - ) -> Tuple[List[ToolCatalogAction], Optional[str], int]: - """Returns (items, next_cursor, total_items).""" - ... - - @abstractmethod - async def get_action( - self, - *, - integration_key: str, + + @abstractmethod + async def list_integrations( + self, + *, + search: Optional[str] = None, + sort_by: Optional[str] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[ToolCatalogIntegration], Optional[str], int]: + """Returns (items, next_cursor, total_items).""" + ... + + @abstractmethod + async def get_integration( + self, + *, + integration_key: str, + ) -> Optional[ToolCatalogIntegration]: ... + + @abstractmethod + async def list_actions( + self, + *, + integration_key: str, + query: Optional[str] = None, + categories: Optional[List[str]] = None, + important: Optional[bool] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[ToolCatalogAction], Optional[str], int]: + """Returns (items, next_cursor, total_items).""" + ... + + @abstractmethod + async def get_action( + self, + *, + integration_key: str, action_key: str, ) -> Optional[ToolCatalogActionDetails]: ... @abstractmethod async def execute( self, - *, - request: ToolExecutionRequest, - ) -> ToolExecutionResponse: - """Execute a tool action.""" - ... + *, + request: ToolExecutionRequest, + ) -> ToolExecutionResponse: + """Execute a tool action.""" + ... diff --git a/api/oss/src/core/tools/service.py b/api/oss/src/core/tools/service.py index 4b30ca6121..bf8eed2b9d 100644 --- a/api/oss/src/core/tools/service.py +++ b/api/oss/src/core/tools/service.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from uuid import UUID from oss.src.utils.logging import get_module_logger @@ -18,9 +18,9 @@ log = get_module_logger(__name__) - - -class ToolsService: + + +class ToolsService: def __init__( self, *, @@ -31,95 +31,95 @@ def __init__( self.adapter_registry = adapter_registry # ----------------------------------------------------------------------- - # Catalog browse - # ----------------------------------------------------------------------- - - async def list_providers(self) -> List[ToolCatalogProvider]: - """Return all providers across registered adapters.""" - results: List[ToolCatalogProvider] = [] - for _key, adapter in self.adapter_registry.items(): - providers = await adapter.list_providers() - results.extend(providers) - return results - - async def get_provider( - self, - *, - provider_key: str, - ) -> Optional[ToolCatalogProvider]: - """Return a single provider by key, or None if not found.""" - adapter = self.adapter_registry.get(provider_key) - providers = await adapter.list_providers() - for p in providers: - if p.key == provider_key: - return p - return None - - async def list_integrations( - self, - *, - provider_key: str, - # - search: Optional[str] = None, - sort_by: Optional[str] = None, - limit: Optional[int] = None, - cursor: Optional[str] = None, - ) -> Tuple[List[ToolCatalogIntegration], Optional[str], int]: - """List integrations for a provider with optional filtering and pagination.""" - adapter = self.adapter_registry.get(provider_key) - integrations, next_cursor, total = await adapter.list_integrations( - search=search, - sort_by=sort_by, - limit=limit, - cursor=cursor, - ) - return integrations, next_cursor, total - - async def get_integration( - self, - *, - provider_key: str, - integration_key: str, - ) -> Optional[ToolCatalogIntegration]: - """Return a single integration by key, or None if not found.""" - adapter = self.adapter_registry.get(provider_key) - return await adapter.get_integration(integration_key=integration_key) - - async def list_actions( - self, - *, - provider_key: str, - integration_key: str, - # - query: Optional[str] = None, - categories: Optional[List[str]] = None, - important: Optional[bool] = None, - limit: Optional[int] = None, - cursor: Optional[str] = None, - ) -> Tuple[List[ToolCatalogAction], Optional[str], int]: - """List actions for an integration with optional search and pagination.""" - adapter = self.adapter_registry.get(provider_key) - return await adapter.list_actions( - integration_key=integration_key, - query=query, - categories=categories, - important=important, - limit=limit, - cursor=cursor, - ) - - async def get_action( - self, - *, - provider_key: str, - integration_key: str, - action_key: str, - ) -> Optional[ToolCatalogActionDetails]: - """Return full action detail including input/output schema, or None if not found.""" - adapter = self.adapter_registry.get(provider_key) - return await adapter.get_action( - integration_key=integration_key, - action_key=action_key, + # Catalog browse + # ----------------------------------------------------------------------- + + async def list_providers(self) -> List[ToolCatalogProvider]: + """Return all providers across registered adapters.""" + results: List[ToolCatalogProvider] = [] + for _key, adapter in self.adapter_registry.items(): + providers = await adapter.list_providers() + results.extend(providers) + return results + + async def get_provider( + self, + *, + provider_key: str, + ) -> Optional[ToolCatalogProvider]: + """Return a single provider by key, or None if not found.""" + adapter = self.adapter_registry.get(provider_key) + providers = await adapter.list_providers() + for p in providers: + if p.key == provider_key: + return p + return None + + async def list_integrations( + self, + *, + provider_key: str, + # + search: Optional[str] = None, + sort_by: Optional[str] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[ToolCatalogIntegration], Optional[str], int]: + """List integrations for a provider with optional filtering and pagination.""" + adapter = self.adapter_registry.get(provider_key) + integrations, next_cursor, total = await adapter.list_integrations( + search=search, + sort_by=sort_by, + limit=limit, + cursor=cursor, + ) + return integrations, next_cursor, total + + async def get_integration( + self, + *, + provider_key: str, + integration_key: str, + ) -> Optional[ToolCatalogIntegration]: + """Return a single integration by key, or None if not found.""" + adapter = self.adapter_registry.get(provider_key) + return await adapter.get_integration(integration_key=integration_key) + + async def list_actions( + self, + *, + provider_key: str, + integration_key: str, + # + query: Optional[str] = None, + categories: Optional[List[str]] = None, + important: Optional[bool] = None, + limit: Optional[int] = None, + cursor: Optional[str] = None, + ) -> Tuple[List[ToolCatalogAction], Optional[str], int]: + """List actions for an integration with optional search and pagination.""" + adapter = self.adapter_registry.get(provider_key) + return await adapter.list_actions( + integration_key=integration_key, + query=query, + categories=categories, + important=important, + limit=limit, + cursor=cursor, + ) + + async def get_action( + self, + *, + provider_key: str, + integration_key: str, + action_key: str, + ) -> Optional[ToolCatalogActionDetails]: + """Return full action detail including input/output schema, or None if not found.""" + adapter = self.adapter_registry.get(provider_key) + return await adapter.get_action( + integration_key=integration_key, + action_key=action_key, ) # ----------------------------------------------------------------------- @@ -127,10 +127,10 @@ async def get_action( # ----------------------------------------------------------------------- async def query_connections( - self, - *, - project_id: UUID, - # + self, + *, + project_id: UUID, + # provider_key: Optional[str] = None, integration_key: Optional[str] = None, is_active: Optional[bool] = True, @@ -153,10 +153,10 @@ async def list_connections( project_id=project_id, provider_key=provider_key, integration_key=integration_key, - ) - - async def get_connection( - self, + ) + + async def get_connection( + self, *, project_id: UUID, connection_id: UUID, @@ -198,12 +198,12 @@ async def create_connection( project_id=project_id, user_id=user_id, # - connection_create=connection_create, - ) - - async def delete_connection( - self, - *, + connection_create=connection_create, + ) + + async def delete_connection( + self, + *, project_id: UUID, connection_id: UUID, ) -> bool: @@ -211,9 +211,9 @@ async def delete_connection( project_id=project_id, connection_id=connection_id, ) - - async def revoke_connection( - self, + + async def revoke_connection( + self, *, project_id: UUID, connection_id: UUID, @@ -226,7 +226,7 @@ async def revoke_connection( async def refresh_connection( self, *, - project_id: UUID, + project_id: UUID, connection_id: UUID, # force: bool = False, @@ -239,27 +239,27 @@ async def refresh_connection( # ----------------------------------------------------------------------- # Tool execution - # ----------------------------------------------------------------------- - - async def execute_tool( - self, - *, - provider_key: str, - integration_key: str, - action_key: str, - provider_connection_id: str, - user_id: Optional[str] = None, - arguments: Dict[str, Any], - ) -> ToolExecutionResponse: - """Execute a tool action using the provider adapter.""" - adapter = self.adapter_registry.get(provider_key) - - return await adapter.execute( - request=ToolExecutionRequest( - integration_key=integration_key, - action_key=action_key, - provider_connection_id=provider_connection_id, - user_id=user_id, - arguments=arguments, - ), - ) + # ----------------------------------------------------------------------- + + async def execute_tool( + self, + *, + provider_key: str, + integration_key: str, + action_key: str, + provider_connection_id: str, + user_id: Optional[str] = None, + arguments: Dict[str, Any], + ) -> ToolExecutionResponse: + """Execute a tool action using the provider adapter.""" + adapter = self.adapter_registry.get(provider_key) + + return await adapter.execute( + request=ToolExecutionRequest( + integration_key=integration_key, + action_key=action_key, + provider_connection_id=provider_connection_id, + user_id=user_id, + arguments=arguments, + ), + ) diff --git a/api/oss/src/core/webhooks/delivery.py b/api/oss/src/core/webhooks/delivery.py index 280c3e1a8b..9ca44f3e87 100644 --- a/api/oss/src/core/webhooks/delivery.py +++ b/api/oss/src/core/webhooks/delivery.py @@ -8,7 +8,7 @@ import httpx -from agenta.sdk.utils.resolvers import resolve_json_selector +from agenta.sdk.utils.resolvers import resolve_target_fields from oss.src.core.webhooks.types import ( EVENT_CONTEXT_FIELDS, @@ -23,8 +23,6 @@ log = get_module_logger(__name__) -MAX_RESOLVE_DEPTH = 10 - NON_OVERRIDABLE_HEADERS = { "content-type", "content-length", @@ -92,29 +90,6 @@ def _merge_headers( return merged -def resolve_payload_fields( - fields: Any, - context: Dict[str, Any], - *, - _depth: int = 0, -) -> Any: - if _depth > MAX_RESOLVE_DEPTH: - return None - if isinstance(fields, dict): - return { - k: resolve_payload_fields(v, context, _depth=_depth + 1) - for k, v in fields.items() - } - if isinstance(fields, list): - return [ - resolve_payload_fields(item, context, _depth=_depth + 1) for item in fields - ] - try: - return resolve_json_selector(fields, context) - except Exception: - return None - - def prepare_webhook_request( *, project_id: UUID, @@ -147,7 +122,7 @@ def prepare_webhook_request( } resolved_fields = payload_fields if payload_fields is not None else "$" - payload = resolve_payload_fields(resolved_fields, context) + payload = resolve_target_fields(resolved_fields, context) base_data = WebhookDeliveryData( event_type=typed_event_type, diff --git a/api/oss/tests/pytest/unit/webhooks/test_webhooks_tasks.py b/api/oss/tests/pytest/unit/webhooks/test_webhooks_tasks.py index 1ca605df49..0f35b272a3 100644 --- a/api/oss/tests/pytest/unit/webhooks/test_webhooks_tasks.py +++ b/api/oss/tests/pytest/unit/webhooks/test_webhooks_tasks.py @@ -5,11 +5,14 @@ from unittest.mock import patch -from oss.src.core.webhooks.delivery import ( +from agenta.sdk.utils.resolvers import ( MAX_RESOLVE_DEPTH, + resolve_target_fields, +) + +from oss.src.core.webhooks.delivery import ( NON_OVERRIDABLE_HEADERS, _merge_headers, - resolve_payload_fields, ) from oss.src.core.webhooks.types import ( EVENT_CONTEXT_FIELDS, @@ -35,18 +38,18 @@ "scope": {"project_id": "proj-1"}, } -_RESOLVE_PATH = "oss.src.core.webhooks.delivery.resolve_json_selector" +_RESOLVE_PATH = "agenta.sdk.utils.resolvers.resolve_json_selector" # --------------------------------------------------------------------------- -# resolve_payload_fields +# resolve_target_fields # --------------------------------------------------------------------------- class TestResolvePayloadFields: def test_dict_recurses_into_values(self): with patch(_RESOLVE_PATH, side_effect=lambda expr, ctx: f"resolved:{expr}"): - result = resolve_payload_fields( + result = resolve_target_fields( {"key": "$.event.event_id"}, _MOCK_CONTEXT, ) @@ -54,7 +57,7 @@ def test_dict_recurses_into_values(self): def test_list_recurses_into_items(self): with patch(_RESOLVE_PATH, side_effect=lambda expr, ctx: f"resolved:{expr}"): - result = resolve_payload_fields( + result = resolve_target_fields( ["$.event.event_id", "$.scope.project_id"], _MOCK_CONTEXT, ) @@ -65,12 +68,12 @@ def test_list_recurses_into_items(self): def test_primitive_delegates_to_resolve_json_selector(self): with patch(_RESOLVE_PATH, return_value="abc123") as mock_resolve: - result = resolve_payload_fields("$.event.event_id", _MOCK_CONTEXT) + result = resolve_target_fields("$.event.event_id", _MOCK_CONTEXT) assert result == "abc123" mock_resolve.assert_called_once_with("$.event.event_id", _MOCK_CONTEXT) def test_depth_exceeds_limit_returns_none(self): - result = resolve_payload_fields( + result = resolve_target_fields( "$.event.event_id", _MOCK_CONTEXT, _depth=MAX_RESOLVE_DEPTH + 1, @@ -79,7 +82,7 @@ def test_depth_exceeds_limit_returns_none(self): def test_depth_at_limit_still_resolves(self): with patch(_RESOLVE_PATH, return_value="ok"): - result = resolve_payload_fields( + result = resolve_target_fields( "$.event.event_id", _MOCK_CONTEXT, _depth=MAX_RESOLVE_DEPTH, @@ -88,7 +91,7 @@ def test_depth_at_limit_still_resolves(self): def test_resolve_error_returns_none(self): with patch(_RESOLVE_PATH, side_effect=ValueError("bad selector")): - result = resolve_payload_fields("$.bad[", _MOCK_CONTEXT) + result = resolve_target_fields("$.bad[", _MOCK_CONTEXT) assert result is None def test_error_leaf_in_dict_does_not_affect_other_keys(self): @@ -98,7 +101,7 @@ def side_effect(expr, ctx): return "good" with patch(_RESOLVE_PATH, side_effect=side_effect): - result = resolve_payload_fields( + result = resolve_target_fields( {"ok": "$.event.event_id", "bad": "$.bad["}, _MOCK_CONTEXT, ) @@ -106,14 +109,14 @@ def side_effect(expr, ctx): def test_dollar_selector_resolves_full_context(self): with patch(_RESOLVE_PATH, return_value=_MOCK_CONTEXT) as mock_resolve: - result = resolve_payload_fields("$", _MOCK_CONTEXT) + result = resolve_target_fields("$", _MOCK_CONTEXT) assert result == _MOCK_CONTEXT mock_resolve.assert_called_once_with("$", _MOCK_CONTEXT) def test_nested_dict_depth_tracking(self): # Three levels deep should still work (depth starts at 0) with patch(_RESOLVE_PATH, return_value="leaf"): - result = resolve_payload_fields( + result = resolve_target_fields( {"a": {"b": {"c": "$.event.event_id"}}}, _MOCK_CONTEXT, ) diff --git a/sdks/python/agenta/sdk/utils/resolvers.py b/sdks/python/agenta/sdk/utils/resolvers.py index b7b51ed5c4..a512a27489 100644 --- a/sdks/python/agenta/sdk/utils/resolvers.py +++ b/sdks/python/agenta/sdk/utils/resolvers.py @@ -12,6 +12,8 @@ log = get_module_logger(__name__) +MAX_RESOLVE_DEPTH = 10 + # ========= Scheme detection ========= @@ -132,3 +134,33 @@ def resolve_json_selector(value: Any, data: Dict[str, Any]) -> Any: log.debug("Failed to resolve JSON selector %r: %s", value, exc) return None return value + + +def resolve_target_fields( + template: Any, + context: Dict[str, Any], + *, + _depth: int = 0, +) -> Any: + """Resolve a template into a target by resolving its selector leaves. + + Walks ``template`` (arbitrary JSON); each leaf is passed through + ``resolve_json_selector`` against *context* (``$``/``/`` selectors resolved, + everything else returned literally). Null-on-miss, depth-capped at + ``MAX_RESOLVE_DEPTH``. + """ + if _depth > MAX_RESOLVE_DEPTH: + return None + if isinstance(template, dict): + return { + k: resolve_target_fields(v, context, _depth=_depth + 1) + for k, v in template.items() + } + if isinstance(template, list): + return [ + resolve_target_fields(item, context, _depth=_depth + 1) for item in template + ] + try: + return resolve_json_selector(template, context) + except Exception: + return None From 5e9e904014e153600f0801a235af7161529551b0 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Thu, 18 Jun 2026 18:23:23 +0200 Subject: [PATCH 3/5] feat(api): add trigger subscriptions and deliveries with CRUD and dedup Adds the two-table subscriptions/deliveries heart of the triggers domain, modeled on webhook_subscriptions/webhook_deliveries. Subscriptions FK the shared gateway_connections row and carry the inputs_fields mapping template; deliveries record resolved inputs/result/error with an event_id dedup column. Subscription CRUD drives the Composio ti_* lifecycle through the adapter; deleting a subscription leaves the shared connection intact. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../triggers/test_triggers_subscriptions.py | 226 ++++++++++ api/entrypoints/routers.py | 5 + ...dd_trigger_subscriptions_and_deliveries.py | 179 ++++++++ api/oss/src/apis/fastapi/triggers/models.py | 57 +++ api/oss/src/apis/fastapi/triggers/router.py | 390 +++++++++++++++++- api/oss/src/core/triggers/dtos.py | 134 ++++++ api/oss/src/core/triggers/exceptions.py | 16 + api/oss/src/core/triggers/interfaces.py | 119 ++++++ api/oss/src/core/triggers/service.py | 304 +++++++++++++- api/oss/src/dbs/postgres/triggers/dao.py | 378 +++++++++++++++++ api/oss/src/dbs/postgres/triggers/dbas.py | 53 +++ api/oss/src/dbs/postgres/triggers/dbes.py | 75 ++++ api/oss/src/dbs/postgres/triggers/mappings.py | 179 ++++++++ .../triggers/test_triggers_subscriptions.py | 155 +++++++ 14 files changed, 2267 insertions(+), 3 deletions(-) create mode 100644 api/ee/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py create mode 100644 api/oss/databases/postgres/migrations/core_oss/versions/oss000000003_add_trigger_subscriptions_and_deliveries.py create mode 100644 api/oss/src/dbs/postgres/triggers/dao.py create mode 100644 api/oss/src/dbs/postgres/triggers/dbas.py create mode 100644 api/oss/src/dbs/postgres/triggers/dbes.py create mode 100644 api/oss/src/dbs/postgres/triggers/mappings.py create mode 100644 api/oss/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py diff --git a/api/ee/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py b/api/ee/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py new file mode 100644 index 0000000000..d68b42acaa --- /dev/null +++ b/api/ee/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py @@ -0,0 +1,226 @@ +"""EE acceptance tests for /triggers/subscriptions/* and /triggers/deliveries/*. + +Mirrors the OSS suite but exercises the routes as a business-plan, +developer-role account. Subscription CRUD is gated on EDIT_TRIGGERS and reads on +VIEW_TRIGGERS; a developer role carries both, so this verifies the routes behave +once the gate is satisfied. + +The read/query surfaces are DB-only (no Composio needed). The full create -> +list -> disable -> delete roundtrip, including the C7 invariant (deleting a +subscription leaves the shared connection intact), mints a provider-side trigger +instance and is gated on COMPOSIO_API_KEY. + +Requires a running API. +""" + +import os +from uuid import uuid4 + +import pytest +import requests + +from utils.constants import BASE_TIMEOUT + + +_COMPOSIO_ENABLED = bool(os.getenv("COMPOSIO_API_KEY")) +_requires_composio = pytest.mark.skipif( + not _COMPOSIO_ENABLED, + reason="needs live Composio credentials (COMPOSIO_API_KEY)", +) + + +def _create_developer_business_account(admin_api): + uid = uuid4().hex[:12] + email = f"triggers-sub-dev-{uid}@test.agenta.ai" + resp = admin_api( + "POST", + "/admin/simple/accounts/", + json={ + "accounts": { + "u": { + "user": {"email": email}, + "options": { + "create_api_keys": True, + "return_api_keys": True, + "seed_defaults": False, + }, + "subscription": {"plan": "cloud_v0_business"}, + "organization_memberships": [ + { + "organization_ref": {"ref": "org"}, + "user_ref": {"ref": "user"}, + "role": "developer", + } + ], + "workspace_memberships": [ + { + "workspace_ref": {"ref": "wrk"}, + "user_ref": {"ref": "user"}, + "role": "developer", + } + ], + "project_memberships": [ + { + "project_ref": {"ref": "prj"}, + "user_ref": {"ref": "user"}, + "role": "developer", + } + ], + } + } + }, + ) + assert resp.status_code == 200, resp.text + account = resp.json()["accounts"]["u"] + return { + "email": email, + "credentials": f"ApiKey {account['api_keys']['key']}", + } + + +def _delete_account_by_email(admin_api, *, email): + resp = admin_api( + "DELETE", + "/admin/simple/accounts/", + json={"accounts": {"u": {"user": {"email": email}}}, "confirm": "delete"}, + ) + assert resp.status_code == 204, resp.text + + +@pytest.fixture(scope="class") +def triggers_api(admin_api, ag_env): + account = _create_developer_business_account(admin_api) + + def _request(method: str, endpoint: str, **kwargs): + headers = kwargs.pop("headers", {}) + headers.setdefault("Authorization", account["credentials"]) + return requests.request( + method=method, + url=f"{ag_env['api_url']}{endpoint}", + headers=headers, + timeout=BASE_TIMEOUT, + **kwargs, + ) + + yield _request + + _delete_account_by_email(admin_api, email=account["email"]) + + +# --------------------------------------------------------------------------- +# DB-only: reads, queries, 404s +# --------------------------------------------------------------------------- + + +class TestTriggerSubscriptionsReads: + def test_list_subscriptions_returns_200_empty(self, triggers_api): + response = triggers_api("GET", "/triggers/subscriptions/") + assert response.status_code == 200 + body = response.json() + assert "count" in body + assert isinstance(body["subscriptions"], list) + assert body["count"] == len(body["subscriptions"]) + + def test_query_subscriptions_returns_200(self, triggers_api): + response = triggers_api("POST", "/triggers/subscriptions/query", json={}) + assert response.status_code == 200 + body = response.json() + assert body["count"] == len(body["subscriptions"]) + + def test_fetch_unknown_subscription_returns_404(self, triggers_api): + response = triggers_api("GET", f"/triggers/subscriptions/{uuid4()}") + assert response.status_code == 404 + + def test_delete_unknown_subscription_returns_404(self, triggers_api): + response = triggers_api("DELETE", f"/triggers/subscriptions/{uuid4()}") + assert response.status_code == 404 + + +class TestTriggerDeliveriesReads: + def test_list_deliveries_returns_200_empty(self, triggers_api): + response = triggers_api("GET", "/triggers/deliveries") + assert response.status_code == 200 + body = response.json() + assert isinstance(body["deliveries"], list) + assert body["count"] == len(body["deliveries"]) + + def test_query_deliveries_returns_200(self, triggers_api): + response = triggers_api("POST", "/triggers/deliveries/query", json={}) + assert response.status_code == 200 + body = response.json() + assert body["count"] == len(body["deliveries"]) + + def test_fetch_unknown_delivery_returns_404(self, triggers_api): + response = triggers_api("GET", f"/triggers/deliveries/{uuid4()}") + assert response.status_code == 404 + + +# --------------------------------------------------------------------------- +# Full lifecycle (needs Composio) — C7 invariant included +# --------------------------------------------------------------------------- + + +@_requires_composio +class TestTriggerSubscriptionsLifecycle: + def _create_connection(self, triggers_api): + slug = f"acc-{uuid4().hex[:8]}" + create = triggers_api( + "POST", + "/tools/connections/", + json={ + "connection": { + "slug": slug, + "provider_key": "composio", + "integration_key": "github", + "data": {"auth_scheme": "oauth"}, + } + }, + ) + assert create.status_code == 200, create.text + return create.json()["connection"]["id"] + + def test_create_list_disable_delete_keeps_connection(self, triggers_api): + connection_id = self._create_connection(triggers_api) + + create = triggers_api( + "POST", + "/triggers/subscriptions/", + json={ + "subscription": { + "name": f"sub-{uuid4().hex[:8]}", + "connection_id": connection_id, + "data": { + "event_key": "GITHUB_STAR_ADDED_EVENT", + "trigger_config": {}, + "inputs_fields": {"repo": "$.event.data.repository"}, + "references": {"workflow": {"slug": "triage"}}, + }, + } + }, + ) + assert create.status_code == 200, create.text + sub = create.json()["subscription"] + subscription_id = sub["id"] + assert sub["connection_id"] == connection_id + assert sub["data"]["ti_id"] is not None + + listing = triggers_api("GET", "/triggers/subscriptions/").json() + assert any(s["id"] == subscription_id for s in listing["subscriptions"]) + + revoke = triggers_api( + "POST", f"/triggers/subscriptions/{subscription_id}/revoke" + ) + assert revoke.status_code == 200, revoke.text + assert revoke.json()["subscription"]["enabled"] is False + + delete = triggers_api("DELETE", f"/triggers/subscriptions/{subscription_id}") + assert delete.status_code == 204 + + fetch = triggers_api("GET", f"/triggers/subscriptions/{subscription_id}") + assert fetch.status_code == 404 + + # C7: deleting the subscription must NOT delete/revoke the connection. + conn = triggers_api("GET", f"/tools/connections/{connection_id}") + assert conn.status_code == 200, conn.text + + triggers_api("DELETE", f"/tools/connections/{connection_id}") diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index 96c11eb9a8..97e5b02314 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -142,6 +142,7 @@ from oss.src.core.tools.registry import ToolsGatewayRegistry from oss.src.core.tools.service import ToolsService from oss.src.apis.fastapi.tools.router import ToolsRouter +from oss.src.dbs.postgres.triggers.dao import TriggersDAO from oss.src.core.triggers.providers.composio import ComposioTriggersAdapter from oss.src.core.triggers.registry import TriggersGatewayRegistry from oss.src.core.triggers.service import TriggersService @@ -640,8 +641,12 @@ async def lifespan(*args, **kwargs): adapters=_composio_triggers_adapters, ) +triggers_dao = TriggersDAO(engine=_transactions_engine) + triggers_service = TriggersService( adapter_registry=triggers_adapter_registry, + triggers_dao=triggers_dao, + connections_service=connections_service, ) _t_services_done = time.perf_counter() - _t_services diff --git a/api/oss/databases/postgres/migrations/core_oss/versions/oss000000003_add_trigger_subscriptions_and_deliveries.py b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000003_add_trigger_subscriptions_and_deliveries.py new file mode 100644 index 0000000000..c755fbebcc --- /dev/null +++ b/api/oss/databases/postgres/migrations/core_oss/versions/oss000000003_add_trigger_subscriptions_and_deliveries.py @@ -0,0 +1,179 @@ +"""add trigger_subscriptions and trigger_deliveries tables + +The two-table heart of the gateway-triggers domain (WP3), modeled on +webhook_subscriptions + webhook_deliveries. A subscription FKs the shared +gateway_connections row (many subscriptions per connection); a delivery dedups +on the provider event id (metadata.id) per subscription (I4). Authored once in +the shared core_oss chain so it runs in BOTH editions. + +Revision ID: oss000000003 +Revises: oss000000002 +Create Date: 2026-06-18 00:00:01.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "oss000000003" +down_revision: Union[str, None] = "oss000000002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # -- TRIGGER SUBSCRIPTIONS -------------------------------------------------- + op.create_table( + "trigger_subscriptions", + sa.Column("project_id", sa.UUID(), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("connection_id", sa.UUID(), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("data", postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column( + "flags", + postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), + nullable=True, + ), + sa.Column("meta", postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column( + "tags", + postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), + nullable=True, + ), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + server_onupdate=sa.text("CURRENT_TIMESTAMP"), + nullable=True, + ), + sa.Column("deleted_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("created_by_id", sa.UUID(), nullable=True), + sa.Column("updated_by_id", sa.UUID(), nullable=True), + sa.Column("deleted_by_id", sa.UUID(), nullable=True), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["project_id", "connection_id"], + ["gateway_connections.project_id", "gateway_connections.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_id", "id"), + ) + + op.create_index( + "ix_trigger_subscriptions_project_id_created_at", + "trigger_subscriptions", + ["project_id", "created_at"], + unique=False, + ) + op.create_index( + "ix_trigger_subscriptions_project_id_deleted_at", + "trigger_subscriptions", + ["project_id", "deleted_at"], + unique=False, + ) + op.create_index( + "ix_trigger_subscriptions_connection_id", + "trigger_subscriptions", + ["project_id", "connection_id"], + unique=False, + ) + + # -- TRIGGER DELIVERIES ----------------------------------------------------- + op.create_table( + "trigger_deliveries", + sa.Column("project_id", sa.UUID(), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("subscription_id", sa.UUID(), nullable=False), + sa.Column("event_id", sa.String(), nullable=False), + sa.Column( + "status", + postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), + nullable=True, + ), + sa.Column("data", postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + server_onupdate=sa.text("CURRENT_TIMESTAMP"), + nullable=True, + ), + sa.Column("deleted_at", sa.TIMESTAMP(timezone=True), nullable=True), + sa.Column("created_by_id", sa.UUID(), nullable=True), + sa.Column("updated_by_id", sa.UUID(), nullable=True), + sa.Column("deleted_by_id", sa.UUID(), nullable=True), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["project_id", "subscription_id"], + ["trigger_subscriptions.project_id", "trigger_subscriptions.id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("project_id", "id"), + ) + + op.create_index( + "ix_trigger_deliveries_project_id_created_at", + "trigger_deliveries", + ["project_id", "created_at"], + unique=False, + ) + op.create_index( + "ix_trigger_deliveries_subscription_id_created_at", + "trigger_deliveries", + ["subscription_id", "created_at"], + unique=False, + ) + op.create_index( + "ix_trigger_deliveries_subscription_id_event_id", + "trigger_deliveries", + ["project_id", "subscription_id", "event_id"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index( + "ix_trigger_deliveries_subscription_id_event_id", + table_name="trigger_deliveries", + ) + op.drop_index( + "ix_trigger_deliveries_subscription_id_created_at", + table_name="trigger_deliveries", + ) + op.drop_index( + "ix_trigger_deliveries_project_id_created_at", + table_name="trigger_deliveries", + ) + op.drop_table("trigger_deliveries") + + op.drop_index( + "ix_trigger_subscriptions_connection_id", + table_name="trigger_subscriptions", + ) + op.drop_index( + "ix_trigger_subscriptions_project_id_deleted_at", + table_name="trigger_subscriptions", + ) + op.drop_index( + "ix_trigger_subscriptions_project_id_created_at", + table_name="trigger_subscriptions", + ) + op.drop_table("trigger_subscriptions") diff --git a/api/oss/src/apis/fastapi/triggers/models.py b/api/oss/src/apis/fastapi/triggers/models.py index 93e9d5ab4e..d89035dca6 100644 --- a/api/oss/src/apis/fastapi/triggers/models.py +++ b/api/oss/src/apis/fastapi/triggers/models.py @@ -2,10 +2,17 @@ from pydantic import BaseModel +from oss.src.core.shared.dtos import Windowing from oss.src.core.triggers.dtos import ( TriggerCatalogEvent, TriggerCatalogEventDetails, TriggerCatalogProvider, + TriggerDelivery, + TriggerDeliveryQuery, + TriggerSubscription, + TriggerSubscriptionCreate, + TriggerSubscriptionEdit, + TriggerSubscriptionQuery, ) @@ -34,3 +41,53 @@ class TriggerCatalogEventsResponse(BaseModel): total: int = 0 cursor: Optional[str] = None events: List[TriggerCatalogEvent] = [] + + +# --------------------------------------------------------------------------- +# Trigger Subscriptions +# --------------------------------------------------------------------------- + + +class TriggerSubscriptionCreateRequest(BaseModel): + subscription: TriggerSubscriptionCreate + + +class TriggerSubscriptionEditRequest(BaseModel): + subscription: TriggerSubscriptionEdit + + +class TriggerSubscriptionQueryRequest(BaseModel): + subscription: Optional[TriggerSubscriptionQuery] = None + + windowing: Optional[Windowing] = None + + +class TriggerSubscriptionResponse(BaseModel): + count: int = 0 + subscription: Optional[TriggerSubscription] = None + + +class TriggerSubscriptionsResponse(BaseModel): + count: int = 0 + subscriptions: List[TriggerSubscription] = [] + + +# --------------------------------------------------------------------------- +# Trigger Deliveries +# --------------------------------------------------------------------------- + + +class TriggerDeliveryQueryRequest(BaseModel): + delivery: Optional[TriggerDeliveryQuery] = None + + windowing: Optional[Windowing] = None + + +class TriggerDeliveryResponse(BaseModel): + count: int = 0 + delivery: Optional[TriggerDelivery] = None + + +class TriggerDeliveriesResponse(BaseModel): + count: int = 0 + deliveries: List[TriggerDelivery] = [] diff --git a/api/oss/src/apis/fastapi/triggers/router.py b/api/oss/src/apis/fastapi/triggers/router.py index 5270682dc4..2090d28942 100644 --- a/api/oss/src/apis/fastapi/triggers/router.py +++ b/api/oss/src/apis/fastapi/triggers/router.py @@ -1,5 +1,6 @@ from functools import wraps from typing import Optional +from uuid import UUID import httpx from fastapi import APIRouter, HTTPException, Query, Request, status @@ -15,8 +16,20 @@ TriggerCatalogEventsResponse, TriggerCatalogProviderResponse, TriggerCatalogProvidersResponse, + TriggerDeliveriesResponse, + TriggerDeliveryQueryRequest, + TriggerDeliveryResponse, + TriggerSubscriptionCreateRequest, + TriggerSubscriptionEditRequest, + TriggerSubscriptionQueryRequest, + TriggerSubscriptionResponse, + TriggerSubscriptionsResponse, +) +from oss.src.core.triggers.exceptions import ( + AdapterError, + ConnectionNotFoundError, + SubscriptionNotFoundError, ) -from oss.src.core.triggers.exceptions import AdapterError from oss.src.core.triggers.service import TriggersService @@ -101,6 +114,107 @@ def __init__( response_model_exclude_none=True, ) + # --- Trigger Subscriptions --- + self.router.add_api_route( + "/subscriptions/", + self.create_subscription, + methods=["POST"], + operation_id="create_trigger_subscription", + response_model=TriggerSubscriptionResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/", + self.list_subscriptions, + methods=["GET"], + operation_id="list_trigger_subscriptions", + response_model=TriggerSubscriptionsResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/query", + self.query_subscriptions, + methods=["POST"], + operation_id="query_trigger_subscriptions", + response_model=TriggerSubscriptionsResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/{subscription_id}/refresh", + self.refresh_subscription, + methods=["POST"], + operation_id="refresh_trigger_subscription", + response_model=TriggerSubscriptionResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/{subscription_id}/revoke", + self.revoke_subscription, + methods=["POST"], + operation_id="revoke_trigger_subscription", + response_model=TriggerSubscriptionResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/{subscription_id}", + self.fetch_subscription, + methods=["GET"], + operation_id="fetch_trigger_subscription", + response_model=TriggerSubscriptionResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/{subscription_id}", + self.edit_subscription, + methods=["PUT"], + operation_id="edit_trigger_subscription", + response_model=TriggerSubscriptionResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/subscriptions/{subscription_id}", + self.delete_subscription, + methods=["DELETE"], + operation_id="delete_trigger_subscription", + status_code=status.HTTP_204_NO_CONTENT, + ) + + # --- Trigger Deliveries --- + self.router.add_api_route( + "/deliveries", + self.list_deliveries, + methods=["GET"], + operation_id="list_trigger_deliveries", + response_model=TriggerDeliveriesResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/deliveries/query", + self.query_deliveries, + methods=["POST"], + operation_id="query_trigger_deliveries", + response_model=TriggerDeliveriesResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + self.router.add_api_route( + "/deliveries/{delivery_id}", + self.fetch_delivery, + methods=["GET"], + operation_id="fetch_trigger_delivery", + response_model=TriggerDeliveryResponse, + response_model_exclude_none=True, + status_code=status.HTTP_200_OK, + ) + # ----------------------------------------------------------------------- # Trigger Catalog # ----------------------------------------------------------------------- @@ -317,3 +431,277 @@ async def get_event( ) return response + + # ----------------------------------------------------------------------- + # Trigger Subscriptions + # ----------------------------------------------------------------------- + + async def _check(self, request: Request, permission) -> None: + if is_ee(): + has_permission = await check_action_access( + user_uid=str(request.state.user_id), + project_id=str(request.state.project_id), + permission=permission, + ) + if not has_permission: + raise FORBIDDEN_EXCEPTION + + @intercept_exceptions() + @handle_adapter_exceptions() + async def create_subscription( + self, + request: Request, + *, + body: TriggerSubscriptionCreateRequest, + ) -> TriggerSubscriptionResponse: + await self._check(request, Permission.EDIT_TRIGGERS if is_ee() else None) + + try: + subscription = await self.triggers_service.create_subscription( + project_id=UUID(request.state.project_id), + user_id=UUID(str(request.state.user_id)), + # + subscription=body.subscription, + ) + except ConnectionNotFoundError as e: + raise HTTPException(status_code=404, detail=e.message) from e + + return TriggerSubscriptionResponse( + count=1 if subscription else 0, + subscription=subscription, + ) + + @intercept_exceptions() + async def list_subscriptions( + self, + request: Request, + ) -> TriggerSubscriptionsResponse: + await self._check(request, Permission.VIEW_TRIGGERS if is_ee() else None) + + subscriptions = await self.triggers_service.query_subscriptions( + project_id=UUID(request.state.project_id), + ) + + return TriggerSubscriptionsResponse( + count=len(subscriptions), + subscriptions=subscriptions, + ) + + @intercept_exceptions() + async def query_subscriptions( + self, + request: Request, + *, + body: TriggerSubscriptionQueryRequest, + ) -> TriggerSubscriptionsResponse: + await self._check(request, Permission.VIEW_TRIGGERS if is_ee() else None) + + subscriptions = await self.triggers_service.query_subscriptions( + project_id=UUID(request.state.project_id), + # + subscription=body.subscription, + # + windowing=body.windowing, + ) + + return TriggerSubscriptionsResponse( + count=len(subscriptions), + subscriptions=subscriptions, + ) + + @intercept_exceptions() + async def fetch_subscription( + self, + request: Request, + *, + subscription_id: UUID, + ) -> TriggerSubscriptionResponse: + await self._check(request, Permission.VIEW_TRIGGERS if is_ee() else None) + + subscription = await self.triggers_service.fetch_subscription( + project_id=UUID(request.state.project_id), + # + subscription_id=subscription_id, + ) + if not subscription: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Trigger subscription not found", + ) + + return TriggerSubscriptionResponse( + count=1, + subscription=subscription, + ) + + @intercept_exceptions() + @handle_adapter_exceptions() + async def edit_subscription( + self, + request: Request, + *, + subscription_id: UUID, + body: TriggerSubscriptionEditRequest, + ) -> TriggerSubscriptionResponse: + await self._check(request, Permission.EDIT_TRIGGERS if is_ee() else None) + + if str(subscription_id) != str(body.subscription.id): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Path subscription_id does not match body id", + ) + + subscription = await self.triggers_service.edit_subscription( + project_id=UUID(request.state.project_id), + user_id=UUID(str(request.state.user_id)), + # + subscription=body.subscription, + ) + if not subscription: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Trigger subscription not found", + ) + + return TriggerSubscriptionResponse( + count=1, + subscription=subscription, + ) + + @intercept_exceptions() + @handle_adapter_exceptions() + async def delete_subscription( + self, + request: Request, + *, + subscription_id: UUID, + ) -> None: + await self._check(request, Permission.EDIT_TRIGGERS if is_ee() else None) + + deleted = await self.triggers_service.delete_subscription( + project_id=UUID(request.state.project_id), + # + subscription_id=subscription_id, + ) + if not deleted: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Trigger subscription not found", + ) + + @intercept_exceptions() + @handle_adapter_exceptions() + async def refresh_subscription( + self, + request: Request, + *, + subscription_id: UUID, + ) -> TriggerSubscriptionResponse: + await self._check(request, Permission.EDIT_TRIGGERS if is_ee() else None) + + try: + subscription = await self.triggers_service.refresh_subscription( + project_id=UUID(request.state.project_id), + user_id=UUID(str(request.state.user_id)), + # + subscription_id=subscription_id, + ) + except SubscriptionNotFoundError as e: + raise HTTPException(status_code=404, detail=e.message) from e + + return TriggerSubscriptionResponse( + count=1, + subscription=subscription, + ) + + @intercept_exceptions() + @handle_adapter_exceptions() + async def revoke_subscription( + self, + request: Request, + *, + subscription_id: UUID, + ) -> TriggerSubscriptionResponse: + await self._check(request, Permission.EDIT_TRIGGERS if is_ee() else None) + + try: + subscription = await self.triggers_service.revoke_subscription( + project_id=UUID(request.state.project_id), + user_id=UUID(str(request.state.user_id)), + # + subscription_id=subscription_id, + ) + except SubscriptionNotFoundError as e: + raise HTTPException(status_code=404, detail=e.message) from e + + return TriggerSubscriptionResponse( + count=1, + subscription=subscription, + ) + + # ----------------------------------------------------------------------- + # Trigger Deliveries + # ----------------------------------------------------------------------- + + @intercept_exceptions() + async def list_deliveries( + self, + request: Request, + ) -> TriggerDeliveriesResponse: + await self._check(request, Permission.VIEW_TRIGGERS if is_ee() else None) + + deliveries = await self.triggers_service.query_deliveries( + project_id=UUID(request.state.project_id), + ) + + return TriggerDeliveriesResponse( + count=len(deliveries), + deliveries=deliveries, + ) + + @intercept_exceptions() + async def query_deliveries( + self, + request: Request, + *, + body: TriggerDeliveryQueryRequest, + ) -> TriggerDeliveriesResponse: + await self._check(request, Permission.VIEW_TRIGGERS if is_ee() else None) + + deliveries = await self.triggers_service.query_deliveries( + project_id=UUID(request.state.project_id), + # + delivery=body.delivery, + # + windowing=body.windowing, + ) + + return TriggerDeliveriesResponse( + count=len(deliveries), + deliveries=deliveries, + ) + + @intercept_exceptions() + async def fetch_delivery( + self, + request: Request, + *, + delivery_id: UUID, + ) -> TriggerDeliveryResponse: + await self._check(request, Permission.VIEW_TRIGGERS if is_ee() else None) + + delivery = await self.triggers_service.fetch_delivery( + project_id=UUID(request.state.project_id), + # + delivery_id=delivery_id, + ) + if not delivery: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Trigger delivery not found", + ) + + return TriggerDeliveryResponse( + count=1, + delivery=delivery, + ) diff --git a/api/oss/src/core/triggers/dtos.py b/api/oss/src/core/triggers/dtos.py index 656a7ce56b..d9d2919d69 100644 --- a/api/oss/src/core/triggers/dtos.py +++ b/api/oss/src/core/triggers/dtos.py @@ -1,8 +1,19 @@ from enum import Enum from typing import Any, Dict, List, Optional +from uuid import UUID from pydantic import BaseModel +from oss.src.core.shared.dtos import ( + Header, + Identifier, + Lifecycle, + Metadata, + Reference, + Selector, + Status, +) + # --------------------------------------------------------------------------- # Trigger Enums @@ -47,3 +58,126 @@ class TriggerCatalogProvider(BaseModel): # name: str description: Optional[str] = None + + +# --------------------------------------------------------------------------- +# Context allowlists (mapping; see mapping.md §3) +# +# The inbound analogue of webhooks' EVENT_CONTEXT_FIELDS / SUBSCRIPTION_CONTEXT_FIELDS. +# A subscription's inputs_fields template may only reference these context keys; +# ca_*/secrets/connection internals are never exposed. +# --------------------------------------------------------------------------- + +TRIGGER_EVENT_FIELDS = { + "data", + "type", + "timestamp", + "metadata", +} + +SUBSCRIPTION_CONTEXT_FIELDS = { + "id", + "name", + "tags", + "meta", + "created_at", + "updated_at", +} + + +# --------------------------------------------------------------------------- +# Trigger Subscriptions +# +# A standing watch on one provider event. Mirrors a webhook subscription +# (subscribe-to-events lifecycle, CRUD) + FK to the shared gateway_connections +# row + a bound workflow reference. The provider-side trigger instance id +# (``ti_*``) lives on the row alongside its ``trigger_config``. +# --------------------------------------------------------------------------- + + +class TriggerSubscriptionData(BaseModel): + event_key: str + # + ti_id: Optional[str] = None + trigger_config: Optional[Dict[str, Any]] = None + # + # MAPPING — inputs-only template resolved into WorkflowServiceRequest.data.inputs. + inputs_fields: Optional[Dict[str, Any]] = None + # + # DESTINATION — the bound workflow, by reference (the /retrieve shape). + references: Optional[Dict[str, Reference]] = None + selector: Optional[Selector] = None + + +class TriggerSubscription(Identifier, Lifecycle, Header, Metadata): + connection_id: UUID + # + data: TriggerSubscriptionData + # + enabled: bool = True + valid: bool = True + + +class TriggerSubscriptionCreate(Header, Metadata): + connection_id: UUID + # + data: TriggerSubscriptionData + + +class TriggerSubscriptionEdit(Identifier, Header, Metadata): + connection_id: UUID + # + data: TriggerSubscriptionData + # + enabled: bool = True + valid: bool = True + + +class TriggerSubscriptionQuery(BaseModel): + name: Optional[str] = None + connection_id: Optional[UUID] = None + event_key: Optional[str] = None + + +# --------------------------------------------------------------------------- +# Trigger Deliveries +# +# One audit row per inbound event dispatched to its workflow — the inbound dual +# of webhook_deliveries. ``event_id`` is the I4 dedup key (provider metadata.id), +# unique per subscription. +# --------------------------------------------------------------------------- + + +class TriggerDeliveryData(BaseModel): + event_key: Optional[str] = None + # + references: Optional[Dict[str, Reference]] = None + inputs: Optional[Dict[str, Any]] = None + # + result: Optional[Dict[str, Any]] = None + error: Optional[str] = None + + +class TriggerDelivery(Identifier, Lifecycle): + status: Status + + data: Optional[TriggerDeliveryData] = None + + subscription_id: UUID + event_id: str + + +class TriggerDeliveryCreate(Identifier): + status: Status + + data: Optional[TriggerDeliveryData] = None + + subscription_id: UUID + event_id: str + + +class TriggerDeliveryQuery(BaseModel): + status: Optional[Status] = None + + subscription_id: Optional[UUID] = None + event_id: Optional[str] = None diff --git a/api/oss/src/core/triggers/exceptions.py b/api/oss/src/core/triggers/exceptions.py index 473b4094a4..092144ceff 100644 --- a/api/oss/src/core/triggers/exceptions.py +++ b/api/oss/src/core/triggers/exceptions.py @@ -17,6 +17,22 @@ def __init__(self, provider_key: str): super().__init__(f"Provider not found: {provider_key}") +class SubscriptionNotFoundError(TriggersError): + """Raised when a subscription_id does not exist in the project.""" + + def __init__(self, *, subscription_id: str): + self.subscription_id = subscription_id + super().__init__(f"Trigger subscription not found: {subscription_id}") + + +class ConnectionNotFoundError(TriggersError): + """Raised when a subscription references a connection that does not exist.""" + + def __init__(self, *, connection_id: str): + self.connection_id = connection_id + super().__init__(f"Connection not found: {connection_id}") + + class AdapterError(TriggersError): """Raised when an adapter operation fails.""" diff --git a/api/oss/src/core/triggers/interfaces.py b/api/oss/src/core/triggers/interfaces.py index 2b07ca835f..a7280c90ea 100644 --- a/api/oss/src/core/triggers/interfaces.py +++ b/api/oss/src/core/triggers/interfaces.py @@ -2,10 +2,18 @@ from typing import Any, Dict, List, Optional, Tuple from uuid import UUID +from oss.src.core.shared.dtos import Windowing from oss.src.core.triggers.dtos import ( TriggerCatalogEvent, TriggerCatalogEventDetails, TriggerCatalogProvider, + TriggerDelivery, + TriggerDeliveryCreate, + TriggerDeliveryQuery, + TriggerSubscription, + TriggerSubscriptionCreate, + TriggerSubscriptionEdit, + TriggerSubscriptionQuery, ) @@ -73,3 +81,114 @@ async def delete_subscription( ) -> None: """Permanently delete the provider-side trigger instance.""" ... + + +class TriggersDAOInterface(ABC): + """Persistence contract for the triggers domain (subscriptions + deliveries).""" + + # --- subscriptions ------------------------------------------------------ # + + @abstractmethod + async def create_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionCreate, + # + ti_id: str, + ) -> TriggerSubscription: ... + + @abstractmethod + async def fetch_subscription( + self, + *, + project_id: UUID, + # + subscription_id: UUID, + ) -> Optional[TriggerSubscription]: ... + + @abstractmethod + async def edit_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionEdit, + ) -> Optional[TriggerSubscription]: ... + + @abstractmethod + async def delete_subscription( + self, + *, + project_id: UUID, + # + subscription_id: UUID, + ) -> bool: ... + + @abstractmethod + async def query_subscriptions( + self, + *, + project_id: UUID, + # + subscription: Optional[TriggerSubscriptionQuery] = None, + # + windowing: Optional[Windowing] = None, + ) -> List[TriggerSubscription]: ... + + @abstractmethod + async def get_subscription_by_trigger_id( + self, + *, + trigger_id: str, + ) -> Optional[TriggerSubscription]: + """FROZEN (WP4): resolve an inbound event's ``ti_*`` to its local row.""" + ... + + # --- deliveries --------------------------------------------------------- # + + @abstractmethod + async def write_delivery( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + # + delivery: TriggerDeliveryCreate, + ) -> TriggerDelivery: + """FROZEN (WP4): upsert a delivery row (idempotent on event_id).""" + ... + + @abstractmethod + async def fetch_delivery( + self, + *, + project_id: UUID, + # + delivery_id: UUID, + ) -> Optional[TriggerDelivery]: ... + + @abstractmethod + async def query_deliveries( + self, + *, + project_id: UUID, + # + delivery: Optional[TriggerDeliveryQuery] = None, + # + windowing: Optional[Windowing] = None, + ) -> List[TriggerDelivery]: ... + + @abstractmethod + async def dedup_seen( + self, + *, + project_id: UUID, + subscription_id: UUID, + event_id: str, + ) -> bool: + """FROZEN (WP4): True if a delivery for this event_id already exists (I4).""" + ... diff --git a/api/oss/src/core/triggers/service.py b/api/oss/src/core/triggers/service.py index bc08263c2f..ea5c9f0c77 100644 --- a/api/oss/src/core/triggers/service.py +++ b/api/oss/src/core/triggers/service.py @@ -1,13 +1,28 @@ from typing import List, Optional, Tuple +from uuid import UUID from oss.src.utils.logging import get_module_logger +from oss.src.core.connections.service import ConnectionsService from oss.src.core.triggers.dtos import ( TriggerCatalogEvent, TriggerCatalogEventDetails, TriggerCatalogProvider, + TriggerDelivery, + TriggerDeliveryCreate, + TriggerDeliveryQuery, + TriggerSubscription, + TriggerSubscriptionCreate, + TriggerSubscriptionEdit, + TriggerSubscriptionQuery, ) +from oss.src.core.triggers.exceptions import ( + ConnectionNotFoundError, + SubscriptionNotFoundError, +) +from oss.src.core.triggers.interfaces import TriggersDAOInterface from oss.src.core.triggers.registry import TriggersGatewayRegistry +from oss.src.core.shared.dtos import Windowing log = get_module_logger(__name__) @@ -16,16 +31,22 @@ class TriggersService: """Triggers domain orchestration. - WP1 scope is the read-only events catalog. Subscriptions/deliveries CRUD and - ingress/dispatch land in later WPs (WP3/WP4) and will extend this service. + Covers the read-only events catalog (WP1) and subscription/delivery + CRUD (WP3). Subscriptions bind a provider event to a workflow on top of a + shared gateway connection; the provider-side trigger instance (``ti_*``) is + minted/managed through the adapter, never the catalog routes. """ def __init__( self, *, adapter_registry: TriggersGatewayRegistry, + triggers_dao: Optional[TriggersDAOInterface] = None, + connections_service: Optional[ConnectionsService] = None, ): self.adapter_registry = adapter_registry + self.dao = triggers_dao + self.connections_service = connections_service # ----------------------------------------------------------------------- # Catalog browse @@ -84,3 +105,282 @@ async def get_event( integration_key=integration_key, event_key=event_key, ) + + # ----------------------------------------------------------------------- + # Subscriptions + # ----------------------------------------------------------------------- + + async def _require_connection( + self, + *, + project_id: UUID, + connection_id: UUID, + ): + connection = await self.connections_service.get_connection( + project_id=project_id, + connection_id=connection_id, + ) + if not connection: + raise ConnectionNotFoundError(connection_id=str(connection_id)) + return connection + + async def create_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionCreate, + ) -> TriggerSubscription: + """Mint the provider-side ``ti_*`` on a shared connection, then persist.""" + connection = await self._require_connection( + project_id=project_id, + connection_id=subscription.connection_id, + ) + + adapter = self.adapter_registry.get(connection.provider_key.value) + + ti_id = await adapter.create_subscription( + project_id=project_id, + event_key=subscription.data.event_key, + connected_account_id=connection.provider_connection_id, + trigger_config=subscription.data.trigger_config or {}, + ) + + return await self.dao.create_subscription( + project_id=project_id, + user_id=user_id, + # + subscription=subscription, + # + ti_id=ti_id, + ) + + async def fetch_subscription( + self, + *, + project_id: UUID, + # + subscription_id: UUID, + ) -> Optional[TriggerSubscription]: + return await self.dao.fetch_subscription( + project_id=project_id, + subscription_id=subscription_id, + ) + + async def query_subscriptions( + self, + *, + project_id: UUID, + # + subscription: Optional[TriggerSubscriptionQuery] = None, + # + windowing: Optional[Windowing] = None, + ) -> List[TriggerSubscription]: + return await self.dao.query_subscriptions( + project_id=project_id, + subscription=subscription, + windowing=windowing, + ) + + async def edit_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionEdit, + ) -> Optional[TriggerSubscription]: + """Full-PUT edit. Reflects the enabled flag onto the provider ``ti_*``.""" + existing = await self.dao.fetch_subscription( + project_id=project_id, + subscription_id=subscription.id, + ) + if existing is None: + return None + + ti_id = existing.data.ti_id + if ti_id is not None and subscription.enabled != existing.enabled: + connection = await self._require_connection( + project_id=project_id, + connection_id=existing.connection_id, + ) + adapter = self.adapter_registry.get(connection.provider_key.value) + await adapter.set_subscription_status( + trigger_id=ti_id, + enabled=subscription.enabled, + ) + + return await self.dao.edit_subscription( + project_id=project_id, + user_id=user_id, + subscription=subscription, + ) + + async def delete_subscription( + self, + *, + project_id: UUID, + # + subscription_id: UUID, + ) -> bool: + """Delete the local row and the provider ``ti_*``. + + Deleting a subscription must NOT revoke the shared connection (C7): the + adapter call below targets only the trigger instance, never the ``ca_*``. + """ + existing = await self.dao.fetch_subscription( + project_id=project_id, + subscription_id=subscription_id, + ) + if existing is None: + return False + + ti_id = existing.data.ti_id + if ti_id is not None: + connection = await self.connections_service.get_connection( + project_id=project_id, + connection_id=existing.connection_id, + ) + if connection is not None: + adapter = self.adapter_registry.get(connection.provider_key.value) + try: + await adapter.delete_subscription(trigger_id=ti_id) + except Exception: + log.warning( + "Failed to delete provider trigger %s; proceeding with local delete", + ti_id, + ) + + return await self.dao.delete_subscription( + project_id=project_id, + subscription_id=subscription_id, + ) + + async def refresh_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription_id: UUID, + ) -> TriggerSubscription: + """Re-enable the provider ``ti_*`` and mark the row enabled+valid.""" + return await self._set_enabled( + project_id=project_id, + user_id=user_id, + subscription_id=subscription_id, + enabled=True, + ) + + async def revoke_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription_id: UUID, + ) -> TriggerSubscription: + """Disable the provider ``ti_*`` and mark the row disabled. + + Local + provider trigger-instance only; the shared connection is never + touched (C7). + """ + return await self._set_enabled( + project_id=project_id, + user_id=user_id, + subscription_id=subscription_id, + enabled=False, + ) + + async def _set_enabled( + self, + *, + project_id: UUID, + user_id: UUID, + subscription_id: UUID, + enabled: bool, + ) -> TriggerSubscription: + existing = await self.dao.fetch_subscription( + project_id=project_id, + subscription_id=subscription_id, + ) + if existing is None: + raise SubscriptionNotFoundError(subscription_id=str(subscription_id)) + + ti_id = existing.data.ti_id + if ti_id is not None: + connection = await self._require_connection( + project_id=project_id, + connection_id=existing.connection_id, + ) + adapter = self.adapter_registry.get(connection.provider_key.value) + await adapter.set_subscription_status( + trigger_id=ti_id, + enabled=enabled, + ) + + edit = TriggerSubscriptionEdit( + id=existing.id, + connection_id=existing.connection_id, + name=existing.name, + description=existing.description, + tags=existing.tags, + meta=existing.meta, + data=existing.data, + enabled=enabled, + valid=existing.valid, + ) + + updated = await self.dao.edit_subscription( + project_id=project_id, + user_id=user_id, + subscription=edit, + ) + + return updated or existing + + # ----------------------------------------------------------------------- + # Deliveries + # ----------------------------------------------------------------------- + + async def fetch_delivery( + self, + *, + project_id: UUID, + # + delivery_id: UUID, + ) -> Optional[TriggerDelivery]: + return await self.dao.fetch_delivery( + project_id=project_id, + delivery_id=delivery_id, + ) + + async def query_deliveries( + self, + *, + project_id: UUID, + # + delivery: Optional[TriggerDeliveryQuery] = None, + # + windowing: Optional[Windowing] = None, + ) -> List[TriggerDelivery]: + return await self.dao.query_deliveries( + project_id=project_id, + delivery=delivery, + windowing=windowing, + ) + + async def write_delivery( + self, + *, + project_id: UUID, + user_id: Optional[UUID] = None, + # + delivery: TriggerDeliveryCreate, + ) -> TriggerDelivery: + return await self.dao.write_delivery( + project_id=project_id, + user_id=user_id, + delivery=delivery, + ) diff --git a/api/oss/src/dbs/postgres/triggers/dao.py b/api/oss/src/dbs/postgres/triggers/dao.py new file mode 100644 index 0000000000..b3a1a51e3c --- /dev/null +++ b/api/oss/src/dbs/postgres/triggers/dao.py @@ -0,0 +1,378 @@ +from datetime import datetime, timezone +from typing import List, Optional +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert + +from oss.src.core.shared.dtos import Windowing +from oss.src.core.triggers.dtos import ( + TriggerDelivery, + TriggerDeliveryCreate, + TriggerDeliveryQuery, + TriggerSubscription, + TriggerSubscriptionCreate, + TriggerSubscriptionEdit, + TriggerSubscriptionQuery, +) +from oss.src.core.triggers.interfaces import TriggersDAOInterface + +from oss.src.dbs.postgres.shared.engine import ( + TransactionsEngine, + get_transactions_engine, +) +from oss.src.dbs.postgres.shared.utils import apply_windowing +from oss.src.dbs.postgres.triggers.dbes import ( + TriggerDeliveryDBE, + TriggerSubscriptionDBE, +) +from oss.src.dbs.postgres.triggers.mappings import ( + map_delivery_dbe_to_dto, + map_delivery_dto_to_dbe_create, + map_subscription_dbe_to_dto, + map_subscription_dto_to_dbe_create, + map_subscription_dto_to_dbe_edit, +) + + +class TriggersDAO(TriggersDAOInterface): + def __init__(self, engine: TransactionsEngine = None): + if engine is None: + engine = get_transactions_engine() + self.engine = engine + + # --- SUBSCRIPTIONS ------------------------------------------------------ # + + async def create_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionCreate, + # + ti_id: str, + ) -> TriggerSubscription: + subscription_dbe = map_subscription_dto_to_dbe_create( + project_id=project_id, + user_id=user_id, + # + subscription=subscription, + # + ti_id=ti_id, + ) + + async with self.engine.session() as session: + session.add(subscription_dbe) + + await session.commit() + + await session.refresh(subscription_dbe) + + return map_subscription_dbe_to_dto( + subscription_dbe=subscription_dbe, + ) + + async def fetch_subscription( + self, + *, + project_id: UUID, + # + subscription_id: UUID, + ) -> Optional[TriggerSubscription]: + async with self.engine.session() as session: + stmt = select(TriggerSubscriptionDBE).where( + TriggerSubscriptionDBE.project_id == project_id, + TriggerSubscriptionDBE.id == subscription_id, + ) + + result = await session.execute(stmt) + + subscription_dbe = result.scalar_one_or_none() + + if not subscription_dbe: + return None + + return map_subscription_dbe_to_dto( + subscription_dbe=subscription_dbe, + ) + + async def edit_subscription( + self, + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionEdit, + ) -> Optional[TriggerSubscription]: + async with self.engine.session() as session: + stmt = select(TriggerSubscriptionDBE).where( + TriggerSubscriptionDBE.id == subscription.id, + TriggerSubscriptionDBE.project_id == project_id, + ) + + result = await session.execute(stmt) + + subscription_dbe = result.scalar_one_or_none() + + if not subscription_dbe: + return None + + map_subscription_dto_to_dbe_edit( + subscription_dbe=subscription_dbe, + # + user_id=user_id, + # + subscription=subscription, + ) + + await session.commit() + + await session.refresh(subscription_dbe) + + return map_subscription_dbe_to_dto( + subscription_dbe=subscription_dbe, + ) + + async def delete_subscription( + self, + *, + project_id: UUID, + # + subscription_id: UUID, + ) -> bool: + async with self.engine.session() as session: + stmt = select(TriggerSubscriptionDBE).where( + TriggerSubscriptionDBE.project_id == project_id, + TriggerSubscriptionDBE.id == subscription_id, + ) + + result = await session.execute(stmt) + + subscription_dbe = result.scalar_one_or_none() + + if not subscription_dbe: + return False + + await session.delete(subscription_dbe) + + await session.commit() + + return True + + async def query_subscriptions( + self, + *, + project_id: UUID, + # + subscription: Optional[TriggerSubscriptionQuery] = None, + # + windowing: Optional[Windowing] = None, + ) -> List[TriggerSubscription]: + async with self.engine.session() as session: + stmt = select(TriggerSubscriptionDBE).filter( + TriggerSubscriptionDBE.project_id == project_id, + ) + + if subscription: + if subscription.name is not None: + stmt = stmt.filter( + TriggerSubscriptionDBE.name.ilike(f"%{subscription.name}%"), + ) + + if subscription.connection_id is not None: + stmt = stmt.filter( + TriggerSubscriptionDBE.connection_id + == subscription.connection_id, + ) + + if subscription.event_key is not None: + stmt = stmt.filter( + TriggerSubscriptionDBE.data["event_key"].astext + == subscription.event_key, + ) + + if windowing: + stmt = apply_windowing( + stmt=stmt, + DBE=TriggerSubscriptionDBE, + attribute="id", + order="descending", + windowing=windowing, + ) + + result = await session.execute(stmt) + + return [ + map_subscription_dbe_to_dto(subscription_dbe=dbe) + for dbe in result.scalars().all() + ] + + async def get_subscription_by_trigger_id( + self, + *, + trigger_id: str, + ) -> Optional[TriggerSubscription]: + async with self.engine.session() as session: + stmt = ( + select(TriggerSubscriptionDBE) + .filter( + TriggerSubscriptionDBE.data["ti_id"].astext == trigger_id, + ) + .limit(1) + ) + + result = await session.execute(stmt) + + subscription_dbe = result.scalars().first() + + if not subscription_dbe: + return None + + return map_subscription_dbe_to_dto( + subscription_dbe=subscription_dbe, + ) + + # --- DELIVERIES --------------------------------------------------------- # + + async def write_delivery( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + # + delivery: TriggerDeliveryCreate, + ) -> TriggerDelivery: + delivery_dbe = map_delivery_dto_to_dbe_create( + project_id=project_id, + user_id=user_id, + # + delivery=delivery, + ) + + async with self.engine.session() as session: + values = { + c.name: getattr(delivery_dbe, c.name) + for c in TriggerDeliveryDBE.__table__.columns + if not ( + c.name in ("id", "created_at", "updated_at", "deleted_at") + and getattr(delivery_dbe, c.name) is None + ) + } + + stmt = insert(TriggerDeliveryDBE).values(**values) + stmt = stmt.on_conflict_do_update( + index_elements=["project_id", "subscription_id", "event_id"], + set_={ + "status": stmt.excluded.status, + "data": stmt.excluded.data, + "updated_at": datetime.now(timezone.utc), + "updated_by_id": stmt.excluded.created_by_id, + }, + ) + await session.execute(stmt) + await session.commit() + + refreshed_stmt = select(TriggerDeliveryDBE).where( + TriggerDeliveryDBE.project_id == project_id, + TriggerDeliveryDBE.subscription_id == delivery.subscription_id, + TriggerDeliveryDBE.event_id == delivery.event_id, + ) + delivery_dbe = (await session.execute(refreshed_stmt)).scalar_one() + + return map_delivery_dbe_to_dto( + delivery_dbe=delivery_dbe, + ) + + async def fetch_delivery( + self, + *, + project_id: UUID, + # + delivery_id: UUID, + ) -> Optional[TriggerDelivery]: + async with self.engine.session() as session: + stmt = select(TriggerDeliveryDBE).where( + TriggerDeliveryDBE.project_id == project_id, + TriggerDeliveryDBE.id == delivery_id, + ) + + result = await session.execute(stmt) + + delivery_dbe = result.scalar_one_or_none() + + if not delivery_dbe: + return None + + return map_delivery_dbe_to_dto( + delivery_dbe=delivery_dbe, + ) + + async def query_deliveries( + self, + *, + project_id: UUID, + # + delivery: Optional[TriggerDeliveryQuery] = None, + # + windowing: Optional[Windowing] = None, + ) -> List[TriggerDelivery]: + async with self.engine.session() as session: + stmt = select(TriggerDeliveryDBE).filter( + TriggerDeliveryDBE.project_id == project_id, + ) + + if delivery: + if delivery.status is not None and delivery.status.code is not None: + stmt = stmt.filter( + TriggerDeliveryDBE.status["code"].astext + == str(delivery.status.code), + ) + + if delivery.subscription_id is not None: + stmt = stmt.filter( + TriggerDeliveryDBE.subscription_id == delivery.subscription_id, + ) + + if delivery.event_id is not None: + stmt = stmt.filter( + TriggerDeliveryDBE.event_id == delivery.event_id, + ) + + if windowing: + stmt = apply_windowing( + stmt=stmt, + DBE=TriggerDeliveryDBE, + attribute="created_at", + order="descending", + windowing=windowing, + ) + + result = await session.execute(stmt) + + return [ + map_delivery_dbe_to_dto(delivery_dbe=dbe) + for dbe in result.scalars().all() + ] + + async def dedup_seen( + self, + *, + project_id: UUID, + subscription_id: UUID, + event_id: str, + ) -> bool: + async with self.engine.session() as session: + stmt = ( + select(TriggerDeliveryDBE.id) + .where( + TriggerDeliveryDBE.project_id == project_id, + TriggerDeliveryDBE.subscription_id == subscription_id, + TriggerDeliveryDBE.event_id == event_id, + ) + .limit(1) + ) + + result = await session.execute(stmt) + + return result.scalar_one_or_none() is not None diff --git a/api/oss/src/dbs/postgres/triggers/dbas.py b/api/oss/src/dbs/postgres/triggers/dbas.py new file mode 100644 index 0000000000..2f2e7b199b --- /dev/null +++ b/api/oss/src/dbs/postgres/triggers/dbas.py @@ -0,0 +1,53 @@ +from sqlalchemy import Column, String +from sqlalchemy.dialects.postgresql import UUID + +from oss.src.dbs.postgres.shared.dbas import ( + DataDBA, + FlagsDBA, + HeaderDBA, + IdentifierDBA, + LifecycleDBA, + MetaDBA, + ProjectScopeDBA, + StatusDBA, + TagsDBA, +) + + +class TriggerSubscriptionDBA( + ProjectScopeDBA, + LifecycleDBA, + IdentifierDBA, + HeaderDBA, + DataDBA, + FlagsDBA, + TagsDBA, + MetaDBA, +): + __abstract__ = True + + connection_id = Column( + UUID(as_uuid=True), + nullable=False, + ) + + +class TriggerDeliveryDBA( + ProjectScopeDBA, + LifecycleDBA, + IdentifierDBA, + StatusDBA, + DataDBA, +): + __abstract__ = True + + subscription_id = Column( + UUID(as_uuid=True), + nullable=False, + ) + + # I4: provider metadata.id — an arbitrary provider string, unique per subscription. + event_id = Column( + String, + nullable=False, + ) diff --git a/api/oss/src/dbs/postgres/triggers/dbes.py b/api/oss/src/dbs/postgres/triggers/dbes.py new file mode 100644 index 0000000000..9caf012350 --- /dev/null +++ b/api/oss/src/dbs/postgres/triggers/dbes.py @@ -0,0 +1,75 @@ +from sqlalchemy import ForeignKeyConstraint, Index, PrimaryKeyConstraint + +from oss.src.dbs.postgres.shared.base import Base +from oss.src.dbs.postgres.triggers.dbas import ( + TriggerDeliveryDBA, + TriggerSubscriptionDBA, +) + + +class TriggerSubscriptionDBE(Base, TriggerSubscriptionDBA): + __tablename__ = "trigger_subscriptions" + + __table_args__ = ( + ForeignKeyConstraint( + ["project_id"], + ["projects.id"], + ondelete="CASCADE", + ), + ForeignKeyConstraint( + ["project_id", "connection_id"], + ["gateway_connections.project_id", "gateway_connections.id"], + ondelete="CASCADE", + ), + PrimaryKeyConstraint("project_id", "id"), + Index( + "ix_trigger_subscriptions_project_id_created_at", + "project_id", + "created_at", + ), + Index( + "ix_trigger_subscriptions_project_id_deleted_at", + "project_id", + "deleted_at", + ), + Index( + "ix_trigger_subscriptions_connection_id", + "project_id", + "connection_id", + ), + ) + + +class TriggerDeliveryDBE(Base, TriggerDeliveryDBA): + __tablename__ = "trigger_deliveries" + + __table_args__ = ( + ForeignKeyConstraint( + ["project_id"], + ["projects.id"], + ondelete="CASCADE", + ), + ForeignKeyConstraint( + ["project_id", "subscription_id"], + ["trigger_subscriptions.project_id", "trigger_subscriptions.id"], + ondelete="CASCADE", + ), + PrimaryKeyConstraint("project_id", "id"), + Index( + "ix_trigger_deliveries_project_id_created_at", + "project_id", + "created_at", + ), + Index( + "ix_trigger_deliveries_subscription_id_created_at", + "subscription_id", + "created_at", + ), + Index( + "ix_trigger_deliveries_subscription_id_event_id", + "project_id", + "subscription_id", + "event_id", + unique=True, + ), + ) diff --git a/api/oss/src/dbs/postgres/triggers/mappings.py b/api/oss/src/dbs/postgres/triggers/mappings.py new file mode 100644 index 0000000000..97b1eaed92 --- /dev/null +++ b/api/oss/src/dbs/postgres/triggers/mappings.py @@ -0,0 +1,179 @@ +from uuid import UUID + +from oss.src.core.shared.dtos import Status +from oss.src.core.triggers.dtos import ( + TriggerDelivery, + TriggerDeliveryCreate, + TriggerDeliveryData, + TriggerSubscription, + TriggerSubscriptionCreate, + TriggerSubscriptionData, + TriggerSubscriptionEdit, +) + +from oss.src.dbs.postgres.triggers.dbes import ( + TriggerDeliveryDBE, + TriggerSubscriptionDBE, +) + + +# --- Subscription ----------------------------------------------------------- # + +_SUBSCRIPTION_FLAGS = ("enabled", "valid") + + +def _flags_to_dbe(*, enabled: bool, valid: bool) -> dict: + return {"enabled": enabled, "valid": valid} + + +def map_subscription_dto_to_dbe_create( + *, + project_id: UUID, + user_id: UUID, + # + subscription: TriggerSubscriptionCreate, + # + ti_id: str, +) -> TriggerSubscriptionDBE: + data = subscription.data.model_copy(update={"ti_id": ti_id}) + + return TriggerSubscriptionDBE( + project_id=project_id, + # + created_by_id=user_id, + # + connection_id=subscription.connection_id, + # + name=subscription.name, + description=subscription.description, + tags=subscription.tags, + meta=subscription.meta, + # + flags=_flags_to_dbe(enabled=True, valid=True), + # + data=data.model_dump(mode="json", exclude_none=True), + ) + + +def map_subscription_dbe_to_dto( + *, + subscription_dbe: TriggerSubscriptionDBE, +) -> TriggerSubscription: + flags = subscription_dbe.flags or {} + + return TriggerSubscription( + id=subscription_dbe.id, + # + created_at=subscription_dbe.created_at, + updated_at=subscription_dbe.updated_at, + deleted_at=subscription_dbe.deleted_at, + created_by_id=subscription_dbe.created_by_id, + updated_by_id=subscription_dbe.updated_by_id, + deleted_by_id=subscription_dbe.deleted_by_id, + # + connection_id=subscription_dbe.connection_id, + # + name=subscription_dbe.name, + description=subscription_dbe.description, + # + tags=subscription_dbe.tags, + meta=subscription_dbe.meta, + # + data=TriggerSubscriptionData.model_validate(subscription_dbe.data), + # + enabled=bool(flags.get("enabled", True)), + valid=bool(flags.get("valid", True)), + ) + + +def map_subscription_dto_to_dbe_edit( + *, + subscription_dbe: TriggerSubscriptionDBE, + # + user_id: UUID, + # + subscription: TriggerSubscriptionEdit, +) -> None: + subscription_dbe.updated_by_id = user_id + + subscription_dbe.connection_id = subscription.connection_id + + subscription_dbe.name = subscription.name + subscription_dbe.description = subscription.description + + subscription_dbe.tags = subscription.tags + subscription_dbe.meta = subscription.meta + + # Preserve the provider ti_id even if the client omitted it on the full-PUT. + existing_ti_id = (subscription_dbe.data or {}).get("ti_id") + data = subscription.data + if data.ti_id is None and existing_ti_id is not None: + data = data.model_copy(update={"ti_id": existing_ti_id}) + + subscription_dbe.data = data.model_dump(mode="json", exclude_none=True) + + subscription_dbe.flags = _flags_to_dbe( + enabled=subscription.enabled, + valid=subscription.valid, + ) + + +# --- Delivery --------------------------------------------------------------- # + + +def map_delivery_dto_to_dbe_create( + *, + project_id: UUID, + user_id: UUID | None, + # + delivery: TriggerDeliveryCreate, +) -> TriggerDeliveryDBE: + dbe_kwargs = dict( + project_id=project_id, + # + created_by_id=user_id, + # + status=delivery.status.model_dump(mode="json", exclude_none=True) + if delivery.status + else None, + # + data=delivery.data.model_dump(mode="json", exclude_none=True) + if delivery.data + else None, + # + subscription_id=delivery.subscription_id, + # + event_id=delivery.event_id, + ) + if delivery.id is not None: + dbe_kwargs["id"] = delivery.id + + return TriggerDeliveryDBE(**dbe_kwargs) + + +def map_delivery_dbe_to_dto( + *, + delivery_dbe: TriggerDeliveryDBE, +) -> TriggerDelivery: + return TriggerDelivery( + id=delivery_dbe.id, + # + created_at=delivery_dbe.created_at, + updated_at=delivery_dbe.updated_at, + deleted_at=delivery_dbe.deleted_at, + created_by_id=delivery_dbe.created_by_id, + updated_by_id=delivery_dbe.updated_by_id, + deleted_by_id=delivery_dbe.deleted_by_id, + # + status=Status.model_validate(delivery_dbe.status) + if delivery_dbe.status + else Status(), + # + data=TriggerDeliveryData.model_validate(delivery_dbe.data) + if delivery_dbe.data + else None, + # + subscription_id=delivery_dbe.subscription_id, + # + event_id=delivery_dbe.event_id, + ) diff --git a/api/oss/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py b/api/oss/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py new file mode 100644 index 0000000000..cd519cc3f2 --- /dev/null +++ b/api/oss/tests/pytest/acceptance/triggers/test_triggers_subscriptions.py @@ -0,0 +1,155 @@ +"""Acceptance tests for /triggers/subscriptions/* and /triggers/deliveries/*. + +The read/query surfaces are DB-only — a fresh project returns well-shaped empty +lists and 404s with no Composio credentials, which also proves the +trigger_subscriptions / trigger_deliveries tables landed (migration ran). + +Creating a subscription mints a provider-side trigger instance (ti_*) on a +shared gateway connection, so the full create -> list -> disable -> delete +roundtrip (and the C7 invariant — deleting a subscription leaves the connection +intact) is gated on COMPOSIO_API_KEY being present in the runner's environment. + +Requires a running API. +""" + +import os +from uuid import uuid4 + +import pytest + + +_COMPOSIO_ENABLED = bool(os.getenv("COMPOSIO_API_KEY")) +_requires_composio = pytest.mark.skipif( + not _COMPOSIO_ENABLED, + reason="needs live Composio credentials (COMPOSIO_API_KEY)", +) + + +# --------------------------------------------------------------------------- +# DB-only: reads, queries, 404s (no Composio needed) +# --------------------------------------------------------------------------- + + +class TestTriggerSubscriptionsReads: + def test_list_subscriptions_returns_200_empty(self, authed_api): + body = authed_api("GET", "/triggers/subscriptions/").json() + assert "count" in body + assert "subscriptions" in body + assert isinstance(body["subscriptions"], list) + assert body["count"] == len(body["subscriptions"]) + + def test_query_subscriptions_returns_200(self, authed_api): + response = authed_api("POST", "/triggers/subscriptions/query", json={}) + assert response.status_code == 200 + body = response.json() + assert body["count"] == len(body["subscriptions"]) + + def test_fetch_unknown_subscription_returns_404(self, authed_api): + response = authed_api("GET", f"/triggers/subscriptions/{uuid4()}") + assert response.status_code == 404 + + def test_delete_unknown_subscription_returns_404(self, authed_api): + response = authed_api("DELETE", f"/triggers/subscriptions/{uuid4()}") + assert response.status_code == 404 + + def test_refresh_unknown_subscription_returns_404(self, authed_api): + response = authed_api("POST", f"/triggers/subscriptions/{uuid4()}/refresh") + assert response.status_code == 404 + + def test_revoke_unknown_subscription_returns_404(self, authed_api): + response = authed_api("POST", f"/triggers/subscriptions/{uuid4()}/revoke") + assert response.status_code == 404 + + +class TestTriggerDeliveriesReads: + def test_list_deliveries_returns_200_empty(self, authed_api): + body = authed_api("GET", "/triggers/deliveries").json() + assert "count" in body + assert "deliveries" in body + assert isinstance(body["deliveries"], list) + assert body["count"] == len(body["deliveries"]) + + def test_query_deliveries_returns_200(self, authed_api): + response = authed_api("POST", "/triggers/deliveries/query", json={}) + assert response.status_code == 200 + body = response.json() + assert body["count"] == len(body["deliveries"]) + + def test_fetch_unknown_delivery_returns_404(self, authed_api): + response = authed_api("GET", f"/triggers/deliveries/{uuid4()}") + assert response.status_code == 404 + + +# --------------------------------------------------------------------------- +# Full lifecycle (needs Composio) — create on a shared connection bound to a +# workflow, list/disable/delete it, and prove the connection survives (C7). +# --------------------------------------------------------------------------- + + +@_requires_composio +class TestTriggerSubscriptionsLifecycle: + def _create_connection(self, authed_api): + slug = f"acc-{uuid4().hex[:8]}" + create = authed_api( + "POST", + "/tools/connections/", + json={ + "connection": { + "slug": slug, + "provider_key": "composio", + "integration_key": "github", + "data": {"auth_scheme": "oauth"}, + } + }, + ) + assert create.status_code == 200, create.text + return create.json()["connection"]["id"] + + def test_create_list_disable_delete_keeps_connection(self, authed_api): + connection_id = self._create_connection(authed_api) + + # CREATE — binds the event to a workflow reference on the shared connection + create = authed_api( + "POST", + "/triggers/subscriptions/", + json={ + "subscription": { + "name": f"sub-{uuid4().hex[:8]}", + "connection_id": connection_id, + "data": { + "event_key": "GITHUB_STAR_ADDED_EVENT", + "trigger_config": {}, + "inputs_fields": {"repo": "$.event.data.repository"}, + "references": {"workflow": {"slug": "triage"}}, + }, + } + }, + ) + assert create.status_code == 200, create.text + sub = create.json()["subscription"] + subscription_id = sub["id"] + assert sub["connection_id"] == connection_id + assert sub["data"]["ti_id"] is not None + assert sub["enabled"] is True + + # LIST + listing = authed_api("GET", "/triggers/subscriptions/").json() + assert any(s["id"] == subscription_id for s in listing["subscriptions"]) + + # DISABLE (revoke the subscription, not the connection) + revoke = authed_api("POST", f"/triggers/subscriptions/{subscription_id}/revoke") + assert revoke.status_code == 200, revoke.text + assert revoke.json()["subscription"]["enabled"] is False + + # DELETE + delete = authed_api("DELETE", f"/triggers/subscriptions/{subscription_id}") + assert delete.status_code == 204 + + fetch = authed_api("GET", f"/triggers/subscriptions/{subscription_id}") + assert fetch.status_code == 404 + + # C7: deleting the subscription must NOT delete/revoke the connection. + conn = authed_api("GET", f"/tools/connections/{connection_id}") + assert conn.status_code == 200, conn.text + + authed_api("DELETE", f"/tools/connections/{connection_id}") From fcd3cda2a096cecb55d6dd72ae33abeeadb7193e Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Thu, 18 Jun 2026 19:24:18 +0200 Subject: [PATCH 4/5] feat(api): add Composio trigger ingress, async dispatch, and worker Inbound dual of webhooks: a global ingress endpoint POST /triggers/composio/events fast-ACKs and enqueues onto the queues:triggers Redis Stream; a dedicated worker_triggers process consumes it and the TriggersDispatcher invokes the bound workflow. - Ingress endpoint with HMAC-SHA256 signature verification against COMPOSIO_WEBHOOK_SECRET; whitelisted in auth middleware as public. - Async pipeline mirroring webhooks: Redis Streams broker + taskiq worker with retry_on_error and TRIGGER_MAX_RETRIES=5; dedup by event_id for idempotency. - Dispatcher attributes the run to the subscription creator (created_by_id) or null; binds the workflow key-agnostically from the references dict via the /retrieve selector shape. Co-Authored-By: Claude Opus 4.8 (1M context) --- api/entrypoints/routers.py | 28 ++ api/entrypoints/worker_triggers.py | 142 ++++++++++ api/oss/src/apis/fastapi/triggers/models.py | 25 +- api/oss/src/apis/fastapi/triggers/router.py | 98 ++++++- api/oss/src/core/triggers/dtos.py | 7 + api/oss/src/core/triggers/interfaces.py | 9 + api/oss/src/dbs/postgres/triggers/dao.py | 28 +- api/oss/src/middlewares/auth.py | 5 + .../src/tasks/asyncio/triggers/__init__.py | 0 .../src/tasks/asyncio/triggers/dispatcher.py | 244 ++++++++++++++++++ api/oss/src/tasks/taskiq/triggers/__init__.py | 0 api/oss/src/tasks/taskiq/triggers/worker.py | 63 +++++ api/oss/src/utils/env.py | 1 + .../triggers/test_triggers_ingress.py | 163 ++++++++++++ 14 files changed, 810 insertions(+), 3 deletions(-) create mode 100644 api/entrypoints/worker_triggers.py create mode 100644 api/oss/src/tasks/asyncio/triggers/__init__.py create mode 100644 api/oss/src/tasks/asyncio/triggers/dispatcher.py create mode 100644 api/oss/src/tasks/taskiq/triggers/__init__.py create mode 100644 api/oss/src/tasks/taskiq/triggers/worker.py create mode 100644 api/oss/tests/pytest/acceptance/triggers/test_triggers_ingress.py diff --git a/api/entrypoints/routers.py b/api/entrypoints/routers.py index 97e5b02314..1213f12159 100644 --- a/api/entrypoints/routers.py +++ b/api/entrypoints/routers.py @@ -147,6 +147,9 @@ from oss.src.core.triggers.registry import TriggersGatewayRegistry from oss.src.core.triggers.service import TriggersService from oss.src.apis.fastapi.triggers.router import TriggersRouter +from oss.src.tasks.asyncio.triggers.dispatcher import TriggersDispatcher +from oss.src.tasks.taskiq.triggers.worker import TriggersWorker +from taskiq_redis import RedisStreamBroker from oss.src.apis.fastapi.shared.utils import SupportHeadersMiddleware @@ -212,8 +215,12 @@ async def lifespan(*args, **kwargs): warn_deprecated_env_vars() validate_required_env_vars() + await _triggers_broker.startup() + yield + await _triggers_broker.shutdown() + for adapter in _composio_adapters.values(): await adapter.close() @@ -649,6 +656,26 @@ async def lifespan(*args, **kwargs): connections_service=connections_service, ) +# Producer side of the inbound dispatch pipeline: the ingress route enqueues +# `triggers.dispatch` tasks here; entrypoints/worker_triggers.py consumes them. +_triggers_broker = RedisStreamBroker( + url=env.redis.uri_durable, + queue_name="queues:triggers", + consumer_group_name="api-triggers-producer", + maxlen=100_000, + approximate=True, +) + +_triggers_dispatcher = TriggersDispatcher( + triggers_dao=triggers_dao, + workflows_service=workflows_service, +) + +_triggers_worker = TriggersWorker( + broker=_triggers_broker, + dispatcher=_triggers_dispatcher, +) + _t_services_done = time.perf_counter() - _t_services print(f"[STARTUP] Service initialization completed (+{_t_services_done:.3f}s)") _t_routers = time.perf_counter() @@ -765,6 +792,7 @@ async def lifespan(*args, **kwargs): triggers = TriggersRouter( triggers_service=triggers_service, + dispatch_task=_triggers_worker.dispatch_trigger, ) simple_traces = SimpleTracesRouter( diff --git a/api/entrypoints/worker_triggers.py b/api/entrypoints/worker_triggers.py new file mode 100644 index 0000000000..1b25bef5a7 --- /dev/null +++ b/api/entrypoints/worker_triggers.py @@ -0,0 +1,142 @@ +import sys + +from taskiq.cli.worker.run import run_worker +from taskiq.cli.worker.args import WorkerArgs +from taskiq_redis import RedisStreamBroker + +from oss.src.utils.logging import get_module_logger +from oss.src.utils.helpers import warn_deprecated_env_vars, validate_required_env_vars +from oss.src.utils.env import env + +from oss.src.utils.common import is_ee +from oss.src.dbs.postgres.git.dao import GitDAO +from oss.src.dbs.postgres.triggers.dao import TriggersDAO +from oss.src.dbs.postgres.workflows.dbes import ( + WorkflowArtifactDBE, + WorkflowVariantDBE, + WorkflowRevisionDBE, +) +from oss.src.dbs.postgres.environments.dbes import ( + EnvironmentArtifactDBE, + EnvironmentVariantDBE, + EnvironmentRevisionDBE, +) +from oss.src.core.workflows.service import WorkflowsService +from oss.src.core.environments.service import EnvironmentsService +from oss.src.core.embeds.service import EmbedsService +from oss.src.tasks.asyncio.triggers.dispatcher import TriggersDispatcher +from oss.src.tasks.taskiq.triggers.worker import TriggersWorker + +# Guard EE imports — see worker_tracing.py for the rationale. +if is_ee(): + from ee.src.core.access.entitlements.service import bootstrap_entitlements_services + + +import agenta as ag + +log = get_module_logger(__name__) + +# Initialize Agenta SDK +ag.init( + api_url=env.agenta.api_url, +) + +# Bound the stream so acked entries are trimmed; without this it grows unbounded. +MAXLEN_QUEUES_TRIGGERS = 100_000 + +# BROKER ------------------------------------------------------------------- +broker = RedisStreamBroker( + url=env.redis.uri_durable, + queue_name="queues:triggers", + consumer_group_name="worker-triggers", + maxlen=MAXLEN_QUEUES_TRIGGERS, + approximate=True, +) + + +# WORKERS ------------------------------------------------------------------ +triggers_dao = TriggersDAO() + +workflows_dao = GitDAO( + ArtifactDBE=WorkflowArtifactDBE, + VariantDBE=WorkflowVariantDBE, + RevisionDBE=WorkflowRevisionDBE, +) + +environments_dao = GitDAO( + ArtifactDBE=EnvironmentArtifactDBE, + VariantDBE=EnvironmentVariantDBE, + RevisionDBE=EnvironmentRevisionDBE, +) + +workflows_service = WorkflowsService( + workflows_dao=workflows_dao, +) + +environments_service = EnvironmentsService( + environments_dao=environments_dao, +) + +embeds_service = EmbedsService( + workflows_service=workflows_service, + environments_service=environments_service, +) + +workflows_service.environments_service = environments_service +workflows_service.embeds_service = embeds_service +environments_service.embeds_service = embeds_service + +triggers_dispatcher = TriggersDispatcher( + triggers_dao=triggers_dao, + workflows_service=workflows_service, +) + +triggers_worker = TriggersWorker( + broker=broker, + dispatcher=triggers_dispatcher, +) + + +def main() -> int: + """ + Main entry point for the worker. + + Returns: + Exit code (0 for success, non-zero for failure) + """ + try: + log.info("[TRIGGERS] Initializing Taskiq worker") + + # Validate environment + warn_deprecated_env_vars() + validate_required_env_vars() + + # Wire EE entitlement services so `check_entitlements` works in + # this worker process. Gated on `is_ee()` to match the import above. + if is_ee(): + bootstrap_entitlements_services() + + log.info("[TRIGGERS] Starting Taskiq worker with Redis Streams") + + # Run Taskiq worker + args = WorkerArgs( + broker="entrypoints.worker_triggers:broker", # Reference broker from this module + modules=[], + fs_discover=False, + workers=1, + max_async_tasks=50, + ) + + result = run_worker(args) + return result if result is not None else 0 + + except KeyboardInterrupt: + log.info("[TRIGGERS] Shutdown requested") + return 0 + except Exception as e: + log.error("[TRIGGERS] Fatal error", error=str(e)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/api/oss/src/apis/fastapi/triggers/models.py b/api/oss/src/apis/fastapi/triggers/models.py index d89035dca6..8944273a83 100644 --- a/api/oss/src/apis/fastapi/triggers/models.py +++ b/api/oss/src/apis/fastapi/triggers/models.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Any, Dict, List, Optional from pydantic import BaseModel @@ -91,3 +91,26 @@ class TriggerDeliveryResponse(BaseModel): class TriggerDeliveriesResponse(BaseModel): count: int = 0 deliveries: List[TriggerDelivery] = [] + + +# --------------------------------------------------------------------------- +# Trigger Ingress (inbound provider events) +# --------------------------------------------------------------------------- + + +class TriggerEventAck(BaseModel): + status: str = "accepted" + detail: Optional[str] = None + + +class ComposioEventEnvelope(BaseModel): + """Loose view of a Composio trigger webhook envelope (`{data, type, ...}`). + + Demultiplexing keys live under ``metadata`` (``trigger_id``, ``id``); the rest + is passed through to the resolver as the inbound event. + """ + + type: Optional[str] = None + timestamp: Optional[str] = None + data: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None diff --git a/api/oss/src/apis/fastapi/triggers/router.py b/api/oss/src/apis/fastapi/triggers/router.py index 2090d28942..ddd320e87f 100644 --- a/api/oss/src/apis/fastapi/triggers/router.py +++ b/api/oss/src/apis/fastapi/triggers/router.py @@ -1,5 +1,8 @@ +import hashlib +import hmac from functools import wraps -from typing import Optional +from json import JSONDecodeError, loads +from typing import Any, Optional from uuid import UUID import httpx @@ -10,6 +13,7 @@ from oss.src.utils.logging import get_module_logger from oss.src.utils.caching import get_cache, set_cache from oss.src.utils.common import is_ee +from oss.src.utils.env import env from oss.src.apis.fastapi.triggers.models import ( TriggerCatalogEventResponse, @@ -19,6 +23,7 @@ TriggerDeliveriesResponse, TriggerDeliveryQueryRequest, TriggerDeliveryResponse, + TriggerEventAck, TriggerSubscriptionCreateRequest, TriggerSubscriptionEditRequest, TriggerSubscriptionQueryRequest, @@ -70,16 +75,58 @@ async def wrapper(*args, **kwargs): return decorator +def _verify_composio_signature( + *, + body: bytes, + headers: Any, +) -> bool: + """HMAC-SHA256 verify over ``{id}.{ts}.{body}`` with ``COMPOSIO_WEBHOOK_SECRET``. + + Returns True when the secret is unset (no-op) or the signature matches. + """ + secret = env.composio.webhook_secret + if not secret: + return True + + signature = headers.get("webhook-signature") or headers.get("x-composio-signature") + webhook_id = headers.get("webhook-id") or "" + timestamp = headers.get("webhook-timestamp") or "" + if not signature: + return False + + signed = f"{webhook_id}.{timestamp}.{body.decode('utf-8', errors='replace')}" + expected = hmac.new( + secret.encode("utf-8"), + signed.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + provided = signature.split(",")[-1].strip() + return hmac.compare_digest(expected, provided) + + class TriggersRouter: def __init__( self, *, triggers_service: TriggersService, + dispatch_task: Optional[Any] = None, ): self.triggers_service = triggers_service + self.dispatch_task = dispatch_task self.router = APIRouter() + # --- Trigger Ingress (inbound provider events) --- + self.router.add_api_route( + "/composio/events", + self.ingest_composio_event, + methods=["POST"], + operation_id="ingest_composio_event", + response_model=TriggerEventAck, + status_code=status.HTTP_202_ACCEPTED, + ) + # --- Trigger Catalog --- self.router.add_api_route( "/catalog/providers/", @@ -705,3 +752,52 @@ async def fetch_delivery( count=1, delivery=delivery, ) + + # ----------------------------------------------------------------------- + # Trigger Ingress (inbound provider events) + # ----------------------------------------------------------------------- + + @intercept_exceptions() + async def ingest_composio_event( + self, + request: Request, + ) -> Any: + """Receive a Composio provider event; verify, demux, ack-fast, enqueue. + + Public (no Agenta auth) — mirrors the Stripe events receiver. Scope and + attribution are recovered downstream from the resolved subscription row. + """ + body = await request.body() + + if not _verify_composio_signature(body=body, headers=request.headers): + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + content={"status": "error", "detail": "Signature verification failed"}, + ) + + try: + envelope = loads(body) if body else {} + except JSONDecodeError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid payload", + ) + + metadata = envelope.get("metadata") or {} + trigger_id = metadata.get("trigger_id") or metadata.get("nano_id") + event_id = metadata.get("id") + + if not trigger_id or not event_id: + # Nothing to route — accept (no-op) so the provider does not retry. + return TriggerEventAck( + status="accepted", detail="No trigger_id/id to route" + ) + + if self.dispatch_task is not None: + await self.dispatch_task.kiq( + trigger_id=str(trigger_id), + event_id=str(event_id), + event=envelope, + ) + + return TriggerEventAck(status="accepted") diff --git a/api/oss/src/core/triggers/dtos.py b/api/oss/src/core/triggers/dtos.py index d9d2919d69..2c3d8a281e 100644 --- a/api/oss/src/core/triggers/dtos.py +++ b/api/oss/src/core/triggers/dtos.py @@ -15,6 +15,13 @@ ) +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +TRIGGER_MAX_RETRIES = 5 + + # --------------------------------------------------------------------------- # Trigger Enums # --------------------------------------------------------------------------- diff --git a/api/oss/src/core/triggers/interfaces.py b/api/oss/src/core/triggers/interfaces.py index a7280c90ea..5221b52349 100644 --- a/api/oss/src/core/triggers/interfaces.py +++ b/api/oss/src/core/triggers/interfaces.py @@ -148,6 +148,15 @@ async def get_subscription_by_trigger_id( """FROZEN (WP4): resolve an inbound event's ``ti_*`` to its local row.""" ... + @abstractmethod + async def get_project_and_subscription_by_trigger_id( + self, + *, + trigger_id: str, + ) -> Optional[Tuple[UUID, TriggerSubscription]]: + """Resolve a ``ti_*`` to its (project_id, subscription); the DTO omits project scope.""" + ... + # --- deliveries --------------------------------------------------------- # @abstractmethod diff --git a/api/oss/src/dbs/postgres/triggers/dao.py b/api/oss/src/dbs/postgres/triggers/dao.py index b3a1a51e3c..c53bf2b9eb 100644 --- a/api/oss/src/dbs/postgres/triggers/dao.py +++ b/api/oss/src/dbs/postgres/triggers/dao.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -from typing import List, Optional +from typing import List, Optional, Tuple from uuid import UUID from sqlalchemy import select @@ -233,6 +233,32 @@ async def get_subscription_by_trigger_id( subscription_dbe=subscription_dbe, ) + async def get_project_and_subscription_by_trigger_id( + self, + *, + trigger_id: str, + ) -> Optional[Tuple[UUID, TriggerSubscription]]: + async with self.engine.session() as session: + stmt = ( + select(TriggerSubscriptionDBE) + .filter( + TriggerSubscriptionDBE.data["ti_id"].astext == trigger_id, + ) + .limit(1) + ) + + result = await session.execute(stmt) + + subscription_dbe = result.scalars().first() + + if not subscription_dbe: + return None + + return ( + subscription_dbe.project_id, + map_subscription_dbe_to_dto(subscription_dbe=subscription_dbe), + ) + # --- DELIVERIES --------------------------------------------------------- # async def write_delivery( diff --git a/api/oss/src/middlewares/auth.py b/api/oss/src/middlewares/auth.py index 1cf4ab698b..bdbc1ee8c9 100644 --- a/api/oss/src/middlewares/auth.py +++ b/api/oss/src/middlewares/auth.py @@ -69,6 +69,11 @@ "/api/tools/connections/callback", "/preview/tools/connections/callback", "/api/preview/tools/connections/callback", + # TRIGGERS — inbound provider events arrive from Composio with no auth token + "/triggers/composio/events", + "/api/triggers/composio/events", + "/preview/triggers/composio/events", + "/api/preview/triggers/composio/events", ) _ADMIN_ENDPOINT_IDENTIFIER = "/admin/" diff --git a/api/oss/src/tasks/asyncio/triggers/__init__.py b/api/oss/src/tasks/asyncio/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/tasks/asyncio/triggers/dispatcher.py b/api/oss/src/tasks/asyncio/triggers/dispatcher.py new file mode 100644 index 0000000000..3c2bcbdfe3 --- /dev/null +++ b/api/oss/src/tasks/asyncio/triggers/dispatcher.py @@ -0,0 +1,244 @@ +"""Trigger dispatcher — asyncio side of the inbound pipeline. + +The inbound dual of ``webhooks/dispatcher.py``. Given a verified Composio event +(``ti_*`` trigger id + ``metadata.id`` dedup key + raw payload), it resolves the +local subscription, dedups, maps ``inputs_fields`` into the workflow inputs, runs +the bound workflow, and records a single delivery row with the outcome. + +Self-contained so it can run inside its own TaskIQ worker process. +""" + +from typing import Any, Dict, Optional +from uuid import UUID + +import uuid_utils.compat as uuid_compat + +from oss.src.core.shared.dtos import Status +from oss.src.core.triggers.dtos import ( + TRIGGER_EVENT_FIELDS, + SUBSCRIPTION_CONTEXT_FIELDS, + TriggerDeliveryCreate, + TriggerDeliveryData, + TriggerSubscription, +) +from oss.src.core.triggers.interfaces import TriggersDAOInterface +from oss.src.core.workflows.service import WorkflowsService +from oss.src.utils.logging import get_module_logger + +from agenta.sdk.decorators.running import WorkflowServiceRequest +from agenta.sdk.models.workflows import WorkflowRequestData +from agenta.sdk.utils.resolvers import resolve_target_fields + +log = get_module_logger(__name__) + + +class TriggersDispatcher: + """Resolves and runs one inbound provider event against its bound workflow.""" + + def __init__( + self, + *, + triggers_dao: TriggersDAOInterface, + workflows_service: WorkflowsService, + ): + self.triggers_dao = triggers_dao + self.workflows_service = workflows_service + + def _build_context( + self, + *, + event: Dict[str, Any], + subscription: TriggerSubscription, + project_id: UUID, + ) -> Dict[str, Any]: + sub_dump = subscription.model_dump(mode="json", exclude_none=True) + return { + "event": {k: v for k, v in event.items() if k in TRIGGER_EVENT_FIELDS}, + "subscription": { + k: v for k, v in sub_dump.items() if k in SUBSCRIPTION_CONTEXT_FIELDS + }, + "scope": {"project_id": str(project_id)}, + } + + async def dispatch( + self, + *, + trigger_id: str, + event_id: str, + event: Dict[str, Any], + ) -> None: + """Run the bound workflow for one inbound event (idempotent on event_id).""" + resolved = await self.triggers_dao.get_project_and_subscription_by_trigger_id( + trigger_id=trigger_id, + ) + + if resolved is None: + log.info( + "[TRIGGERS DISPATCHER] Unknown trigger_id %s — skipping", trigger_id + ) + return + + project_id, subscription = resolved + + if not subscription.enabled: + log.info( + "[TRIGGERS DISPATCHER] Subscription %s disabled — skipping", + subscription.id, + ) + return + + already_seen = await self.triggers_dao.dedup_seen( + project_id=project_id, + subscription_id=subscription.id, + event_id=event_id, + ) + if already_seen: + log.info( + "[TRIGGERS DISPATCHER] Duplicate event %s for subscription %s — skipping", + event_id, + subscription.id, + ) + return + + context = self._build_context( + event=event, + subscription=subscription, + project_id=project_id, + ) + + # MAPPING — inputs-only template (default whole-context "$" like webhooks). + template = subscription.data.inputs_fields + inputs = resolve_target_fields( + template if template is not None else "$", context + ) + + references = ( + { + k: ref.model_dump(mode="json", exclude_none=True) + for k, ref in subscription.data.references.items() + } + if subscription.data.references + else None + ) + selector = ( + subscription.data.selector.model_dump(mode="json", exclude_none=True) + if subscription.data.selector + else None + ) + + delivery_id = uuid_compat.uuid7() + user_id = subscription.created_by_id # M6 — attribute to the creator, or None + + delivery_data = TriggerDeliveryData( + event_key=subscription.data.event_key, + references=subscription.data.references, + inputs=inputs if isinstance(inputs, dict) else {"value": inputs}, + ) + + if not references: + await self._write_delivery( + project_id=project_id, + user_id=user_id, + delivery_id=delivery_id, + subscription_id=subscription.id, + event_id=event_id, + status=Status(code="400", message="failed"), + data=delivery_data.model_copy( + update={"error": "Subscription has no bound workflow reference"} + ), + ) + return + + try: + request = WorkflowServiceRequest( + references=references, + selector=selector, + data=WorkflowRequestData( + inputs=inputs if isinstance(inputs, dict) else {"value": inputs}, + ), + ) + + response = await self.workflows_service.invoke_workflow( + project_id=project_id, + user_id=user_id, + request=request, + ) + except Exception as e: + await self._write_delivery( + project_id=project_id, + user_id=user_id, + delivery_id=delivery_id, + subscription_id=subscription.id, + event_id=event_id, + status=Status(code="500", message="failed"), + data=delivery_data.model_copy(update={"error": str(e)}), + ) + raise + + status_obj = getattr(response, "status", None) + status_code = getattr(status_obj, "code", None) + outputs = getattr(response, "outputs", None) or getattr( + getattr(response, "data", None), "outputs", None + ) + + if status_code not in (None, 200): + await self._write_delivery( + project_id=project_id, + user_id=user_id, + delivery_id=delivery_id, + subscription_id=subscription.id, + event_id=event_id, + status=Status(code=str(status_code), message="failed"), + data=delivery_data.model_copy( + update={ + "error": getattr(status_obj, "message", None) + or "Workflow failed", + "result": { + "trace_id": getattr(response, "trace_id", None), + "span_id": getattr(response, "span_id", None), + }, + } + ), + ) + return + + await self._write_delivery( + project_id=project_id, + user_id=user_id, + delivery_id=delivery_id, + subscription_id=subscription.id, + event_id=event_id, + status=Status(code="200", message="success"), + data=delivery_data.model_copy( + update={ + "result": { + "trace_id": getattr(response, "trace_id", None), + "span_id": getattr(response, "span_id", None), + "outputs": outputs, + } + } + ), + ) + + async def _write_delivery( + self, + *, + project_id: UUID, + user_id: Optional[UUID], + delivery_id: UUID, + subscription_id: UUID, + event_id: str, + status: Status, + data: TriggerDeliveryData, + ) -> None: + await self.triggers_dao.write_delivery( + project_id=project_id, + user_id=user_id, + delivery=TriggerDeliveryCreate( + id=delivery_id, + subscription_id=subscription_id, + event_id=event_id, + status=status, + data=data, + ), + ) diff --git a/api/oss/src/tasks/taskiq/triggers/__init__.py b/api/oss/src/tasks/taskiq/triggers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/oss/src/tasks/taskiq/triggers/worker.py b/api/oss/src/tasks/taskiq/triggers/worker.py new file mode 100644 index 0000000000..8bcf26d553 --- /dev/null +++ b/api/oss/src/tasks/taskiq/triggers/worker.py @@ -0,0 +1,63 @@ +from typing import Any, Dict + +from taskiq import AsyncBroker, Context, TaskiqDepends + +from oss.src.core.triggers.dtos import TRIGGER_MAX_RETRIES +from oss.src.tasks.asyncio.triggers.dispatcher import TriggersDispatcher +from oss.src.utils.logging import get_module_logger + +log = get_module_logger(__name__) + + +class TriggersWorker: + """Registers and owns the TaskIQ trigger dispatch task. + + The dispatch task receives the verified Composio event inline and runs the + bound workflow, writing a single delivery row on the outcome. Idempotency + comes from the WP3 ``dedup_seen`` guard, so provider + TaskIQ retries are safe. + """ + + def __init__( + self, + *, + broker: AsyncBroker, + dispatcher: TriggersDispatcher, + ): + self.broker = broker + self.dispatcher = dispatcher + + self._register_tasks() + + def _register_tasks(self): + @self.broker.task( + task_name="triggers.dispatch", + retry_on_error=True, + max_retries=TRIGGER_MAX_RETRIES, + ) + async def dispatch_trigger( + *, + trigger_id: str, + event_id: str, + event: Dict[str, Any], + # + context: Context = TaskiqDepends(), + ) -> None: + retry_count_raw = context.message.labels.get("_taskiq_retry_count", 0) or 0 + try: + retry_count = int(retry_count_raw) + except (TypeError, ValueError): + retry_count = 0 + + log.info( + f"[TASK] triggers.dispatch " + f"trigger={trigger_id} event={event_id} " + f"attempt={retry_count}/{TRIGGER_MAX_RETRIES}" + ) + + await self.dispatcher.dispatch( + trigger_id=trigger_id, + event_id=event_id, + event=event, + ) + + self.dispatch_trigger = dispatch_trigger diff --git a/api/oss/src/utils/env.py b/api/oss/src/utils/env.py index 585386c33e..993ab83725 100644 --- a/api/oss/src/utils/env.py +++ b/api/oss/src/utils/env.py @@ -510,6 +510,7 @@ class ComposioConfig(BaseModel): api_key: str | None = os.getenv("COMPOSIO_API_KEY") api_url: str = os.getenv("COMPOSIO_API_URL", "https://backend.composio.dev/api/v3") + webhook_secret: str | None = os.getenv("COMPOSIO_WEBHOOK_SECRET") @property def enabled(self) -> bool: diff --git a/api/oss/tests/pytest/acceptance/triggers/test_triggers_ingress.py b/api/oss/tests/pytest/acceptance/triggers/test_triggers_ingress.py new file mode 100644 index 0000000000..d76db95ed3 --- /dev/null +++ b/api/oss/tests/pytest/acceptance/triggers/test_triggers_ingress.py @@ -0,0 +1,163 @@ +"""Acceptance tests for POST /triggers/composio/events (inbound ingress). + +The ingress is the inbound dual of webhooks: a public (no Agenta auth) endpoint +that Composio POSTs provider events to. It ACKs fast (202) and enqueues dispatch +asynchronously; the actual workflow run + delivery write happen in a separate +worker, so the unconditional paths here are DB-free: + + - an event for an unknown trigger id is a clean 202 no-op (nothing to route); + - an event with no routable metadata is a clean 202 no-op. + +The signature-rejection path only bites when COMPOSIO_WEBHOOK_SECRET is set +(unset → 200/202 no-op, mirroring the Stripe receiver), so it is gated on that. +The full signed-event -> workflow-invoked -> single-delivery roundtrip needs the +live Composio adapter and a bound workflow, so it is gated on COMPOSIO_API_KEY. + +Requires a running API. +""" + +import os +from uuid import uuid4 + +import pytest + + +_COMPOSIO_ENABLED = bool(os.getenv("COMPOSIO_API_KEY")) +_WEBHOOK_SECRET = os.getenv("COMPOSIO_WEBHOOK_SECRET") + +_requires_composio = pytest.mark.skipif( + not _COMPOSIO_ENABLED, + reason="needs live Composio credentials (COMPOSIO_API_KEY)", +) +_requires_webhook_secret = pytest.mark.skipif( + not _WEBHOOK_SECRET, + reason="needs COMPOSIO_WEBHOOK_SECRET set to verify signature rejection", +) + + +# --------------------------------------------------------------------------- +# DB-only: unknown trigger / no metadata are clean 202 no-ops +# --------------------------------------------------------------------------- + + +class TestTriggerIngressNoOps: + def test_unknown_trigger_id_is_accepted_noop(self, unauthed_api): + response = unauthed_api( + "POST", + "/triggers/composio/events", + json={ + "type": "github_star_added_event", + "metadata": { + "trigger_id": f"ti_{uuid4().hex}", + "id": uuid4().hex, + }, + "data": {"repository": "acme/widgets"}, + }, + ) + assert response.status_code == 202, response.text + assert response.json()["status"] == "accepted" + + def test_no_routable_metadata_is_accepted_noop(self, unauthed_api): + response = unauthed_api( + "POST", + "/triggers/composio/events", + json={"type": "some_event", "data": {}}, + ) + assert response.status_code == 202, response.text + assert response.json()["status"] == "accepted" + + def test_empty_body_is_accepted_noop(self, unauthed_api): + response = unauthed_api("POST", "/triggers/composio/events", data=b"") + assert response.status_code == 202, response.text + + +@_requires_webhook_secret +class TestTriggerIngressSignature: + def test_forged_signature_is_rejected(self, unauthed_api): + response = unauthed_api( + "POST", + "/triggers/composio/events", + headers={ + "webhook-id": "msg_1", + "webhook-timestamp": "1700000000", + "webhook-signature": "v1,deadbeef", + }, + json={ + "metadata": {"trigger_id": f"ti_{uuid4().hex}", "id": uuid4().hex}, + }, + ) + assert response.status_code == 401, response.text + + +# --------------------------------------------------------------------------- +# Dedup (needs Composio) — a duplicate metadata.id does not double-write a +# delivery. Exercised end-to-end via a real subscription bound to a workflow. +# --------------------------------------------------------------------------- + + +@_requires_composio +class TestTriggerIngressDedup: + def test_duplicate_event_id_writes_single_delivery(self, authed_api, unauthed_api): + # Create a connection + subscription so an inbound ti_* resolves locally. + slug = f"acc-{uuid4().hex[:8]}" + conn = authed_api( + "POST", + "/tools/connections/", + json={ + "connection": { + "slug": slug, + "provider_key": "composio", + "integration_key": "github", + "data": {"auth_scheme": "oauth"}, + } + }, + ) + assert conn.status_code == 200, conn.text + connection_id = conn.json()["connection"]["id"] + + create = authed_api( + "POST", + "/triggers/subscriptions/", + json={ + "subscription": { + "name": f"sub-{uuid4().hex[:8]}", + "connection_id": connection_id, + "data": { + "event_key": "GITHUB_STAR_ADDED_EVENT", + "trigger_config": {}, + "inputs_fields": {"repo": "$.event.data.repository"}, + "references": {"workflow": {"slug": "triage"}}, + }, + } + }, + ) + assert create.status_code == 200, create.text + sub = create.json()["subscription"] + subscription_id = sub["id"] + ti_id = sub["data"]["ti_id"] + + event_id = uuid4().hex + envelope = { + "type": "github_star_added_event", + "metadata": {"trigger_id": ti_id, "id": event_id}, + "data": {"repository": "acme/widgets"}, + } + + # Post the same event twice (provider redelivery) — dedup must hold. + for _ in range(2): + ack = unauthed_api("POST", "/triggers/composio/events", json=envelope) + assert ack.status_code == 202, ack.text + + # The dispatch is async; the dedup guard means at most one delivery row + # exists for this (subscription, event_id). + deliveries = authed_api( + "POST", + "/triggers/deliveries/query", + json={ + "delivery": {"subscription_id": subscription_id, "event_id": event_id} + }, + ).json()["deliveries"] + assert len(deliveries) <= 1 + + authed_api("DELETE", f"/triggers/subscriptions/{subscription_id}") + authed_api("DELETE", f"/tools/connections/{connection_id}") From 7b1dbd7ac7f14dc7541697bc70c7f35a1f977a52 Mon Sep 17 00:00:00 2001 From: Juan Pablo Vega Date: Thu, 18 Jun 2026 19:26:35 +0200 Subject: [PATCH 5/5] chore(hosting): mount worker-triggers in all compose stacks Adds a worker-triggers service mirroring worker-webhooks across every OSS and EE compose file so the trigger dispatch worker actually runs: gh/gh.local/gh.ssl (newrelic array command, or plain python for ee.gh) and dev (watchmedo auto-restart over the api source dirs). Each service reuses its sibling's env_file so COMPOSIO_WEBHOOK_SECRET flows in, and gets a /proc/1/cmdline healthcheck on entrypoints.worker_triggers. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../docker-compose/ee/docker-compose.dev.yml | 45 +++++++++++++++++++ .../ee/docker-compose.gh.local.yml | 41 +++++++++++++++++ .../docker-compose/ee/docker-compose.gh.yml | 39 ++++++++++++++++ .../docker-compose/oss/docker-compose.dev.yml | 44 ++++++++++++++++++ .../oss/docker-compose.gh.local.yml | 41 +++++++++++++++++ .../oss/docker-compose.gh.ssl.yml | 41 +++++++++++++++++ .../docker-compose/oss/docker-compose.gh.yml | 44 ++++++++++++++++++ 7 files changed, 295 insertions(+) diff --git a/hosting/docker-compose/ee/docker-compose.dev.yml b/hosting/docker-compose/ee/docker-compose.dev.yml index c08109d846..ead20d0bcd 100644 --- a/hosting/docker-compose/ee/docker-compose.dev.yml +++ b/hosting/docker-compose/ee/docker-compose.dev.yml @@ -268,6 +268,51 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + image: agenta-ee-dev-api:latest + # === EXECUTION ============================================ # + command: > + watchmedo auto-restart --directory=/app/ee/src --directory=/app/ee/databases --directory=/app/oss/src + --directory=/app/oss/databases --directory=/app/entrypoints --directory=/sdks/python/agenta + --directory=/clients/python/agenta_client --pattern=*.py --recursive --ignore-patterns=*/tests/* -- + python -m entrypoints.worker_triggers + # === STORAGE ============================================== # + volumes: + - ../../../api/ee:/app/ee + - ../../../api/oss:/app/oss + - ../../../api/entrypoints:/app/entrypoints + - ../../../sdks/python:/sdks/python + - ../../../clients/python:/clients/python + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.ee.dev} + environment: + DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} + # === NETWORK ============================================== # + networks: + - agenta-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # image: agenta-ee-dev-api:latest diff --git a/hosting/docker-compose/ee/docker-compose.gh.local.yml b/hosting/docker-compose/ee/docker-compose.gh.local.yml index 7e72548082..4565e37e32 100644 --- a/hosting/docker-compose/ee/docker-compose.gh.local.yml +++ b/hosting/docker-compose/ee/docker-compose.gh.local.yml @@ -191,6 +191,47 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + build: + context: ../../.. + dockerfile: api/ee/docker/Dockerfile.gh + # === EXECUTION ============================================ # + command: + [ + "newrelic-admin", + "run-program", + "python", + "-m", + "entrypoints.worker_triggers", + ] + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.ee.gh} + # === NETWORK ============================================== # + networks: + - agenta-ee-gh-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # build: diff --git a/hosting/docker-compose/ee/docker-compose.gh.yml b/hosting/docker-compose/ee/docker-compose.gh.yml index a74e799626..e25c845cc8 100644 --- a/hosting/docker-compose/ee/docker-compose.gh.yml +++ b/hosting/docker-compose/ee/docker-compose.gh.yml @@ -188,6 +188,45 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + image: ghcr.io/agenta-ai/${AGENTA_API_IMAGE_NAME:-internal-ee-agenta-api}:${AGENTA_API_IMAGE_TAG:-latest} + # === EXECUTION ============================================ # + command: + [ + "python", + "-m", + "entrypoints.worker_triggers", + ] + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.ee.gh} + environment: + - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} + # === NETWORK ============================================== # + networks: + - agenta-ee-gh-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # image: ghcr.io/agenta-ai/${AGENTA_API_IMAGE_NAME:-internal-ee-agenta-api}:${AGENTA_API_IMAGE_TAG:-latest} diff --git a/hosting/docker-compose/oss/docker-compose.dev.yml b/hosting/docker-compose/oss/docker-compose.dev.yml index 583d8b9496..7d482328cc 100644 --- a/hosting/docker-compose/oss/docker-compose.dev.yml +++ b/hosting/docker-compose/oss/docker-compose.dev.yml @@ -260,6 +260,50 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + image: agenta-oss-dev-api:latest + # === EXECUTION ============================================ # + command: > + watchmedo auto-restart --directory=/app/oss/src --directory=/app/oss/databases --directory=/app/entrypoints + --directory=/sdks/python/agenta --directory=/clients/python/agenta_client --pattern=*.py --recursive --ignore-patterns=*/tests/* -- + python -m entrypoints.worker_triggers + # === STORAGE ============================================== # + volumes: + # + - ../../../api/oss:/app/oss + - ../../../api/entrypoints:/app/entrypoints + - ../../../sdks/python:/sdks/python + - ../../../clients/python:/clients/python + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.oss.dev} + environment: + DOCKER_NETWORK_MODE: ${DOCKER_NETWORK_MODE:-bridge} + # === NETWORK ============================================== # + networks: + - agenta-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # image: agenta-oss-dev-api:latest diff --git a/hosting/docker-compose/oss/docker-compose.gh.local.yml b/hosting/docker-compose/oss/docker-compose.gh.local.yml index 4bc21b5293..c84f4db9fd 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.local.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.local.yml @@ -189,6 +189,47 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + build: + context: ../../.. + dockerfile: api/oss/docker/Dockerfile.gh + # === EXECUTION ============================================ # + command: + [ + "newrelic-admin", + "run-program", + "python", + "-m", + "entrypoints.worker_triggers", + ] + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.oss.gh} + # === NETWORK ============================================== # + networks: + - agenta-oss-gh-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # build: diff --git a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml index 71dda7e426..94700680ad 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.ssl.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.ssl.yml @@ -202,6 +202,47 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + build: + context: ../../.. + dockerfile: api/oss/docker/Dockerfile.gh + # === EXECUTION ============================================ # + command: + [ + "newrelic-admin", + "run-program", + "python", + "-m", + "entrypoints.worker_triggers", + ] + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.oss.gh} + # === NETWORK ============================================== # + networks: + - agenta-gh-ssl-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # build: diff --git a/hosting/docker-compose/oss/docker-compose.gh.yml b/hosting/docker-compose/oss/docker-compose.gh.yml index d39ffb8643..f0a78b3b66 100644 --- a/hosting/docker-compose/oss/docker-compose.gh.yml +++ b/hosting/docker-compose/oss/docker-compose.gh.yml @@ -201,6 +201,50 @@ services: retries: 3 start_period: 20s + worker-triggers: + # === IMAGE ================================================ # + # build: + # context: ../../.. + # dockerfile: api/oss/docker/Dockerfile.gh + image: ghcr.io/agenta-ai/${AGENTA_API_IMAGE_NAME:-agenta-api}:${AGENTA_API_IMAGE_TAG:-latest} + # === EXECUTION ============================================ # + command: + [ + "newrelic-admin", + "run-program", + "python", + "-m", + "entrypoints.worker_triggers", + ] + # === CONFIGURATION ======================================== # + env_file: + - ${ENV_FILE:-./.env.oss.gh} + environment: + - DOCKER_NETWORK_MODE=${DOCKER_NETWORK_MODE:-bridge} + # === NETWORK ============================================== # + networks: + - agenta-oss-gh-network + extra_hosts: + - "host.docker.internal:host-gateway" + # === ORCHESTRATION ======================================== # + depends_on: + postgres: + condition: service_healthy + alembic: + condition: service_completed_successfully + redis-volatile: + condition: service_healthy + redis-durable: + condition: service_healthy + # === LIFECYCLE ============================================ # + restart: always + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; cmd=pathlib.Path('/proc/1/cmdline').read_bytes().decode('utf-8','ignore'); sys.exit(0 if 'entrypoints.worker_triggers' in cmd else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + worker-events: # === IMAGE ================================================ # # build: