diff --git a/arize_toolkit/cli/alert_integrations.py b/arize_toolkit/cli/alert_integrations.py new file mode 100644 index 0000000..c939570 --- /dev/null +++ b/arize_toolkit/cli/alert_integrations.py @@ -0,0 +1,33 @@ +import click + +from arize_toolkit.cli.client_factory import get_client +from arize_toolkit.cli.output import print_result + +PROVIDER_CHOICES = ["slack", "pagerduty", "opsgenie"] + + +@click.group("alert-integrations") +def alert_integrations_group(): + """Manage alert integrations (Slack, PagerDuty, OpsGenie).""" + pass + + +@alert_integrations_group.command("list") +@click.option( + "--provider", + type=click.Choice(PROVIDER_CHOICES), + default=None, + help="Filter by provider.", +) +@click.option("--search", default=None, help="Search by integration name.") +@click.pass_context +def integrations_list(ctx, provider, search): + """List alert integrations for the organization.""" + client = get_client(ctx) + data = client.list_integrations(provider_name=provider, search=search) + print_result( + data, + columns=["id", "name", "providerName", "channelName", "alertSeverity"], + title="Alert Integrations", + json_mode=ctx.obj["json_mode"], + ) diff --git a/arize_toolkit/cli/main.py b/arize_toolkit/cli/main.py index 3052438..13804d2 100644 --- a/arize_toolkit/cli/main.py +++ b/arize_toolkit/cli/main.py @@ -3,6 +3,7 @@ import click from arize_toolkit import __version__ +from arize_toolkit.cli.alert_integrations import alert_integrations_group from arize_toolkit.cli.config_cmd import config_group, get_profile, update_profile from arize_toolkit.cli.custom_metrics import custom_metrics_group from arize_toolkit.cli.dashboards import dashboards_group @@ -80,6 +81,7 @@ def persist_client_state(ctx, *args, **kwargs): update_profile(profile_name, **updates) +cli.add_command(alert_integrations_group) cli.add_command(config_group) cli.add_command(spaces_group) cli.add_command(orgs_group) diff --git a/arize_toolkit/cli/monitors.py b/arize_toolkit/cli/monitors.py index 9299c1a..d017c14 100644 --- a/arize_toolkit/cli/monitors.py +++ b/arize_toolkit/cli/monitors.py @@ -20,6 +20,7 @@ def _common_monitor_options(f): f = click.option("--operator2", type=click.Choice(OPERATOR_CHOICES), default=None, help="Second operator (double mode).")(f) f = click.option("--email", multiple=True, help="Email addresses for notifications.")(f) f = click.option("--integration-key-id", multiple=True, help="Integration key IDs for notifications.")(f) + f = click.option("--integration-name", multiple=True, help="Integration names for notifications (resolved to IDs).")(f) return f @@ -88,6 +89,7 @@ def monitors_create_performance( operator2, email, integration_key_id, + integration_name, ): """Create a performance monitor.""" client = get_client(ctx) @@ -108,6 +110,7 @@ def monitors_create_performance( operator2=operator2, email_addresses=list(email) if email else None, integration_key_ids=list(integration_key_id) if integration_key_id else None, + integration_names=list(integration_name) if integration_name else None, ) print_url(url, label="Created monitor") @@ -138,6 +141,7 @@ def monitors_create_drift( operator2, email, integration_key_id, + integration_name, ): """Create a drift monitor.""" client = get_client(ctx) @@ -158,6 +162,7 @@ def monitors_create_drift( operator2=operator2, email_addresses=list(email) if email else None, integration_key_ids=list(integration_key_id) if integration_key_id else None, + integration_names=list(integration_name) if integration_name else None, ) print_url(url, label="Created monitor") @@ -190,6 +195,7 @@ def monitors_create_data_quality( operator2, email, integration_key_id, + integration_name, ): """Create a data quality monitor.""" client = get_client(ctx) @@ -211,6 +217,7 @@ def monitors_create_data_quality( operator2=operator2, email_addresses=list(email) if email else None, integration_key_ids=list(integration_key_id) if integration_key_id else None, + integration_names=list(integration_name) if integration_name else None, ) print_url(url, label="Created monitor") diff --git a/arize_toolkit/client.py b/arize_toolkit/client.py index f3b32ea..0fe86bb 100644 --- a/arize_toolkit/client.py +++ b/arize_toolkit/client.py @@ -59,6 +59,7 @@ GetEvaluatorQuery, GetEvaluatorsQuery, ) +from arize_toolkit.queries.integration_queries import GetIntegrationKeysQuery from arize_toolkit.queries.llm_utils_queries import ( CreateAnnotationMutation, CreatePromptMutation, @@ -1810,6 +1811,66 @@ def _resolve_llm_integration_id(self, llm_integration_name: Optional[str] = None available = [r.name for r in results] raise ValueError(f"No LLM integration found with name '{llm_integration_name}'. " f"Available integrations: {available}") + def list_integrations( + self, + provider_name: Optional[str] = None, + search: Optional[str] = None, + ) -> List[dict]: + """List alert integration keys (slack, pagerduty, opsgenie) for the organization. + + Args: + provider_name (Optional[str]): Filter by provider name (e.g. "slack", "pagerduty", "opsgenie") + search (Optional[str]): Search by integration name + + Returns: + List[dict]: List of integration keys with id, name, providerName, channelName, alertSeverity, etc. + + Example: + ```python + # List all integrations + integrations = client.list_integrations() + + # List only slack integrations + slack_integrations = client.list_integrations(provider_name="slack") + ``` + """ + results = GetIntegrationKeysQuery.run_graphql_query_to_list( + self._graphql_client, + organization_id=self.org_id, + providerName=provider_name, + search=search, + ) + return [r.to_dict() for r in results] + + def _resolve_integration_key_ids( + self, + integration_names: Union[str, List[str]], + ) -> List[str]: + """Resolve alert integration names to their IDs. + + Args: + integration_names: Name(s) of alert integrations (slack, pagerduty, opsgenie) to look up. + + Returns: + List[str]: The IDs of the matching integrations. + + Raises: + ValueError: If any name does not match an existing integration. + """ + if isinstance(integration_names, str): + integration_names = [integration_names] + results = GetIntegrationKeysQuery.run_graphql_query_to_list( + self._graphql_client, + organization_id=self.org_id, + ) + available = {r.name: r.id for r in results} + resolved_ids = [] + for name in integration_names: + if name not in available: + raise ValueError(f"No alert integration found with name '{name}'. " f"Available integrations: {list(available.keys())}") + resolved_ids.append(available[name]) + return resolved_ids + def get_evaluators( self, search: Optional[str] = None, @@ -2864,6 +2925,7 @@ def create_performance_monitor( std_dev_multiplier2: Optional[float] = None, email_addresses: Optional[Union[str, List[str]]] = None, integration_key_ids: Optional[Union[str, List[str]]] = None, + integration_names: Optional[Union[str, List[str]]] = None, filters: Optional[Union[List[Dict], List[DimensionFilterInput]]] = None, ) -> str: """Creates a new performance metric monitor for a model. @@ -2894,6 +2956,8 @@ def create_performance_monitor( std_dev_multiplier2 (Optional[float]): Standard deviation multiplier for the second threshold (only used if threshold_mode is "double") email_addresses (Optional[Union[str, List[str]]]): Email address(es) to notify when the monitor is triggered integration_key_ids (Optional[Union[str, List[str]]]): ID(s) of integration key(s) to notify when the monitor is triggered + integration_names (Optional[Union[str, List[str]]]): Name(s) of alert integrations (slack, pagerduty, opsgenie) to notify. + These are resolved to integration key IDs automatically. Use this as an alternative to integration_key_ids. filters (Optional[Union[List[Dict], List[DimensionFilterInput]]]): Filters to apply to the monitor - filterType (FilterRowType): Type of filter to apply (featureLabel, tagLabel, actuals, predictionScore, etc) - operator (ComparisonOperator): Comparison operator to apply (equals, notEquals, greaterThan, lessThan, greaterThanOrEqual, lessThanOrEqual) @@ -2908,6 +2972,14 @@ def create_performance_monitor( """ if performance_metric is None and custom_metric_id is None: raise ValueError("Either performance_metric or custom_metric_id must be provided") + if integration_names: + resolved_ids = self._resolve_integration_key_ids(integration_names) + if integration_key_ids: + if isinstance(integration_key_ids, str): + integration_key_ids = [integration_key_ids] + integration_key_ids = list(integration_key_ids) + resolved_ids + else: + integration_key_ids = resolved_ids contacts = [] if email_addresses: if isinstance(email_addresses, str): @@ -2985,6 +3057,7 @@ def create_drift_monitor( std_dev_multiplier2: Optional[float] = 2.0, email_addresses: Optional[Union[str, List[str]]] = None, integration_key_ids: Optional[Union[str, List[str]]] = None, + integration_names: Optional[Union[str, List[str]]] = None, filters: Optional[Union[List[Dict], List[DimensionFilterInput]]] = None, ) -> str: """Creates a new drift monitor for a model. @@ -3013,6 +3086,8 @@ def create_drift_monitor( std_dev_multiplier2 (Optional[float]): Standard deviation multiplier for the second threshold (default is 2.0 if threshold_mode is "double" and a threshold2 is not provided) email_addresses (Optional[List[str]]): Email addresses to notify when the monitor is triggered integration_key_ids (Optional[List[str]]): IDs of integration keys to notify when the monitor is triggered + integration_names (Optional[Union[str, List[str]]]): Name(s) of alert integrations (slack, pagerduty, opsgenie) to notify. + These are resolved to integration key IDs automatically. Use this as an alternative to integration_key_ids. filters (Optional[Union[List[Dict], List[DimensionFilterInput]]]): Filters to apply to the monitor - filterType (FilterRowType): Type of filter to apply (featureLabel, tagLabel, actuals, predictionScore, etc) - operator (ComparisonOperator): Comparison operator to apply (equals, notEquals, greaterThan, lessThan, greaterThanOrEqual, lessThanOrEqual) @@ -3026,6 +3101,14 @@ def create_drift_monitor( ArizeAPIException: If monitor creation fails or there is an API error """ + if integration_names: + resolved_ids = self._resolve_integration_key_ids(integration_names) + if integration_key_ids: + if isinstance(integration_key_ids, str): + integration_key_ids = [integration_key_ids] + integration_key_ids = list(integration_key_ids) + resolved_ids + else: + integration_key_ids = resolved_ids contacts = [] if email_addresses: if isinstance(email_addresses, str): @@ -3102,6 +3185,7 @@ def create_data_quality_monitor( std_dev_multiplier2: Optional[float] = 2.0, email_addresses: Optional[Union[str, List[str]]] = None, integration_key_ids: Optional[Union[str, List[str]]] = None, + integration_names: Optional[Union[str, List[str]]] = None, filters: Optional[Union[List[Dict], List[DimensionFilterInput]]] = None, ) -> str: """Creates a new data quality monitor for a model. @@ -3131,6 +3215,8 @@ def create_data_quality_monitor( std_dev_multiplier2 (Optional[float]): Standard deviation multiplier for the second threshold (default is 2.0 if threshold_mode is "double" and a threshold2 is not provided) email_addresses (Optional[Union[str, List[str]]]): Email address(es) to notify when the monitor is triggered integration_key_ids (Optional[Union[str, List[str]]]): ID(s) of integration key(s) to notify when the monitor is triggered + integration_names (Optional[Union[str, List[str]]]): Name(s) of alert integrations (slack, pagerduty, opsgenie) to notify. + These are resolved to integration key IDs automatically. Use this as an alternative to integration_key_ids. filters (Optional[Union[List[Dict], List[DimensionFilterInput]]]): Filters to apply to the monitor - filterType (FilterRowType): Type of filter to apply (featureLabel, tagLabel, actuals, predictionScore, etc) - operator (ComparisonOperator): Comparison operator to apply (equals, notEquals, greaterThan, lessThan, greaterThanOrEqual, lessThanOrEqual) @@ -3144,6 +3230,14 @@ def create_data_quality_monitor( ArizeAPIException: If monitor creation fails or there is an API error """ + if integration_names: + resolved_ids = self._resolve_integration_key_ids(integration_names) + if integration_key_ids: + if isinstance(integration_key_ids, str): + integration_key_ids = [integration_key_ids] + integration_key_ids = list(integration_key_ids) + resolved_ids + else: + integration_key_ids = resolved_ids contacts = [] if email_addresses: if isinstance(email_addresses, str): diff --git a/arize_toolkit/models/monitor_models.py b/arize_toolkit/models/monitor_models.py index 2064de7..4d7dfe8 100644 --- a/arize_toolkit/models/monitor_models.py +++ b/arize_toolkit/models/monitor_models.py @@ -5,17 +5,28 @@ from arize_toolkit.models.base_models import BaseNode, Dimension, DimensionFilterInput, DimensionValue, User from arize_toolkit.models.custom_metrics_models import CustomMetric -from arize_toolkit.types import ComparisonOperator, DataQualityMetric, DimensionCategory, DriftMetric, FilterRowType, ModelEnvironment, MonitorCategory, PerformanceMetric +from arize_toolkit.types import ( + ComparisonOperator, + DataQualityMetric, + DimensionCategory, + DriftMetric, + FilterRowType, + IntegrationAlertSeverity, + IntegrationProvider, + ModelEnvironment, + MonitorCategory, + PerformanceMetric, +) from arize_toolkit.utils import GraphQLModel ## Monitor GraphQL Models ## class IntegrationKey(BaseNode): - providerName: Literal["slack", "pagerduty", "opsgenie"] + providerName: IntegrationProvider createdAt: Optional[datetime] = Field(default=None) channelName: Optional[str] = Field(default=None) - alertSeverity: Optional[str] = Field(default=None) + alertSeverity: Optional[IntegrationAlertSeverity] = Field(default=None) class MonitorContact(GraphQLModel): diff --git a/arize_toolkit/queries/integration_queries.py b/arize_toolkit/queries/integration_queries.py new file mode 100644 index 0000000..a6b1c2c --- /dev/null +++ b/arize_toolkit/queries/integration_queries.py @@ -0,0 +1,42 @@ +from typing import List, Optional, Tuple + +from arize_toolkit.models.monitor_models import IntegrationKey +from arize_toolkit.queries.basequery import ArizeAPIException, BaseQuery, BaseResponse, BaseVariables + + +class GetIntegrationKeysQuery(BaseQuery): + graphql_query = ( + """ + query getIntegrationKeys($organization_id: ID!, $providerName: IntegrationProvider, $search: String) { + node(id: $organization_id) { + ... on AccountOrganization { + integrations(providerName: $providerName, search: $search) { """ + + IntegrationKey.to_graphql_fields() + + """ + } + } + } + } + """ + ) + query_description = "Get all alert integration keys for an organization" + + class Variables(BaseVariables): + organization_id: str + providerName: Optional[str] = None + search: Optional[str] = None + + class QueryException(ArizeAPIException): + message: str = "Error getting integration keys" + + class QueryResponse(IntegrationKey): + pass + + @classmethod + def _parse_graphql_result(cls, result: dict) -> Tuple[List[BaseResponse], bool, Optional[str]]: + integrations = result.get("node", {}).get("integrations", []) + return ( + [cls.QueryResponse(**integration) for integration in integrations], + False, + None, + ) diff --git a/arize_toolkit/types.py b/arize_toolkit/types.py index 4ac0556..543ce86 100644 --- a/arize_toolkit/types.py +++ b/arize_toolkit/types.py @@ -222,6 +222,28 @@ class PromptVersionInputVariableFormatEnum(InputValidationEnum): MUSTACHE = "MUSTACHE", "Mustache", "mustache", "{{}}" +class IntegrationProvider(InputValidationEnum): + """Alert integration providers for monitor notifications (slack, pagerduty, opsgenie)""" + + slack = "slack", "Slack", "SLACK" + pagerduty = "pagerduty", "PagerDuty", "Pagerduty", "PAGERDUTY" + opsgenie = "opsgenie", "OpsGenie", "Opsgenie", "OPSGENIE" + + +class IntegrationAlertSeverity(InputValidationEnum): + """Alert severity levels for PagerDuty and OpsGenie integrations""" + + opsgenieP1 = "opsgenieP1", "OpsGenie P1", "P1" + opsgenieP2 = "opsgenieP2", "OpsGenie P2", "P2" + opsgenieP3 = "opsgenieP3", "OpsGenie P3", "P3" + opsgenieP4 = "opsgenieP4", "OpsGenie P4", "P4" + opsgenieP5 = "opsgenieP5", "OpsGenie P5", "P5" + pagerdutycritical = "pagerdutycritical", "PagerDuty Critical", "critical" + pagerdutyerror = "pagerdutyerror", "PagerDuty Error", "error" + pagerdutywarning = "pagerdutywarning", "PagerDuty Warning", "warning" + pagerdutyinfo = "pagerdutyinfo", "PagerDuty Info", "info" + + class LLMIntegrationProvider(InputValidationEnum): """The LLM provider used for execution with the prompt""" diff --git a/docs_site/docs/monitor_tools.md b/docs_site/docs/monitor_tools.md index e9fae81..939f381 100644 --- a/docs_site/docs/monitor_tools.md +++ b/docs_site/docs/monitor_tools.md @@ -22,6 +22,7 @@ Click any function name to jump to the detailed section. | Quick-link to a monitor in the UI | [`get_monitor_url`](#get_monitor_url) | | Get monitor metric values over time | [`get_monitor_metric_values`](#get_monitor_metric_values) | | Get latest monitor metric value | [`get_latest_monitor_value`](#get_latest_monitor_value) | +| List alert integrations | [`list_integrations`](#list_integrations) | | Create a **performance** monitor | [`create_performance_monitor`](#create_performance_monitor) | | Create a **drift** monitor | [`create_drift_monitor`](#create_drift_monitor) | | Create a **data-quality** monitor | [`create_data_quality_monitor`](#create_data_quality_monitor) | @@ -340,6 +341,47 @@ if result["metric_value"] > result["threshold_value"]: ______________________________________________________________________ +### `list_integrations` + +```python +integrations: list[dict] = client.list_integrations( + provider_name: str | None = None, # "slack", "pagerduty", "opsgenie" + search: str | None = None, +) +``` + +Lists alert integration keys (Slack, PagerDuty, OpsGenie) configured for your organization. Use this to discover available integrations and their IDs or names for use in monitor notifications. + +**Parameters** + +- `provider_name` (optional) – Filter by provider (`"slack"`, `"pagerduty"`, `"opsgenie"`). +- `search` (optional) – Search by integration name. + +**Returns** + +A list of dictionaries, one per integration. Each contains: + +- `id` – The integration key ID (use with `integration_key_ids` in monitor creation) +- `name` – The integration name (use with `integration_names` in monitor creation) +- `providerName` – The provider (`"slack"`, `"pagerduty"`, `"opsgenie"`) +- `channelName` – The channel name (Slack only) +- `alertSeverity` – The alert severity level (PagerDuty/OpsGenie only) +- `createdAt` – When the integration was created + +**Example** + +```python +# List all integrations +integrations = client.list_integrations() +for i in integrations: + print(f"{i['name']} ({i['providerName']})") + +# List only Slack integrations +slack = client.list_integrations(provider_name="slack") +``` + +______________________________________________________________________ + ## Creating Monitors The three helpers below share a very large surface-area of parameters – most of which are optional.\ @@ -376,6 +418,8 @@ monitor_url: str = client.create_performance_monitor( std_dev_multiplier2: float | None = None, # ––– Notifications ––– email_addresses: list[str] | None = None, + integration_key_ids: list[str] | None = None, + integration_names: list[str] | None = None, filters: list[dict] | None = None, ) ``` @@ -418,7 +462,9 @@ Creates a new performance monitor. Returns a URL path to the newly created monit - `threshold2` – Secondary threshold value used when `threshold_mode` is `"double"`. - `operator2` – Comparison operator for the secondary threshold. - `std_dev_multiplier2` – Standard-deviation multiplier for the secondary adaptive threshold. -- `email_addresses` – List of email addresses that should receive alert notifications. Currently only supports direct email alerting, not other integrations. +- `email_addresses` – List of email addresses that should receive alert notifications. +- `integration_key_ids` – List of integration key IDs (from `list_integrations`) to notify when triggered. +- `integration_names` – List of integration names to notify. These are resolved to IDs automatically using `list_integrations`. Use as a convenient alternative to `integration_key_ids`. - `filters` – List of filters to apply to the monitor. **Returns** @@ -428,12 +474,24 @@ A URL path to the newly created monitor. **Example** ```python +# With email notifications +monitor_url = client.create_performance_monitor( + name="Accuracy < 80%", + model_name="fraud-detection-v3", + model_environment_name="production", + performance_metric="accuracy", + threshold=0.8, + email_addresses=["alerts@my-org.com"], +) + +# With integration notifications (by name) monitor_url = client.create_performance_monitor( name="Accuracy < 80%", model_name="fraud-detection-v3", model_environment_name="production", performance_metric="accuracy", threshold=0.8, + integration_names=["#ml-alerts"], # Slack channel name ) print("Created:", monitor_url) ``` @@ -464,6 +522,8 @@ monitor_url: str = client.create_drift_monitor( evaluation_window_length_seconds: int = 259200, # 3 days delay_seconds: int = 0, email_addresses: list[str] | None = None, + integration_key_ids: list[str] | None = None, + integration_names: list[str] | None = None, filters: list[dict] | None = None, ) ``` @@ -501,7 +561,9 @@ Creates a new drift monitor. Returns a URL path to the newly created monitor. Default is `259 200` s (3 days). - `delay_seconds` – How long to wait before evaluating newly-arrived data (to accommodate ingestion lag).\ Default is `0`. -- `email_addresses` – List of email addresses that should receive alert notifications. Currently only supports direct email alerting, not other integrations. +- `email_addresses` – List of email addresses that should receive alert notifications. +- `integration_key_ids` – List of integration key IDs to notify when triggered. +- `integration_names` – List of integration names to notify (resolved to IDs automatically). - `filters` – List of filters to apply to the monitor. **Returns** @@ -517,6 +579,7 @@ monitor_url = client.create_drift_monitor( drift_metric="psi", dimension_category="prediction", operator="greaterThan", + integration_names=["PagerDuty Alerts"], ) print("Created:", monitor_url) ``` @@ -547,6 +610,8 @@ monitor_url: str = client.create_data_quality_monitor( evaluation_window_length_seconds: int = 259200, # 3 days delay_seconds: int = 0, email_addresses: list[str] | None = None, + integration_key_ids: list[str] | None = None, + integration_names: list[str] | None = None, filters: list[dict] | None = None, ) ``` @@ -582,7 +647,9 @@ Creates a data-quality monitor. Returns a URL path to the newly created monitor. - `scheduled_runtime_days_of_week` – List of ISO weekday numbers (`1` = Mon … `7` = Sun) on which the monitor may run. - `evaluation_window_length_seconds` – Size of the rolling aggregation window. - `delay_seconds` – How long to wait before evaluating newly-arrived data (to accommodate ingestion lag). -- `email_addresses` – List of email addresses that should receive alert notifications. Currently only supports direct email alerting, not other integrations. +- `email_addresses` – List of email addresses that should receive alert notifications. +- `integration_key_ids` – List of integration key IDs to notify when triggered. +- `integration_names` – List of integration names to notify (resolved to IDs automatically). - `filters` – List of filters to apply to the monitor. **Returns** @@ -599,6 +666,7 @@ monitor_url = client.create_data_quality_monitor( model_environment_name="production", operator="greaterThan", dimension_category="prediction", + integration_names=["#data-quality-alerts"], ) print("Created:", monitor_url) ``` diff --git a/tests/test_models/test_monitor_models.py b/tests/test_models/test_monitor_models.py index ac9e047..bd17d18 100644 --- a/tests/test_models/test_monitor_models.py +++ b/tests/test_models/test_monitor_models.py @@ -81,14 +81,24 @@ def test_integration_key(self): providerName="slack", createdAt=created_time, channelName="#alerts", - alertSeverity="high", ) assert key.id == "key123" assert key.name == "Slack Integration" - assert key.providerName == "slack" + assert key.providerName.name == "slack" assert key.channelName == "#alerts" - assert key.alertSeverity == "high" + assert key.alertSeverity is None + + # Test with PagerDuty alert severity + pd_key = IntegrationKey( + id="key456", + name="PagerDuty Integration", + providerName="pagerduty", + createdAt=created_time, + alertSeverity="pagerdutycritical", + ) + assert pd_key.providerName.name == "pagerduty" + assert pd_key.alertSeverity.name == "pagerdutycritical" def test_monitor_contact(self): """Test MonitorContact model.""" @@ -112,7 +122,7 @@ def test_monitor_contact(self): ) assert integration_contact.notificationChannelType == "integration" - assert integration_contact.integration.providerName == "pagerduty" + assert integration_contact.integration.providerName.name == "pagerduty" def test_metric_window(self): """Test MetricWindow model."""