Add more detailed metrics for different throttling types

Author: Theodore Li

apps/sim/lib/core/telemetry.ts

@@ -935,7 +935,31 @@ export const PlatformEvents = {
   },
 
   /**
-   * Track hosted key rate limited
+   * Track when a rate limit error is surfaced to the end user (not retried/absorbed).
+   * Fires for both billing-actor limits and exhausted upstream retries.
+   */
+  userThrottled: (attrs: {
+    toolId: string
+    reason: 'billing_actor_limit' | 'upstream_retries_exhausted'
+    provider?: string
+    retryAfterMs?: number
+    userId?: string
+    workspaceId?: string
+    workflowId?: string
+  }) => {
+    trackPlatformEvent('platform.user.throttled', {
+      'tool.id': attrs.toolId,
+      'throttle.reason': attrs.reason,
+      ...(attrs.provider && { 'provider.id': attrs.provider }),
+      ...(attrs.retryAfterMs != null && { 'rate_limit.retry_after_ms': attrs.retryAfterMs }),
+      ...(attrs.userId && { 'user.id': attrs.userId }),
+      ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }),
+      ...(attrs.workflowId && { 'workflow.id': attrs.workflowId }),
+    })
+  },
+
+  /**
+   * Track hosted key rate limited by upstream provider (429 from the external API)
    */
   hostedKeyRateLimited: (attrs: {
     toolId: string

apps/sim/tools/index.ts

@@ -98,12 +98,11 @@ async function injectHostedKeyIfNeeded(
       retryAfterMs: acquireResult.retryAfterMs,
     })
 
-    PlatformEvents.hostedKeyRateLimited({
+    PlatformEvents.userThrottled({
       toolId: tool.id,
-      envVarName: 'billing_actor_rate_limited',
-      attempt: 0,
-      maxRetries: 0,
-      delayMs: acquireResult.retryAfterMs ?? 0,
+      reason: 'billing_actor_limit',
+      provider,
+      retryAfterMs: acquireResult.retryAfterMs ?? 0,
       userId: executionContext?.userId,
       workspaceId: executionContext?.workspaceId,
       workflowId: executionContext?.workflowId,
@@ -175,6 +174,15 @@ async function executeWithRetry<T>(
       lastError = error
 
       if (!isRateLimitError(error) || attempt === maxRetries) {
+        if (isRateLimitError(error) && attempt === maxRetries) {
+          PlatformEvents.userThrottled({
+            toolId,
+            reason: 'upstream_retries_exhausted',
+            userId: executionContext?.userId,
+            workspaceId: executionContext?.workspaceId,
+            workflowId: executionContext?.workflowId,
+          })
+        }
         throw error
       }