improvement(webhooks): move non-polling off trigger.dev

Author: icecrasher321

apps/sim/app/api/webhooks/route.ts

@@ -367,9 +367,7 @@ export async function POST(request: NextRequest) {
             )
           }
 
-          // Configure each new webhook (for providers that need configuration)
-          const pollingProviders = ['gmail', 'outlook']
-          const needsConfiguration = pollingProviders.includes(provider)
+          const needsConfiguration = provider === 'gmail' || provider === 'outlook'
 
           if (needsConfiguration) {
             const configureFunc =

apps/sim/lib/core/async-jobs/config.ts

@@ -7,6 +7,7 @@ const logger = createLogger('AsyncJobsConfig')
 
 let cachedBackend: JobQueueBackend | null = null
 let cachedBackendType: AsyncBackendType | null = null
+let cachedInlineBackend: JobQueueBackend | null = null
 
 /**
  * Determines which async backend to use based on environment configuration.
@@ -71,6 +72,30 @@ export function getCurrentBackendType(): AsyncBackendType | null {
   return cachedBackendType
 }
 
+/**
+ * Gets a job queue backend that bypasses Trigger.dev (Redis -> Database).
+ * Used for non-polling webhooks that should always execute inline.
+ */
+export async function getInlineJobQueue(): Promise<JobQueueBackend> {
+  if (cachedInlineBackend) {
+    return cachedInlineBackend
+  }
+
+  const redis = getRedisClient()
+  if (redis) {
+    const { RedisJobQueue } = await import('@/lib/core/async-jobs/backends/redis')
+    cachedInlineBackend = new RedisJobQueue(redis)
+  } else {
+    const { DatabaseJobQueue } = await import('@/lib/core/async-jobs/backends/database')
+    cachedInlineBackend = new DatabaseJobQueue()
+  }
+
+  logger.info(
+    `Inline job backend initialized: ${cachedInlineBackend ? 'redis or database' : 'none'}`
+  )
+  return cachedInlineBackend
+}
+
 /**
  * Checks if jobs should be executed inline (fire-and-forget).
  * For Redis/DB backends, we execute inline. Trigger.dev handles execution itself.
@@ -85,4 +110,5 @@ export function shouldExecuteInline(): boolean {
 export function resetJobQueueCache(): void {
   cachedBackend = null
   cachedBackendType = null
+  cachedInlineBackend = null
 }

apps/sim/lib/core/async-jobs/index.ts

@@ -1,6 +1,7 @@
 export {
   getAsyncBackendType,
   getCurrentBackendType,
+  getInlineJobQueue,
   getJobQueue,
   resetJobQueueCache,
   shouldExecuteInline,

apps/sim/lib/webhooks/processor.ts

@@ -5,7 +5,7 @@ import { and, eq, isNull, or } from 'drizzle-orm'
 import { type NextRequest, NextResponse } from 'next/server'
 import { v4 as uuidv4 } from 'uuid'
 import { checkEnterprisePlan, checkTeamPlan } from '@/lib/billing/subscriptions/utils'
-import { getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs'
+import { getInlineJobQueue, getJobQueue, shouldExecuteInline } from '@/lib/core/async-jobs'
 import { isProd } from '@/lib/core/config/feature-flags'
 import { safeCompare } from '@/lib/core/security/encryption'
 import { getEffectiveDecryptedEnv } from '@/lib/environment/utils'
@@ -29,6 +29,7 @@ import {
 import { executeWebhookJob } from '@/background/webhook-execution'
 import { resolveEnvVarReferences } from '@/executor/utils/reference-validation'
 import { isConfluencePayloadMatch } from '@/triggers/confluence/utils'
+import { isPollingWebhookProvider } from '@/triggers/constants'
 import { isGitHubEventMatch } from '@/triggers/github/utils'
 import { isHubSpotContactEventMatch } from '@/triggers/hubspot/utils'
 import { isJiraEventMatch } from '@/triggers/jira/utils'
@@ -1116,15 +1117,24 @@ export async function queueWebhookExecution(
       ...(credentialId ? { credentialId } : {}),
     }
 
-    const jobQueue = await getJobQueue()
-    const jobId = await jobQueue.enqueue('webhook-execution', payload, {
-      metadata: { workflowId: foundWorkflow.id, userId: actorUserId },
-    })
-    logger.info(
-      `[${options.requestId}] Queued webhook execution task ${jobId} for ${foundWebhook.provider} webhook`
-    )
+    const isPolling = isPollingWebhookProvider(payload.provider)
 
-    if (shouldExecuteInline()) {
+    if (isPolling && !shouldExecuteInline()) {
+      const jobQueue = await getJobQueue()
+      const jobId = await jobQueue.enqueue('webhook-execution', payload, {
+        metadata: { workflowId: foundWorkflow.id, userId: actorUserId },
+      })
+      logger.info(
+        `[${options.requestId}] Queued polling webhook execution task ${jobId} for ${foundWebhook.provider} webhook via job queue`
+      )
+    } else {
+      const jobQueue = await getInlineJobQueue()
+      const jobId = await jobQueue.enqueue('webhook-execution', payload, {
+        metadata: { workflowId: foundWorkflow.id, userId: actorUserId },
+      })
+      logger.info(
+        `[${options.requestId}] Executing ${foundWebhook.provider} webhook ${jobId} inline`
+      )
       void (async () => {
         try {
           await jobQueue.startJob(jobId)

apps/sim/lib/webhooks/utils.server.ts

@@ -19,6 +19,7 @@ import {
   refreshAccessTokenIfNeeded,
   resolveOAuthAccountId,
 } from '@/app/api/auth/oauth/utils'
+import { isPollingWebhookProvider } from '@/triggers/constants'
 
 const logger = createLogger('WebhookUtils')
 
@@ -2222,10 +2223,7 @@ export async function syncWebhooksForCredentialSet(params: {
     `[${requestId}] Syncing webhooks for credential set ${credentialSetId}, provider ${provider}`
   )
 
-  // Polling providers get unique paths per credential (for independent state)
-  // External webhook providers share the same path (external service sends to one URL)
-  const pollingProviders = ['gmail', 'outlook', 'rss', 'imap']
-  const useUniquePaths = pollingProviders.includes(provider)
+  const useUniquePaths = isPollingWebhookProvider(provider)
 
   const credentials = await getCredentialsForCredentialSet(credentialSetId, oauthProviderId)
 

apps/sim/triggers/constants.ts

@@ -1,37 +1,11 @@
 /**
- * System subblock IDs that are part of the trigger UI infrastructure
- * and should NOT be aggregated into triggerConfig or validated as user fields.
- *
- * These subblocks provide UI/UX functionality but aren't configuration data.
+ * Set of webhook provider names that use polling-based triggers.
+ * Mirrors the `polling: true` flag on TriggerConfig entries.
+ * Used to route execution: polling providers use the full job queue
+ * (Trigger.dev), non-polling providers execute inline.
  */
-export const SYSTEM_SUBBLOCK_IDS: string[] = [
-  'triggerCredentials', // OAuth credentials subblock
-  'triggerInstructions', // Setup instructions text
-  'webhookUrlDisplay', // Webhook URL display
-  'samplePayload', // Example payload display
-  'setupScript', // Setup script code (e.g., Apps Script)
-  'scheduleInfo', // Schedule status display (next run, last run)
-]
+export const POLLING_PROVIDERS = new Set(['gmail', 'outlook', 'rss', 'imap'])
 
-/**
- * Trigger-related subblock IDs that represent runtime metadata. They should remain
- * in the workflow state but must not be modified or cleared by diff operations.
- *
- * Note: 'triggerConfig' is included because it's an aggregate of individual trigger
- * field subblocks. Those individual fields are compared separately, so comparing
- * triggerConfig would be redundant. Additionally, the client populates triggerConfig
- * with default values from the trigger definition on load, which aren't present in
- * the deployed state, causing false positive change detection.
- */
-export const TRIGGER_RUNTIME_SUBBLOCK_IDS: string[] = [
-  'webhookId',
-  'triggerPath',
-  'triggerConfig',
-  'triggerId',
-]
-
-/**
- * Maximum number of consecutive failures before a trigger (schedule/webhook) is auto-disabled.
- * This prevents runaway errors from continuously executing failing workflows.
- */
-export const MAX_CONSECUTIVE_FAILURES = 100
+export function isPollingWebhookProvider(provider: string): boolean {
+  return POLLING_PROVIDERS.has(provider)
+}

apps/sim/triggers/gmail/poller.ts

@@ -30,6 +30,7 @@ export const gmailPollingTrigger: TriggerConfig = {
   description: 'Triggers when new emails are received in Gmail (requires Gmail credentials)',
   version: '1.0.0',
   icon: GmailIcon,
+  polling: true,
 
   subBlocks: [
     {

apps/sim/triggers/imap/poller.ts

@@ -12,6 +12,7 @@ export const imapPollingTrigger: TriggerConfig = {
   description: 'Triggers when new emails are received via IMAP (works with any email provider)',
   version: '1.0.0',
   icon: MailServerIcon,
+  polling: true,
 
   subBlocks: [
     // Connection settings

apps/sim/triggers/outlook/poller.ts

@@ -24,6 +24,7 @@ export const outlookPollingTrigger: TriggerConfig = {
   description: 'Triggers when new emails are received in Outlook (requires Microsoft credentials)',
   version: '1.0.0',
   icon: OutlookIcon,
+  polling: true,
 
   subBlocks: [
     {

apps/sim/triggers/rss/poller.ts

@@ -8,6 +8,7 @@ export const rssPollingTrigger: TriggerConfig = {
   description: 'Triggers when new items are published to an RSS feed',
   version: '1.0.0',
   icon: RssIcon,
+  polling: true,
 
   subBlocks: [
     {