Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
2cdb896
feat(hosted keys): Implement serper hosted key
Feb 13, 2026
3e6527a
Handle required fields correctly for hosted keys
Feb 13, 2026
e5c8aec
Add rate limiting (3 tries, exponential backoff)
Feb 13, 2026
8a78f80
Add custom pricing, switch to exa as first hosted key
Feb 13, 2026
d174a6a
Add telemetry
Feb 13, 2026
c12e92c
Consolidate byok type definitions
Feb 13, 2026
2a36143
Add warning comment if default calculation is used
Feb 13, 2026
36e6464
Record usage to user stats table
Feb 13, 2026
f237d6f
Fix unit tests, use cost property
Feb 13, 2026
0a002fd
Include more metadata in cost output
Feb 13, 2026
36d49ef
Fix disabled tests
Feb 13, 2026
fbd1cdf
Fix spacing
Feb 14, 2026
dc4c611
Fix lint
Feb 14, 2026
68da290
Move knowledge cost restructuring away from generic block handler
Feb 16, 2026
ce02a30
Migrate knowledge unit tests
Feb 16, 2026
e6d98c6
Lint
Feb 16, 2026
ecdbe29
Fix broken tests
Mar 5, 2026
2325535
Merge branch 'staging' into feat/sim-provided-key
Mar 5, 2026
693a3d3
Add user based hosted key throttling
Mar 5, 2026
242d6e0
Refactor hosted key handling. Add optimistic handling of throttling f…
Mar 5, 2026
7b8e24e
Remove research as hosted key. Recommend BYOK if throtttling occurs
Mar 5, 2026
cd160d3
Make adding api keys adjustable via env vars
Mar 6, 2026
2082bc4
Remove vestigial fields from research
Mar 6, 2026
a90777a
Make billing actor id required for throttling
Mar 6, 2026
d7ea0af
Switch to round robin for api key distribution
Mar 6, 2026
1c5425e
Add helper method for adding hosted key cost
Mar 6, 2026
3832e5c
Strip leading double underscores to avoid breaking change
Mar 6, 2026
34cffdc
Lint fix
Mar 6, 2026
612ea7c
Remove falsy check in favor for explicit null check
Mar 6, 2026
a0fc749
Add more detailed metrics for different throttling types
Mar 6, 2026
5d04ae5
Fix _costDollars field
Mar 6, 2026
8eaf401
Handle hosted agent tool calls
Mar 7, 2026
ee2e123
Fail loudly if cost field isn't found
Mar 7, 2026
09a1b5c
Remove any type
Mar 7, 2026
0836131
Fix type error
Mar 7, 2026
427627a
Fix lint
Mar 7, 2026
d29d613
Fix usage log double logging data
Mar 7, 2026
3e94ce3
Fix test
Mar 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/sim/app/api/workspaces/[id]/byok-keys/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { getUserEntityPermissions, getWorkspaceById } from '@/lib/workspaces/per

const logger = createLogger('WorkspaceBYOKKeysAPI')

const VALID_PROVIDERS = ['openai', 'anthropic', 'google', 'mistral'] as const
const VALID_PROVIDERS = ['openai', 'anthropic', 'google', 'mistral', 'exa'] as const

const UpsertKeySchema = z.object({
providerId: z.enum(VALID_PROVIDERS),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
buildCanonicalIndex,
evaluateSubBlockCondition,
isSubBlockFeatureEnabled,
isSubBlockHiddenByHostedKey,
isSubBlockVisibleForMode,
} from '@/lib/workflows/subblocks/visibility'
import type { BlockConfig, SubBlockConfig, SubBlockType } from '@/blocks/types'
Expand Down Expand Up @@ -108,6 +109,9 @@ export function useEditorSubblockLayout(
// Check required feature if specified - declarative feature gating
if (!isSubBlockFeatureEnabled(block)) return false

// Hide tool API key fields when hosted key is available
if (isSubBlockHiddenByHostedKey(block)) return false

// Special handling for trigger-config type (legacy trigger configuration UI)
if (block.type === ('trigger-config' as SubBlockType)) {
const isPureTriggerBlock = config?.triggers?.enabled && config.category === 'triggers'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
evaluateSubBlockCondition,
hasAdvancedValues,
isSubBlockFeatureEnabled,
isSubBlockHiddenByHostedKey,
isSubBlockVisibleForMode,
resolveDependencyValue,
} from '@/lib/workflows/subblocks/visibility'
Expand Down Expand Up @@ -828,6 +829,7 @@ export const WorkflowBlock = memo(function WorkflowBlock({
if (block.hidden) return false
if (block.hideFromPreview) return false
if (!isSubBlockFeatureEnabled(block)) return false
if (isSubBlockHiddenByHostedKey(block)) return false

const isPureTriggerBlock = config?.triggers?.enabled && config.category === 'triggers'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ import {
ModalFooter,
ModalHeader,
} from '@/components/emcn'
import { AnthropicIcon, GeminiIcon, MistralIcon, OpenAIIcon } from '@/components/icons'
import { AnthropicIcon, ExaAIIcon, GeminiIcon, MistralIcon, OpenAIIcon } from '@/components/icons'
import { Skeleton } from '@/components/ui'
import {
type BYOKKey,
type BYOKProviderId,
useBYOKKeys,
useDeleteBYOKKey,
useUpsertBYOKKey,
} from '@/hooks/queries/byok-keys'
import type { BYOKProviderId } from '@/tools/types'

const logger = createLogger('BYOKSettings')

Expand Down Expand Up @@ -60,6 +60,13 @@ const PROVIDERS: {
description: 'LLM calls and Knowledge Base OCR',
placeholder: 'Enter your API key',
},
{
id: 'exa',
name: 'Exa',
icon: ExaAIIcon,
description: 'AI-powered search and research',
placeholder: 'Enter your Exa API key',
},
]

function BYOKKeySkeleton() {
Expand Down
1 change: 1 addition & 0 deletions apps/sim/blocks/blocks/exa.ts
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,7 @@ export const ExaBlock: BlockConfig<ExaResponse> = {
placeholder: 'Enter your Exa API key',
password: true,
required: true,
hideWhenHosted: true,
},
],
tools: {
Expand Down
1 change: 1 addition & 0 deletions apps/sim/blocks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ export interface SubBlockConfig {
hidden?: boolean
hideFromPreview?: boolean // Hide this subblock from the workflow block preview
requiresFeature?: string // Environment variable name that must be truthy for this subblock to be visible
hideWhenHosted?: boolean // Hide this subblock when running on hosted sim
description?: string
tooltip?: string // Tooltip text displayed via info icon next to the title
value?: (params: Record<string, any>) => string
Expand Down
22 changes: 1 addition & 21 deletions apps/sim/executor/handlers/generic/generic-handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,27 +97,7 @@ export class GenericBlockHandler implements BlockHandler {
throw error
}

const output = result.output
Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved this to knowledge block transformation, so generic handler doesn't need to handle special cases

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

makes sense

let cost = null

if (output?.cost) {
cost = output.cost
}

if (cost) {
return {
...output,
cost: {
input: cost.input,
output: cost.output,
total: cost.total,
},
tokens: cost.tokens,
model: cost.model,
}
}

return output
return result.output
} catch (error: any) {
if (!error.message || error.message === 'undefined (undefined)') {
let errorMessage = `Block execution of ${tool?.name || block.config.tool} failed`
Expand Down
3 changes: 1 addition & 2 deletions apps/sim/hooks/queries/byok-keys.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import { createLogger } from '@sim/logger'
import { keepPreviousData, useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { API_ENDPOINTS } from '@/stores/constants'
import type { BYOKProviderId } from '@/tools/types'

const logger = createLogger('BYOKKeysQueries')

export type BYOKProviderId = 'openai' | 'anthropic' | 'google' | 'mistral'

export interface BYOKKey {
id: string
providerId: BYOKProviderId
Expand Down
3 changes: 1 addition & 2 deletions apps/sim/lib/api-key/byok.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,10 @@ import { isHosted } from '@/lib/core/config/feature-flags'
import { decryptSecret } from '@/lib/core/security/encryption'
import { getHostedModels } from '@/providers/models'
import { useProvidersStore } from '@/stores/providers/store'
import type { BYOKProviderId } from '@/tools/types'

const logger = createLogger('BYOKKeys')

export type BYOKProviderId = 'openai' | 'anthropic' | 'google' | 'mistral'

export interface BYOKKeyResult {
apiKey: string
isBYOK: true
Expand Down
8 changes: 5 additions & 3 deletions apps/sim/lib/billing/core/usage-log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ export interface ModelUsageMetadata {
}

/**
* Metadata for 'fixed' category charges (currently empty, extensible)
* Metadata for 'fixed' category charges (e.g., tool cost breakdown)
*/
export type FixedUsageMetadata = Record<string, never>
export type FixedUsageMetadata = Record<string, unknown>

/**
* Union type for all metadata types
Expand Down Expand Up @@ -60,6 +60,8 @@ export interface LogFixedUsageParams {
workspaceId?: string
workflowId?: string
executionId?: string
/** Optional metadata (e.g., tool cost breakdown from API) */
metadata?: FixedUsageMetadata
}

/**
Expand Down Expand Up @@ -119,7 +121,7 @@ export async function logFixedUsage(params: LogFixedUsageParams): Promise<void>
category: 'fixed',
source: params.source,
description: params.description,
metadata: null,
metadata: params.metadata ?? null,
cost: params.cost.toString(),
workspaceId: params.workspaceId ?? null,
workflowId: params.workflowId ?? null,
Expand Down
25 changes: 25 additions & 0 deletions apps/sim/lib/core/telemetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,31 @@ export const PlatformEvents = {
})
},

/**
* Track hosted key throttled (rate limited)
*/
hostedKeyThrottled: (attrs: {
toolId: string
envVarName: string
attempt: number
maxRetries: number
delayMs: number
userId?: string
workspaceId?: string
workflowId?: string
}) => {
trackPlatformEvent('platform.hosted_key.throttled', {
'tool.id': attrs.toolId,
'hosted_key.env_var': attrs.envVarName,
'throttle.attempt': attrs.attempt,
'throttle.max_retries': attrs.maxRetries,
'throttle.delay_ms': attrs.delayMs,
...(attrs.userId && { 'user.id': attrs.userId }),
...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }),
...(attrs.workflowId && { 'workflow.id': attrs.workflowId }),
})
},

/**
* Track chat deployed (workflow deployed as chat interface)
*/
Expand Down
10 changes: 10 additions & 0 deletions apps/sim/lib/workflows/subblocks/visibility.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { getEnv, isTruthy } from '@/lib/core/config/env'
import { isHosted } from '@/lib/core/config/feature-flags'
import type { SubBlockConfig } from '@/blocks/types'

export type CanonicalMode = 'basic' | 'advanced'
Expand Down Expand Up @@ -270,3 +271,12 @@ export function isSubBlockFeatureEnabled(subBlock: SubBlockConfig): boolean {
if (!subBlock.requiresFeature) return true
return isTruthy(getEnv(subBlock.requiresFeature))
}

/**
* Check if a subblock should be hidden because we're running on hosted Sim.
* Used for tool API key fields that should be hidden when Sim provides hosted keys.
*/
export function isSubBlockHiddenByHostedKey(subBlock: SubBlockConfig): boolean {
if (!subBlock.hideWhenHosted) return false
return isHosted
}
2 changes: 2 additions & 0 deletions apps/sim/serializer/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
isCanonicalPair,
isNonEmptyValue,
isSubBlockFeatureEnabled,
isSubBlockHiddenByHostedKey,
resolveCanonicalMode,
} from '@/lib/workflows/subblocks/visibility'
import { getBlock } from '@/blocks'
Expand Down Expand Up @@ -49,6 +50,7 @@ function shouldSerializeSubBlock(
canonicalModeOverrides?: CanonicalModeOverrides
): boolean {
if (!isSubBlockFeatureEnabled(subBlockConfig)) return false
if (isSubBlockHiddenByHostedKey(subBlockConfig)) return false

if (subBlockConfig.mode === 'trigger') {
if (!isTriggerContext && !isTriggerCategory) return false
Expand Down
21 changes: 21 additions & 0 deletions apps/sim/tools/exa/answer.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { createLogger } from '@sim/logger'
import type { ExaAnswerParams, ExaAnswerResponse } from '@/tools/exa/types'
import type { ToolConfig } from '@/tools/types'

const logger = createLogger('ExaAnswerTool')

export const answerTool: ToolConfig<ExaAnswerParams, ExaAnswerResponse> = {
id: 'exa_answer',
name: 'Exa Answer',
Expand All @@ -27,6 +30,23 @@ export const answerTool: ToolConfig<ExaAnswerParams, ExaAnswerResponse> = {
description: 'Exa AI API Key',
},
},
hosting: {
envKeys: ['EXA_API_KEY_1', 'EXA_API_KEY_2', 'EXA_API_KEY_3'],
apiKeyParam: 'apiKey',
byokProviderId: 'exa',
pricing: {
type: 'custom',
getCost: (_params, output) => {
// Use _costDollars from Exa API response (internal field, stripped from final output)
if (output._costDollars?.total) {
return { cost: output._costDollars.total, metadata: { costDollars: output._costDollars } }
}
// Fallback: $5/1000 requests
logger.warn('Exa answer response missing costDollars, using fallback pricing')
return 0.005
},
},
},

request: {
url: 'https://api.exa.ai/answer',
Expand Down Expand Up @@ -61,6 +81,7 @@ export const answerTool: ToolConfig<ExaAnswerParams, ExaAnswerResponse> = {
url: citation.url,
text: citation.text || '',
})) || [],
_costDollars: data.costDollars,
},
}
},
Expand Down
22 changes: 22 additions & 0 deletions apps/sim/tools/exa/find_similar_links.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { createLogger } from '@sim/logger'
import type { ExaFindSimilarLinksParams, ExaFindSimilarLinksResponse } from '@/tools/exa/types'
import type { ToolConfig } from '@/tools/types'

const logger = createLogger('ExaFindSimilarLinksTool')

export const findSimilarLinksTool: ToolConfig<
ExaFindSimilarLinksParams,
ExaFindSimilarLinksResponse
Expand Down Expand Up @@ -76,6 +79,24 @@ export const findSimilarLinksTool: ToolConfig<
description: 'Exa AI API Key',
},
},
hosting: {
envKeys: ['EXA_API_KEY_1', 'EXA_API_KEY_2', 'EXA_API_KEY_3'],
apiKeyParam: 'apiKey',
byokProviderId: 'exa',
pricing: {
type: 'custom',
getCost: (_params, output) => {
// Use _costDollars from Exa API response (internal field, stripped from final output)
if (output._costDollars?.total) {
return { cost: output._costDollars.total, metadata: { costDollars: output._costDollars } }
}
// Fallback: $5/1000 (1-25 results) or $25/1000 (26-100 results)
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we tend to avoid fallbacks like this -- is there a reason they might not give us costDollars? We want to fail loudly and fix the root cause if things don't work since hard to know when it flips to the fallback

logger.warn('Exa find_similar_links response missing costDollars, using fallback pricing')
const resultCount = output.similarLinks?.length || 0
return resultCount <= 25 ? 0.005 : 0.025
},
},
},

request: {
url: 'https://api.exa.ai/findSimilar',
Expand Down Expand Up @@ -140,6 +161,7 @@ export const findSimilarLinksTool: ToolConfig<
highlights: result.highlights,
score: result.score || 0,
})),
_costDollars: data.costDollars,
},
}
},
Expand Down
21 changes: 21 additions & 0 deletions apps/sim/tools/exa/get_contents.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { createLogger } from '@sim/logger'
import type { ExaGetContentsParams, ExaGetContentsResponse } from '@/tools/exa/types'
import type { ToolConfig } from '@/tools/types'

const logger = createLogger('ExaGetContentsTool')

export const getContentsTool: ToolConfig<ExaGetContentsParams, ExaGetContentsResponse> = {
id: 'exa_get_contents',
name: 'Exa Get Contents',
Expand Down Expand Up @@ -61,6 +64,23 @@ export const getContentsTool: ToolConfig<ExaGetContentsParams, ExaGetContentsRes
description: 'Exa AI API Key',
},
},
hosting: {
envKeys: ['EXA_API_KEY_1', 'EXA_API_KEY_2', 'EXA_API_KEY_3'],
apiKeyParam: 'apiKey',
byokProviderId: 'exa',
pricing: {
type: 'custom',
getCost: (_params, output) => {
// Use _costDollars from Exa API response (internal field, stripped from final output)
if (output._costDollars?.total) {
return { cost: output._costDollars.total, metadata: { costDollars: output._costDollars } }
}
// Fallback: $1/1000 pages
logger.warn('Exa get_contents response missing costDollars, using fallback pricing')
return (output.results?.length || 0) * 0.001
},
},
},

request: {
url: 'https://api.exa.ai/contents',
Expand Down Expand Up @@ -132,6 +152,7 @@ export const getContentsTool: ToolConfig<ExaGetContentsParams, ExaGetContentsRes
summary: result.summary || '',
highlights: result.highlights,
})),
_costDollars: data.costDollars,
},
}
},
Expand Down
Loading
Loading