diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts deleted file mode 100644 index 7f008db7e654..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts +++ /dev/null @@ -1,421 +0,0 @@ -import type { Event } from '@sentry/node'; -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES_ATTRIBUTE, - GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, - GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, - GEN_AI_REQUEST_MODEL_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, - GEN_AI_RESPONSE_MODEL_ATTRIBUTE, - GEN_AI_SYSTEM_ATTRIBUTE, - GEN_AI_TOOL_CALL_ID_ATTRIBUTE, - GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, - GEN_AI_TOOL_INPUT_ATTRIBUTE, - GEN_AI_TOOL_NAME_ATTRIBUTE, - GEN_AI_TOOL_OUTPUT_ATTRIBUTE, - GEN_AI_TOOL_TYPE_ATTRIBUTE, - GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, - GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, - GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, -} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -describe('Vercel AI integration (V6)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans with genAI recording disabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && - span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 10, - ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - - const firstGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && - span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 10, - ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - - const secondInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the second span?"}]', - ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ); - - const secondGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), - ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, - ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(toolInvokeAgentSpan!.status).toBe('ok'); - - const toolGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, - ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(toolGenerateContentSpan!.status).toBe('ok'); - - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); - expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('creates ai related spans with genAI recording enabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the first span?"}]', - ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ); - - const firstGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('First span here!'), - ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); - expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain( - 'First span here!', - ); - - const secondInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the second span?"}]', - ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const secondGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), - ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - - const toolGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] !== undefined, - ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeDefined(); - expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); - expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( - 'Get the current weather for a location', - ); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-error-in-tool.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('captures error in tool', async () => { - let transactionEvent: Event | undefined; - let errorEvent: Event | undefined; - - await createRunner() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); - expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - }, - }) - .expect({ - event: event => { - errorEvent = event; - }, - }) - .start() - .completed(); - - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); - - expect(errorEvent).toBeDefined(); - expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); - - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans with v6', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - const invokeAgentSpans = container.items.filter( - span => span.attributes['sentry.op'].value === 'gen_ai.invoke_agent', - ); - expect(invokeAgentSpans).toHaveLength(3); - - const generateContentSpans = container.items.filter( - span => span.attributes['sentry.op'].value === 'gen_ai.generate_content', - ); - expect(generateContentSpans).toHaveLength(3); - - const toolSpan = container.items.find( - span => span.attributes['sentry.op'].value === 'gen_ai.execute_tool', - ); - expect(toolSpan).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-tool-loop-agent.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates spans for ToolLoopAgent with tool calls', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(4); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent weather_agent'); - expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - - const toolCallsGenerateContentSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["tool-calls"]', - ); - expect(toolCallsGenerateContentSpan).toBeDefined(); - expect(toolCallsGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(toolCallsGenerateContentSpan!.status).toBe('ok'); - expect(toolCallsGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolCallsGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(toolCallsGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); - expect(toolSpan!.status).toBe('ok'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); - expect(toolSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); - - const finalGenerateContentSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["stop"]', - ); - expect(finalGenerateContentSpan).toBeDefined(); - expect(finalGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(finalGenerateContentSpan!.status).toBe('ok'); - expect(finalGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(finalGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - expect(finalGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(25); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-error-in-tool.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-error-in-tool.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-tool-loop-agent.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-tool-loop-agent.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs similarity index 90% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs index ee2dc802cd9c..5c7ca973cd27 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs @@ -21,9 +21,11 @@ async function run() { prompt: 'Where is the first span?', }); - // This span should have input and output prompts attached because telemetry is explicitly enabled. + // This span should have input and output prompts attached because recording is explicitly enabled. + // (v6 enables recording implicitly from `isEnabled: true`; v7's diagnostics channel only carries + // the explicit `recordInputs`/`recordOutputs` flags, so we set them here to exercise both.) await generateText({ - experimental_telemetry: { isEnabled: true }, + experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true }, model: new MockLanguageModelV3({ doGenerate: async () => ({ finishReason: { unified: 'stop', raw: 'stop' }, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts new file mode 100644 index 000000000000..8afd4b1f8559 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -0,0 +1,430 @@ +import type { Event } from '@sentry/node'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_SYSTEM_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe.each([ + ['6', '^6.0.0'], + ['7', '7.0.0-beta.179'], +])('Vercel AI integration (version %s)', (_, vercelAiVersion) => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with genAI recording disabled', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const firstInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.['vercel.ai.operationId']?.value === 'ai.generateText' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && + span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 10, + )!; + expect(firstInvokeAgentSpan).toBeDefined(); + expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan.status).toBe('ok'); + expect(firstInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes?.['vercel.ai.operationId']?.value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_REQUEST_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(10); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(20); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]?.value).toBe(30); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + const firstGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes?.['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && + span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 10, + )!; + expect(firstGenerateContentSpan).toBeDefined(); + expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan.status).toBe('ok'); + expect(firstGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes?.['vercel.ai.operationId']?.value).toBe( + 'ai.generateText.doGenerate', + ); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_SYSTEM_ATTRIBUTE]?.value).toBe('mock-provider'); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(10); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + const secondInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the second span?"}]', + )!; + expect(secondInvokeAgentSpan).toBeDefined(); + expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan.status).toBe('ok'); + expect(secondInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + )!; + expect(secondInvokeAgentSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ); + + const secondGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + (span.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value as string | undefined)?.includes( + 'Second span here!', + ), + )!; + expect(secondGenerateContentSpan).toBeDefined(); + expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(secondGenerateContentSpan.status).toBe('ok'); + expect(secondGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + + const toolInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, + )!; + expect(toolInvokeAgentSpan).toBeDefined(); + expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan.status).toBe('ok'); + + const toolGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, + )!; + expect(toolGenerateContentSpan).toBeDefined(); + expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan.status).toBe('ok'); + + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan).toBeDefined(); + expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan.status).toBe('ok'); + expect(toolExecutionSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_TYPE_ATTRIBUTE]?.value).toBe('function'); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates ai related spans with genAI recording enabled', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const firstInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the first span?"}]', + )!; + expect(firstInvokeAgentSpan).toBeDefined(); + expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan.status).toBe('ok'); + expect(firstInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes?.['vercel.ai.operationId']?.value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"user","content":"Where is the first span?"}]', + ); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ); + + const firstGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + (span.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value as string | undefined)?.includes( + 'First span here!', + ), + )!; + expect(firstGenerateContentSpan).toBeDefined(); + expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan.status).toBe('ok'); + expect(firstGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes?.['vercel.ai.operationId']?.value).toBe( + 'ai.generateText.doGenerate', + ); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value).toContain( + 'First span here!', + ); + + const secondInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the second span?"}]', + )!; + expect(secondInvokeAgentSpan).toBeDefined(); + expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan.status).toBe('ok'); + expect(secondInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + + const secondGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + (span.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value as string | undefined)?.includes( + 'Second span here!', + ), + )!; + expect(secondGenerateContentSpan).toBeDefined(); + expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(secondGenerateContentSpan.status).toBe('ok'); + expect(secondGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + + const toolInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + )!; + expect(toolInvokeAgentSpan).toBeDefined(); + expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan.status).toBe('ok'); + expect(toolInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ); + + const toolGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes?.[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] !== undefined, + )!; + expect(toolGenerateContentSpan).toBeDefined(); + expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan.status).toBe('ok'); + expect(toolGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan.attributes?.[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeDefined(); + expect(toolGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(15); + + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan).toBeDefined(); + expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan.status).toBe('ok'); + expect(toolExecutionSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]?.value).toBe( + 'Get the current weather for a location', + ); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-error-in-tool.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('captures error in tool', async () => { + let transactionEvent: Event | undefined; + let errorEvent: Event | undefined; + + await createRunner() + .expect({ + transaction: transaction => { + transactionEvent = transaction; + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent')!; + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan.name).toBe('invoke_agent'); + expect(invokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id')!; + expect(generateContentSpan).toBeDefined(); + expect(generateContentSpan.name).toBe('generate_content mock-model-id'); + expect(generateContentSpan.status).toBe('ok'); + expect(generateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan).toBeDefined(); + expect(toolSpan.name).toBe('execute_tool getWeather'); + expect(toolSpan.status).toBe('error'); + expect(toolSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + }, + }) + .expect({ + event: event => { + errorEvent = event; + }, + }) + .start() + .completed(); + + expect(transactionEvent).toBeDefined(); + expect(transactionEvent!.transaction).toBe('main'); + + expect(errorEvent).toBeDefined(); + expect(errorEvent!.level).toBe('error'); + expect(errorEvent!.tags).toEqual( + expect.objectContaining({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }), + ); + + // Trace id should be the same for the transaction and error event + expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with v6', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const invokeAgentSpans = container.items.filter( + span => span.attributes?.['sentry.op']?.value === 'gen_ai.invoke_agent', + ); + expect(invokeAgentSpans).toHaveLength(3); + + const generateContentSpans = container.items.filter( + span => span.attributes?.['sentry.op']?.value === 'gen_ai.generate_content', + ); + expect(generateContentSpans).toHaveLength(3); + + const toolSpan = container.items.find( + span => span.attributes?.['sentry.op']?.value === 'gen_ai.execute_tool', + ); + expect(toolSpan).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-tool-loop-agent.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates spans for ToolLoopAgent with tool calls', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan.name).toBe('invoke_agent weather_agent'); + expect(invokeAgentSpan.status).toBe('ok'); + expect(invokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan.attributes?.[GEN_AI_REQUEST_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + + const toolCallsGenerateContentSpan = container.items.find( + span => span.attributes?.[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["tool-calls"]', + )!; + expect(toolCallsGenerateContentSpan).toBeDefined(); + expect(toolCallsGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolCallsGenerateContentSpan.status).toBe('ok'); + expect(toolCallsGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolCallsGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(10); + expect(toolCallsGenerateContentSpan.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(20); + + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan).toBeDefined(); + expect(toolSpan.name).toBe('execute_tool getWeather'); + expect(toolSpan.status).toBe('ok'); + expect(toolSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_TYPE_ATTRIBUTE]?.value).toBe('function'); + + const finalGenerateContentSpan = container.items.find( + span => span.attributes?.[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["stop"]', + )!; + expect(finalGenerateContentSpan).toBeDefined(); + expect(finalGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(finalGenerateContentSpan.status).toBe('ok'); + expect(finalGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(finalGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(15); + expect(finalGenerateContentSpan.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(25); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); +}); diff --git a/packages/node/src/integrations/tracing/vercelai/index.ts b/packages/node/src/integrations/tracing/vercelai/index.ts index a0b3f3126d01..5c357eeaed8a 100644 --- a/packages/node/src/integrations/tracing/vercelai/index.ts +++ b/packages/node/src/integrations/tracing/vercelai/index.ts @@ -1,6 +1,8 @@ import type { Client, IntegrationFn } from '@sentry/core'; import { addVercelAiProcessors, defineIntegration } from '@sentry/core'; import { generateInstrumentOnce, type modulesIntegration } from '@sentry/node-core'; +import { tracingChannel as otelTracingChannel } from '@sentry/opentelemetry/tracing-channel'; +import { subscribeVercelAiTracingChannel } from '@sentry/server-utils'; import { INTEGRATION_NAME } from './constants'; import { SentryVercelAiInstrumentation } from './instrumentation'; import type { VercelAiOptions } from './types'; @@ -24,6 +26,11 @@ const _vercelAIIntegration = ((options: VercelAiOptions = {}) => { options, setupOnce() { instrumentation = instrumentVercelAi(); + + // Subscribe to the `ai` SDK's native telemetry tracing channel (ai >= 7). + // This is a no-op on versions that don't publish to the channel, so it is always safe to call. + // The factory needs the Sentry OTel context manager, which `initOpenTelemetry()` registers after `setupOnce`, so defer a tick. + void Promise.resolve().then(() => subscribeVercelAiTracingChannel(otelTracingChannel)); }, afterAllSetup(client) { // Auto-detect if we should force the integration when running with 'ai' package available diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index 43918e600a28..624790edda9a 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -23,3 +23,4 @@ export type { RedisTracingChannelFactory, RedisTracingChannelSubscribers, } from './redis/redis-dc-subscriber'; +export { subscribeVercelAiTracingChannel } from './vercel-ai/vercel-ai-dc-subscriber'; diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts new file mode 100644 index 000000000000..9dfb173df008 --- /dev/null +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -0,0 +1,586 @@ +/* eslint-disable max-lines */ +// `@sentry/conventions` marks several gen_ai attributes (e.g. `GEN_AI_SYSTEM`, `GEN_AI_TOOL_*`, +// `GEN_AI_REQUEST_AVAILABLE_TOOLS`) as deprecated in favour of newer semconv names. We intentionally +// keep emitting the current names so these spans match the OTel-based (v6) integration and what the +// Sentry product consumes today; migrating to the new names is a separate, coordinated change. +/* eslint-disable typescript-eslint/no-deprecated */ +import { + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_FUNCTION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_REQUEST_AVAILABLE_TOOLS, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_SYSTEM, + GEN_AI_TOOL_INPUT, + GEN_AI_TOOL_NAME, + GEN_AI_TOOL_OUTPUT, + GEN_AI_TOOL_TYPE, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { GEN_AI_EXECUTE_TOOL_SPAN_OP, GEN_AI_INVOKE_AGENT_SPAN_OP } from '@sentry/conventions/op'; +import type { Integration, Span } from '@sentry/core'; +import { + captureException, + debug, + getActiveSpan, + getClient, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SPAN_STATUS_ERROR, + spanToTraceContext, + startInactiveSpan, + withScope, +} from '@sentry/core'; +import { DEBUG_BUILD } from '../debug-build'; + +/** + * The single tracing channel the `ai` package (>= 7) publishes all telemetry lifecycle events to + * via `node:diagnostics_channel`. Events are discriminated by their `type` field. + * @see https://github.com/vercel/ai/pull/15660 + */ +const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry'; + +const ORIGIN = 'auto.vercelai.channel'; +const INTEGRATION_NAME = 'VercelAI'; + +// `@sentry/conventions` does not expose these yet, so we keep the literals here. +const GEN_AI_TOOL_CALL_ID_ATTRIBUTE = 'gen_ai.tool.call.id'; +const GEN_AI_EMBEDDINGS_OPERATION = 'embeddings'; +const GEN_AI_RERANK_OPERATION = 'rerank'; +// The model-call op matches the Vercel AI OTel integration (`gen_ai.generate_content`) rather than +// the generic `gen_ai.chat`, so v6 (OTel) and v7 (channel) produce the same spans. +const GEN_AI_GENERATE_CONTENT_OPERATION = 'generate_content'; + +// Subset of the `vercel.ai.*` passthrough attributes the OTel integration emits that we reproduce. +const VERCEL_AI_OPERATION_ID_ATTRIBUTE = 'vercel.ai.operationId'; +const VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE = 'vercel.ai.model.provider'; +const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries'; + +// Tracks the top-level operationId per `callId` so a model-call span can name its `doGenerate`/ +// `doStream` operation the same way the OTel integration does. Cleared when the top-level span ends. +const operationIdByCallId = new Map(); + +// Only top-level operations own the `callId` → operationId mapping; `step`/`languageModelCall`/ +// `executeTool` share the parent's `callId`, so they must not clear it. +const ROOT_OPERATION_TYPES = new Set(['generateText', 'streamText', 'embed', 'rerank']); + +/** Drop the `callId` → operationId mapping once the owning top-level operation settles (success or error). */ +function clearOperationId(data: VercelAiChannelMessage): void { + if (!ROOT_OPERATION_TYPES.has(data.type)) { + return; + } + const callId = asString(data.event.callId); + if (callId) { + operationIdByCallId.delete(callId); + } +} + +const NOOP = (): void => {}; + +/** The lifecycle event types the `ai:telemetry` channel can carry. */ +type ChannelEventType = + | 'generateText' + | 'streamText' + | 'step' + | 'languageModelCall' + | 'executeTool' + | 'embed' + | 'rerank'; + +/** + * The context object the AI SDK passes through one tracing-channel call. It is the same object + * identity across `start`/`end`/`asyncEnd`/`error`, and Node's `tracingChannel` attaches + * `result`/`error` to it as the traced promise settles. + */ +interface VercelAiChannelMessage { + type: ChannelEventType; + event: Record; + result?: unknown; + error?: unknown; +} + +/** + * Payload observed by subscribers — the channel context with the span stamped on by the factory's + * `start` transform, plus a marker for events we deliberately don't open a span for (`step`). + */ +type VercelAiTracingChannelContextWithSpan = T & { _sentrySpan?: Span; _sentrySkip?: boolean }; + +/** Subscriber object accepted by {@link VercelAiTracingChannel.subscribe}. */ +interface VercelAiTracingChannelSubscribers { + start: (data: VercelAiTracingChannelContextWithSpan) => void; + asyncStart: (data: VercelAiTracingChannelContextWithSpan) => void; + asyncEnd: (data: VercelAiTracingChannelContextWithSpan) => void; + end: (data: VercelAiTracingChannelContextWithSpan) => void; + error: (data: VercelAiTracingChannelContextWithSpan) => void; +} + +/** Minimal tracing-channel surface the subscriber depends on. */ +interface VercelAiTracingChannel { + subscribe(subs: Partial>): void; +} + +/** + * Platform-provided factory that returns a tracing channel for the given channel name. The factory + * is responsible for, when `start` fires, calling `transformStart(data)` and storing the returned + * span on `data._sentrySpan` so the subscriber's `asyncEnd`/`error` handlers can read it. + * + * Node passes `@sentry/opentelemetry/tracing-channel`, which uses `bindStore` to additionally make + * the span the active OTel context for the duration of the traced operation. That is what makes + * nested AI SDK operations (model calls, tool calls) become children of the enclosing span without + * any manual parent bookkeeping here. + */ +type VercelAiTracingChannelFactory = ( + name: string, + transformStart: (data: T) => Span, +) => VercelAiTracingChannel; + +let subscribed = false; + +/** + * Subscribe Sentry span handlers to the `ai` SDK's native telemetry tracing channel (`ai:telemetry`, + * available in `ai` >= 7) and emit fully-formed `gen_ai.*` spans directly — no OpenTelemetry span + * post-processing involved. + * + * Safe to always call: on `ai` versions that don't publish to the channel (e.g. < 7) nothing is + * ever emitted and this is inert, so there is no double-instrumentation against the OTel-based + * patcher. Idempotent. + */ +export function subscribeVercelAiTracingChannel(tracingChannel: VercelAiTracingChannelFactory): void { + if (subscribed) { + return; + } + subscribed = true; + + try { + const channel = tracingChannel(AI_SDK_TELEMETRY_TRACING_CHANNEL, createSpanFromMessage); + + channel.subscribe({ + start: NOOP, + asyncStart: NOOP, + // `end` fires synchronously before the traced promise settles, so it's too early to finish an + // async span. We finish on `asyncEnd` (success) / `error` instead. + end: NOOP, + asyncEnd: data => { + // `step` reuses the active span; the error handler ends on failure. + if (data._sentrySkip || data.error) { + return; + } + const span = data._sentrySpan; + if (!span) { + return; + } + enrichSpanOnEnd(span, data); + span.end(); + clearOperationId(data); + }, + error: data => { + if (data._sentrySkip) { + return; + } + // Always drop the mapping (even if the span is missing) so a rejected call can't leak a stale + // `callId` → operationId entry. + clearOperationId(data); + const span = data._sentrySpan; + if (!span) { + return; + } + span.setStatus({ + code: SPAN_STATUS_ERROR, + message: data.error instanceof Error ? data.error.message : 'unknown_error', + }); + span.end(); + }, + }); + } catch { + // The factory relies on `node:diagnostics_channel`, which isn't always available. Fail closed. + DEBUG_BUILD && debug.log('Vercel AI node:diagnostics_channel subscription failed.'); + } +} + +/** + * Transform a channel `start` payload into the span that should be active for the operation. For + * `step` we deliberately don't open a span (model calls and tool calls are siblings under the + * invoke_agent span, matching the OTel-based output), so we reuse the active span and mark the + * payload to skip ending it. + */ +function createSpanFromMessage(data: VercelAiTracingChannelContextWithSpan): Span { + const { type, event } = data; + + if (type === 'step' || !event || typeof event !== 'object') { + data._sentrySkip = true; + // The active span is the enclosing invoke_agent span; reusing it leaves the tree unchanged. + return getActiveSpan() as Span; + } + + const { recordInputs } = getRecordingOptions(event); + const provider = asString(event.provider); + const modelId = asString(event.modelId); + const callId = asString(event.callId); + const maxRetries = asNumber(event.maxRetries); + + const baseAttributes: Record = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, + ...(provider ? { [GEN_AI_SYSTEM]: provider, [VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider } : {}), + ...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}), + ...(maxRetries !== undefined ? { [VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE]: maxRetries } : {}), + }; + + switch (type) { + case 'generateText': + case 'streamText': + return buildInvokeAgentSpan(event, baseAttributes, recordInputs, callId, type === 'streamText'); + case 'languageModelCall': + return buildModelCallSpan(event, baseAttributes, recordInputs, callId, modelId); + case 'executeTool': + return buildToolSpan(event, recordInputs); + case 'embed': + return buildSimpleModelSpan(GEN_AI_EMBEDDINGS_OPERATION, baseAttributes, modelId, { + ...(recordInputs && event.value !== undefined ? { [GEN_AI_EMBEDDINGS_INPUT]: safeStringify(event.value) } : {}), + }); + case 'rerank': + return buildSimpleModelSpan(GEN_AI_RERANK_OPERATION, baseAttributes, modelId, {}); + default: + data._sentrySkip = true; + return getActiveSpan() as Span; + } +} + +type Attributes = Record; + +function buildInvokeAgentSpan( + event: Record, + baseAttributes: Attributes, + recordInputs: boolean, + callId: string | undefined, + isStream: boolean, +): Span { + const functionId = asString(event.functionId); + const operationId = asString(event.operationId) ?? (isStream ? 'ai.streamText' : 'ai.generateText'); + if (callId) { + operationIdByCallId.set(callId, operationId); + } + return startInactiveSpan({ + name: functionId ? `${GEN_AI_INVOKE_AGENT_SPAN_OP} ${functionId}` : GEN_AI_INVOKE_AGENT_SPAN_OP, + op: `gen_ai.${GEN_AI_INVOKE_AGENT_SPAN_OP}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_INVOKE_AGENT_SPAN_OP, + [VERCEL_AI_OPERATION_ID_ATTRIBUTE]: operationId, + [GEN_AI_RESPONSE_STREAMING]: isStream, + ...(functionId ? { [GEN_AI_FUNCTION_ID]: functionId } : {}), + ...(recordInputs ? buildInputMessageAttributes(event) : {}), + }, + }); +} + +function buildModelCallSpan( + event: Record, + baseAttributes: Attributes, + recordInputs: boolean, + callId: string | undefined, + modelId: string | undefined, +): Span { + const parentOperationId = callId ? operationIdByCallId.get(callId) : undefined; + const operationId = parentOperationId + ? `${parentOperationId}.${parentOperationId.includes('stream') ? 'doStream' : 'doGenerate'}` + : 'ai.generateText.doGenerate'; + return startInactiveSpan({ + name: modelId ? `${GEN_AI_GENERATE_CONTENT_OPERATION} ${modelId}` : GEN_AI_GENERATE_CONTENT_OPERATION, + op: `gen_ai.${GEN_AI_GENERATE_CONTENT_OPERATION}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_GENERATE_CONTENT_OPERATION, + [VERCEL_AI_OPERATION_ID_ATTRIBUTE]: operationId, + ...(recordInputs ? buildInputMessageAttributes(event) : {}), + ...(recordInputs && Array.isArray(event.tools) + ? { [GEN_AI_REQUEST_AVAILABLE_TOOLS]: safeStringify(event.tools) } + : {}), + }, + }); +} + +function buildToolSpan(event: Record, recordInputs: boolean): Span { + const toolCall = isRecord(event.toolCall) ? event.toolCall : {}; + const toolName = asString(toolCall.toolName); + const toolCallId = asString(event.toolCallId) ?? asString(toolCall.toolCallId); + const toolInput = toolCall.input ?? toolCall.args; + return startInactiveSpan({ + name: toolName ? `${GEN_AI_EXECUTE_TOOL_SPAN_OP} ${toolName}` : GEN_AI_EXECUTE_TOOL_SPAN_OP, + op: `gen_ai.${GEN_AI_EXECUTE_TOOL_SPAN_OP}`, + attributes: { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, + [GEN_AI_OPERATION_NAME]: GEN_AI_EXECUTE_TOOL_SPAN_OP, + [GEN_AI_TOOL_TYPE]: 'function', + ...(toolName ? { [GEN_AI_TOOL_NAME]: toolName } : {}), + ...(toolCallId ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId } : {}), + ...(recordInputs && toolInput !== undefined ? { [GEN_AI_TOOL_INPUT]: safeStringify(toolInput) } : {}), + }, + }); +} + +function buildSimpleModelSpan( + operation: string, + baseAttributes: Attributes, + modelId: string | undefined, + extraAttributes: Attributes, +): Span { + return startInactiveSpan({ + name: modelId ? `${operation} ${modelId}` : operation, + op: `gen_ai.${operation}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: operation, + ...extraAttributes, + }, + }); +} + +/** + * Best-effort enrichment from the resolved value the AI SDK attaches to the channel context. + * Everything here is guarded: when a field is missing or the shape differs across `ai` versions, + * we simply don't set the attribute rather than emit a malformed span. + */ +function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { + const { type, result } = data; + if (!isRecord(result)) { + return; + } + + const { recordOutputs } = getRecordingOptions(data.event); + + if (type === 'executeTool') { + if (recordOutputs) { + span.setAttribute(GEN_AI_TOOL_OUTPUT, safeStringify(result.output ?? result)); + } + // From V5 on, tool errors are not rejected (so the `error` channel verb never fires) — they + // surface as `tool-error` content on the resolved result. Mirror the OTel path by marking the + // span and capturing the error. + const output = isRecord(result.output) ? result.output : undefined; + if (output?.type === 'tool-error') { + captureToolError(span, data, output.error); + } + return; + } + + // `languageModelCall` results report usage as `{ total }` objects; top-level/step results report + // flat numbers. `tokenCount` handles both. + const usage = isRecord(result.usage) ? result.usage : undefined; + if (usage) { + const inputTokens = tokenCount(usage.inputTokens) ?? tokenCount(usage.tokens); + const outputTokens = tokenCount(usage.outputTokens); + const totalTokens = tokenCount(usage.totalTokens) ?? sum(inputTokens, outputTokens); + if (inputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_INPUT_TOKENS, inputTokens); + } + if (outputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens); + } + if (totalTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_TOTAL_TOKENS, totalTokens); + } + } + + // Match the OTel integration: finish reasons live on the model-call (`generate_content`) span, not + // on the top-level `invoke_agent` span. + const finishReason = getFinishReason(result); + if (finishReason && type === 'languageModelCall') { + span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, safeStringify([finishReason])); + } + + const response = isRecord(result.response) ? result.response : undefined; + const responseId = asString(response?.id) ?? asString(result.responseId); + if (responseId) { + span.setAttribute(GEN_AI_RESPONSE_ID, responseId); + } + const responseModel = asString(response?.modelId) ?? asString(data.event.modelId); + if (responseModel) { + span.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); + } + + if (recordOutputs) { + // `languageModelCall` exposes the response as a `content` parts array; top-level results expose + // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. + const parts = + type === 'languageModelCall' && Array.isArray(result.content) + ? partsFromContent(result.content) + : partsFromTextAndToolCalls(result.text, result.toolCalls); + const outputMessages = buildOutputMessages(parts, finishReason); + if (outputMessages) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, outputMessages); + } + } +} + +/** Maps a Vercel AI finish reason to the OTel `gen_ai.output.messages` form (`tool-calls` → `tool_call`). */ +function normalizeFinishReason(finishReason: string | undefined): string { + return finishReason === 'tool-calls' ? 'tool_call' : (finishReason ?? 'stop'); +} + +/** Reads the finish reason from a result — a string on top-level results, `{ unified }` on model calls. */ +function getFinishReason(result: Record): string | undefined { + const finishReason = result.finishReason; + if (typeof finishReason === 'string') { + return finishReason; + } + return isRecord(finishReason) ? asString(finishReason.unified) : undefined; +} + +/** Reads a token count that may be a plain number or a `{ total }` object (model-call usage). */ +function tokenCount(value: unknown): number | undefined { + return asNumber(value) ?? (isRecord(value) ? asNumber(value.total) : undefined); +} + +function buildOutputMessages( + parts: Array>, + finishReason: string | undefined, +): string | undefined { + if (!parts.length) { + return undefined; + } + return safeStringify([{ role: 'assistant', parts, finish_reason: normalizeFinishReason(finishReason) }]); +} + +function toolCallPart(toolCall: Record): Record { + const args = toolCall.input ?? toolCall.args; + return { + type: 'tool_call', + id: asString(toolCall.toolCallId), + name: asString(toolCall.toolName), + arguments: typeof args === 'string' ? args : safeStringify(args ?? {}), + }; +} + +function partsFromContent(content: unknown[]): Array> { + const parts: Array> = []; + for (const item of content) { + if (!isRecord(item)) { + continue; + } + if (item.type === 'text' && typeof item.text === 'string') { + parts.push({ type: 'text', content: item.text }); + } else if (item.type === 'tool-call') { + parts.push(toolCallPart(item)); + } + } + return parts; +} + +function partsFromTextAndToolCalls(text: unknown, toolCalls: unknown): Array> { + const parts: Array> = []; + if (typeof text === 'string' && text.length) { + parts.push({ type: 'text', content: text }); + } + if (Array.isArray(toolCalls)) { + for (const toolCall of toolCalls) { + if (isRecord(toolCall)) { + parts.push(toolCallPart(toolCall)); + } + } + } + return parts; +} + +function captureToolError(span: Span, data: VercelAiChannelMessage, error: unknown): void { + span.setStatus({ + code: SPAN_STATUS_ERROR, + message: error instanceof Error ? error.message : 'tool_error', + }); + + const toolCall = isRecord(data.event.toolCall) ? data.event.toolCall : {}; + const toolName = asString(toolCall.toolName); + const toolCallId = asString(data.event.toolCallId) ?? asString(toolCall.toolCallId); + + withScope(scope => { + scope.setContext('trace', spanToTraceContext(span)); + if (toolName) { + scope.setTag('vercel.ai.tool.name', toolName); + } + if (toolCallId) { + scope.setTag('vercel.ai.tool.callId', toolCallId); + } + scope.setLevel('error'); + captureException( + error instanceof Error ? error : new Error(typeof error === 'string' ? error : 'Tool execution failed'), + { + mechanism: { type: 'auto.vercelai.channel', handled: false }, + }, + ); + }); +} + +function getRecordingOptions(event: Record): { recordInputs: boolean; recordOutputs: boolean } { + const client = getClient(); + const genAI = client?.getDataCollectionOptions().genAI; + const options = client?.getIntegrationByName< + Integration & { options?: { recordInputs?: boolean; recordOutputs?: boolean } } + >(INTEGRATION_NAME)?.options; + + return { + recordInputs: resolveRecording(options?.recordInputs, event.recordInputs, genAI?.inputs), + recordOutputs: resolveRecording(options?.recordOutputs, event.recordOutputs, genAI?.outputs), + }; +} + +/** + * Mirrors the OTel integration's `determineRecordingSettings` precedence: an integration-level option + * wins, then the per-call `experimental_telemetry.recordInputs/recordOutputs` flag the AI SDK forwards + * on the channel event, then the global `dataCollection.genAI` default. + * + * NOTE: the OTel integration also defaults recording to `true` for a call with + * `experimental_telemetry: { isEnabled: true }`. The `ai:telemetry` channel does not expose `isEnabled` + * (nor a resolved recording flag), so that per-call default cannot be reproduced here — v7 users who + * want inputs/outputs recorded must enable `dataCollection.genAI` or set `recordInputs`/`recordOutputs`. + */ +function resolveRecording(integrationOption: unknown, perCallOption: unknown, globalDefault: unknown): boolean { + if (typeof integrationOption === 'boolean') { + return integrationOption; + } + if (typeof perCallOption === 'boolean') { + return perCallOption; + } + return globalDefault === true; +} + +function buildInputMessageAttributes(event: Record): Record { + // The AI SDK start events extend `StandardizedPrompt`; messages live on `messages`, otherwise the + // simpler `prompt` field is used. + const messages = event.messages ?? event.prompt; + if (messages === undefined) { + return {}; + } + return { [GEN_AI_INPUT_MESSAGES]: safeStringify(messages) }; +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function asNumber(value: unknown): number | undefined { + return typeof value === 'number' && !isNaN(value) ? value : undefined; +} + +function sum(a: number | undefined, b: number | undefined): number | undefined { + return a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function safeStringify(value: unknown): string { + if (typeof value === 'string') { + return value; + } + try { + return JSON.stringify(value); + } catch { + return '[unserializable]'; + } +}