From 8e3998fced2e92585b0a817c35e197ae0a3e8fef Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 17 Jun 2026 12:11:08 +0200 Subject: [PATCH 1/5] WIP WIP --- .../integrations/tracing/vercelai/index.ts | 10 + .../tracing/vercelai/channel.test.ts | 141 +++++++ packages/server-utils/src/index.ts | 12 + .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 387 ++++++++++++++++++ 4 files changed, 550 insertions(+) create mode 100644 packages/node/test/integrations/tracing/vercelai/channel.test.ts create mode 100644 packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts diff --git a/packages/node/src/integrations/tracing/vercelai/index.ts b/packages/node/src/integrations/tracing/vercelai/index.ts index a0b3f3126d01..d6eac78fa977 100644 --- a/packages/node/src/integrations/tracing/vercelai/index.ts +++ b/packages/node/src/integrations/tracing/vercelai/index.ts @@ -1,6 +1,8 @@ import type { Client, IntegrationFn } from '@sentry/core'; import { addVercelAiProcessors, defineIntegration } from '@sentry/core'; import { generateInstrumentOnce, type modulesIntegration } from '@sentry/node-core'; +import { tracingChannel as otelTracingChannel } from '@sentry/opentelemetry/tracing-channel'; +import { subscribeVercelAiTracingChannel } from '@sentry/server-utils'; import { INTEGRATION_NAME } from './constants'; import { SentryVercelAiInstrumentation } from './instrumentation'; import type { VercelAiOptions } from './types'; @@ -24,6 +26,14 @@ const _vercelAIIntegration = ((options: VercelAiOptions = {}) => { options, setupOnce() { instrumentation = instrumentVercelAi(); + + // Subscribe to the `ai` SDK's native telemetry tracing channel (ai >= 7). This emits + // fully-formed `gen_ai.*` spans directly and is a no-op on versions that don't publish to the + // channel, so it is always safe alongside the OTel-based instrumentation above. We pass + // `@sentry/opentelemetry/tracing-channel` as the factory so spans become the active OTel + // context via `bindStore` (giving correct parent/child nesting). That factory needs the Sentry + // OTel context manager, which `initOpenTelemetry()` registers after `setupOnce`, so defer a tick. + void Promise.resolve().then(() => subscribeVercelAiTracingChannel(otelTracingChannel)); }, afterAllSetup(client) { // Auto-detect if we should force the integration when running with 'ai' package available diff --git a/packages/node/test/integrations/tracing/vercelai/channel.test.ts b/packages/node/test/integrations/tracing/vercelai/channel.test.ts new file mode 100644 index 000000000000..82bf7def37c5 --- /dev/null +++ b/packages/node/test/integrations/tracing/vercelai/channel.test.ts @@ -0,0 +1,141 @@ +import { tracingChannel } from 'node:diagnostics_channel'; +import type { SpanJSON, TransactionEvent } from '@sentry/core'; +import { tracingChannel as otelTracingChannel } from '@sentry/opentelemetry/tracing-channel'; +import { subscribeVercelAiTracingChannel } from '@sentry/server-utils'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import * as Sentry from '../../../../src'; +import { cleanupOtel, mockSdkInit } from '../../../helpers/mockSdkInit'; + +const channel = tracingChannel, Record>('ai:telemetry'); + +const transactions: TransactionEvent[] = []; + +describe('Vercel AI telemetry tracing channel', () => { + beforeAll(() => { + mockSdkInit({ + tracesSampleRate: 1, + dataCollection: { genAI: { inputs: true, outputs: true } }, + beforeSendTransaction: event => { + transactions.push(event); + return null; + }, + }); + // The OTel context manager is registered by `mockSdkInit` above, so we can subscribe directly. + subscribeVercelAiTracingChannel(otelTracingChannel); + }); + + afterAll(() => { + cleanupOtel(); + }); + + beforeEach(() => { + transactions.length = 0; + }); + + it('builds a parented, fully-formed gen_ai span tree from the ai:telemetry channel', async () => { + await channel.tracePromise( + async () => { + await channel.tracePromise( + async () => ({ + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, + finishReason: 'stop', + text: 'Hi there', + response: { id: 'resp-1', modelId: 'gpt-4o-2024' }, + }), + { type: 'languageModelCall', event: { callId: 'call-1', provider: 'openai', modelId: 'gpt-4o' } }, + ); + + await channel.tracePromise(async () => ({ output: { temperature: 70 } }), { + type: 'executeTool', + event: { + callId: 'call-1', + toolCallId: 'tool-1', + toolCall: { toolName: 'getWeather', toolCallId: 'tool-1', input: { city: 'SF' } }, + }, + }); + + return { usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 }, finishReason: 'stop' }; + }, + { + type: 'generateText', + event: { + callId: 'call-1', + provider: 'openai', + modelId: 'gpt-4o', + functionId: 'my-agent', + messages: [{ role: 'user', content: 'Hello' }], + }, + }, + ); + + await Sentry.getClient()!.flush(); + + expect(transactions).toHaveLength(1); + const transaction = transactions[0]!; + + expect(transaction.transaction).toBe('invoke_agent my-agent'); + expect(transaction.contexts?.trace).toEqual( + expect.objectContaining({ + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.channel', + data: expect.objectContaining({ + 'gen_ai.operation.name': 'invoke_agent', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4o', + 'gen_ai.response.streaming': false, + 'gen_ai.function_id': 'my-agent', + 'gen_ai.input.messages': JSON.stringify([{ role: 'user', content: 'Hello' }]), + }), + }), + ); + + const rootSpanId = transaction.contexts?.trace?.span_id; + const spans = transaction.spans ?? []; + const chatSpan = spans.find((span: SpanJSON) => span.op === 'gen_ai.chat'); + const toolSpan = spans.find((span: SpanJSON) => span.op === 'gen_ai.execute_tool'); + + // Model call and tool call are siblings parented under the invoke_agent span. + expect(chatSpan?.parent_span_id).toBe(rootSpanId); + expect(toolSpan?.parent_span_id).toBe(rootSpanId); + + expect(chatSpan?.data).toEqual( + expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'gen_ai.request.model': 'gpt-4o', + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 20, + 'gen_ai.usage.total_tokens': 30, + 'gen_ai.response.finish_reasons': JSON.stringify(['stop']), + 'gen_ai.response.id': 'resp-1', + 'gen_ai.response.model': 'gpt-4o-2024', + 'gen_ai.response.text': 'Hi there', + }), + ); + + expect(toolSpan?.description).toBe('execute_tool getWeather'); + expect(toolSpan?.data).toEqual( + expect.objectContaining({ + 'gen_ai.operation.name': 'execute_tool', + 'gen_ai.tool.type': 'function', + 'gen_ai.tool.name': 'getWeather', + 'gen_ai.tool.call.id': 'tool-1', + 'gen_ai.tool.input': JSON.stringify({ city: 'SF' }), + 'gen_ai.tool.output': JSON.stringify({ temperature: 70 }), + }), + ); + }); + + it('marks the span as errored when the traced operation rejects', async () => { + await expect( + channel.tracePromise(async () => Promise.reject(new Error('boom')), { + type: 'generateText', + event: { callId: 'call-err', provider: 'openai', modelId: 'gpt-4o' }, + }), + ).rejects.toThrow('boom'); + + await Sentry.getClient()!.flush(); + + expect(transactions).toHaveLength(1); + expect(transactions[0]!.contexts?.trace?.status).toBe('internal_error'); + }); +}); diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index 43918e600a28..e9572920be10 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -23,3 +23,15 @@ export type { RedisTracingChannelFactory, RedisTracingChannelSubscribers, } from './redis/redis-dc-subscriber'; +export { + _resetVercelAiTracingChannelForTesting, + AI_SDK_TELEMETRY_TRACING_CHANNEL, + subscribeVercelAiTracingChannel, +} from './vercel-ai/vercel-ai-dc-subscriber'; +export type { + VercelAiChannelMessage, + VercelAiTracingChannel, + VercelAiTracingChannelContextWithSpan, + VercelAiTracingChannelFactory, + VercelAiTracingChannelSubscribers, +} from './vercel-ai/vercel-ai-dc-subscriber'; diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts new file mode 100644 index 000000000000..d49da66b08fe --- /dev/null +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -0,0 +1,387 @@ +import { + GEN_AI_EMBEDDINGS_INPUT, + GEN_AI_FUNCTION_ID, + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_REQUEST_AVAILABLE_TOOLS, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_FINISH_REASONS, + GEN_AI_RESPONSE_ID, + GEN_AI_RESPONSE_MODEL, + GEN_AI_RESPONSE_STREAMING, + GEN_AI_RESPONSE_TEXT, + GEN_AI_RESPONSE_TOOL_CALLS, + GEN_AI_SYSTEM, + GEN_AI_TOOL_INPUT, + GEN_AI_TOOL_NAME, + GEN_AI_TOOL_OUTPUT, + GEN_AI_TOOL_TYPE, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import { GEN_AI_CHAT_SPAN_OP, GEN_AI_EXECUTE_TOOL_SPAN_OP, GEN_AI_INVOKE_AGENT_SPAN_OP } from '@sentry/conventions/op'; +import type { Integration, Span } from '@sentry/core'; +import { + debug, + getActiveSpan, + getClient, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SPAN_STATUS_ERROR, + startInactiveSpan, +} from '@sentry/core'; +import { DEBUG_BUILD } from '../debug-build'; + +/** + * The single tracing channel the `ai` package (>= 7) publishes all telemetry lifecycle events to + * via `node:diagnostics_channel`. Events are discriminated by their `type` field. + * @see https://github.com/vercel/ai/pull/15660 + */ +export const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry'; + +const ORIGIN = 'auto.vercelai.channel'; +const INTEGRATION_NAME = 'VercelAI'; + +// `@sentry/conventions` does not expose these yet, so we keep the literals here. +const GEN_AI_TOOL_CALL_ID_ATTRIBUTE = 'gen_ai.tool.call.id'; +const GEN_AI_EMBEDDINGS_OPERATION = 'embeddings'; +const GEN_AI_RERANK_OPERATION = 'rerank'; + +const NOOP = (): void => {}; + +/** The lifecycle event types the `ai:telemetry` channel can carry. */ +type ChannelEventType = + | 'generateText' + | 'streamText' + | 'step' + | 'languageModelCall' + | 'executeTool' + | 'embed' + | 'rerank'; + +/** + * The context object the AI SDK passes through one tracing-channel call. It is the same object + * identity across `start`/`end`/`asyncEnd`/`error`, and Node's `tracingChannel` attaches + * `result`/`error` to it as the traced promise settles. + */ +export interface VercelAiChannelMessage { + type: ChannelEventType; + event: Record; + result?: unknown; + error?: unknown; +} + +/** + * Payload observed by subscribers — the channel context with the span stamped on by the factory's + * `start` transform, plus a marker for events we deliberately don't open a span for (`step`). + */ +export type VercelAiTracingChannelContextWithSpan = T & { _sentrySpan?: Span; _sentrySkip?: boolean }; + +/** Subscriber object accepted by {@link VercelAiTracingChannel.subscribe}. */ +export interface VercelAiTracingChannelSubscribers { + start: (data: VercelAiTracingChannelContextWithSpan) => void; + asyncStart: (data: VercelAiTracingChannelContextWithSpan) => void; + asyncEnd: (data: VercelAiTracingChannelContextWithSpan) => void; + end: (data: VercelAiTracingChannelContextWithSpan) => void; + error: (data: VercelAiTracingChannelContextWithSpan) => void; +} + +/** Minimal tracing-channel surface the subscriber depends on. */ +export interface VercelAiTracingChannel { + subscribe(subs: Partial>): void; +} + +/** + * Platform-provided factory that returns a tracing channel for the given channel name. The factory + * is responsible for, when `start` fires, calling `transformStart(data)` and storing the returned + * span on `data._sentrySpan` so the subscriber's `asyncEnd`/`error` handlers can read it. + * + * Node passes `@sentry/opentelemetry/tracing-channel`, which uses `bindStore` to additionally make + * the span the active OTel context for the duration of the traced operation. That is what makes + * nested AI SDK operations (model calls, tool calls) become children of the enclosing span without + * any manual parent bookkeeping here. + */ +export type VercelAiTracingChannelFactory = ( + name: string, + transformStart: (data: T) => Span, +) => VercelAiTracingChannel; + +let subscribed = false; + +/** + * Subscribe Sentry span handlers to the `ai` SDK's native telemetry tracing channel (`ai:telemetry`, + * available in `ai` >= 7) and emit fully-formed `gen_ai.*` spans directly — no OpenTelemetry span + * post-processing involved. + * + * Safe to always call: on `ai` versions that don't publish to the channel (e.g. < 7) nothing is + * ever emitted and this is inert, so there is no double-instrumentation against the OTel-based + * patcher. Idempotent. + */ +export function subscribeVercelAiTracingChannel(tracingChannel: VercelAiTracingChannelFactory): void { + if (subscribed) { + return; + } + subscribed = true; + + try { + const channel = tracingChannel(AI_SDK_TELEMETRY_TRACING_CHANNEL, createSpanFromMessage); + + channel.subscribe({ + start: NOOP, + asyncStart: NOOP, + // `end` fires synchronously before the traced promise settles, so it's too early to finish an + // async span. We finish on `asyncEnd` (success) / `error` instead. + end: NOOP, + asyncEnd: data => { + // `step` reuses the active span; the error handler ends on failure. + if (data._sentrySkip || data.error) { + return; + } + const span = data._sentrySpan; + if (!span) { + return; + } + enrichSpanOnEnd(span, data); + span.end(); + }, + error: data => { + if (data._sentrySkip) { + return; + } + const span = data._sentrySpan; + if (!span) { + return; + } + span.setStatus({ + code: SPAN_STATUS_ERROR, + message: data.error instanceof Error ? data.error.message : 'unknown_error', + }); + span.end(); + }, + }); + } catch { + // The factory relies on `node:diagnostics_channel`, which isn't always available. Fail closed. + DEBUG_BUILD && debug.log('Vercel AI node:diagnostics_channel subscription failed.'); + } +} + +/** + * Transform a channel `start` payload into the span that should be active for the operation. For + * `step` we deliberately don't open a span (model calls and tool calls are siblings under the + * invoke_agent span, matching the OTel-based output), so we reuse the active span and mark the + * payload to skip ending it. + */ +function createSpanFromMessage(data: VercelAiTracingChannelContextWithSpan): Span { + const { type, event } = data; + + if (type === 'step' || !event || typeof event !== 'object') { + data._sentrySkip = true; + // The active span is the enclosing invoke_agent span; reusing it leaves the tree unchanged. + return getActiveSpan() as Span; + } + + const { recordInputs } = getRecordingOptions(); + const provider = asString(event.provider); + const modelId = asString(event.modelId); + + const baseAttributes: Record = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, + ...(provider ? { [GEN_AI_SYSTEM]: provider } : {}), + ...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}), + }; + + switch (type) { + case 'generateText': + case 'streamText': { + const functionId = asString(event.functionId); + return startInactiveSpan({ + name: functionId ? `${GEN_AI_INVOKE_AGENT_SPAN_OP} ${functionId}` : GEN_AI_INVOKE_AGENT_SPAN_OP, + op: `gen_ai.${GEN_AI_INVOKE_AGENT_SPAN_OP}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_INVOKE_AGENT_SPAN_OP, + [GEN_AI_RESPONSE_STREAMING]: type === 'streamText', + ...(functionId ? { [GEN_AI_FUNCTION_ID]: functionId } : {}), + ...(recordInputs ? buildInputMessageAttributes(event) : {}), + }, + }); + } + + case 'languageModelCall': + return startInactiveSpan({ + name: modelId ? `${GEN_AI_CHAT_SPAN_OP} ${modelId}` : GEN_AI_CHAT_SPAN_OP, + op: `gen_ai.${GEN_AI_CHAT_SPAN_OP}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_CHAT_SPAN_OP, + ...(recordInputs ? buildInputMessageAttributes(event) : {}), + ...(recordInputs && Array.isArray(event.tools) + ? { [GEN_AI_REQUEST_AVAILABLE_TOOLS]: safeStringify(event.tools) } + : {}), + }, + }); + + case 'executeTool': { + const toolCall = isRecord(event.toolCall) ? event.toolCall : {}; + const toolName = asString(toolCall.toolName); + const toolCallId = asString(event.toolCallId) ?? asString(toolCall.toolCallId); + const toolInput = toolCall.input ?? toolCall.args; + return startInactiveSpan({ + name: toolName ? `${GEN_AI_EXECUTE_TOOL_SPAN_OP} ${toolName}` : GEN_AI_EXECUTE_TOOL_SPAN_OP, + op: `gen_ai.${GEN_AI_EXECUTE_TOOL_SPAN_OP}`, + attributes: { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, + [GEN_AI_OPERATION_NAME]: GEN_AI_EXECUTE_TOOL_SPAN_OP, + [GEN_AI_TOOL_TYPE]: 'function', + ...(toolName ? { [GEN_AI_TOOL_NAME]: toolName } : {}), + ...(toolCallId ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId } : {}), + ...(recordInputs && toolInput !== undefined ? { [GEN_AI_TOOL_INPUT]: safeStringify(toolInput) } : {}), + }, + }); + } + + case 'embed': + return startInactiveSpan({ + name: modelId ? `${GEN_AI_EMBEDDINGS_OPERATION} ${modelId}` : GEN_AI_EMBEDDINGS_OPERATION, + op: `gen_ai.${GEN_AI_EMBEDDINGS_OPERATION}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_EMBEDDINGS_OPERATION, + ...(recordInputs && event.value !== undefined + ? { [GEN_AI_EMBEDDINGS_INPUT]: safeStringify(event.value) } + : {}), + }, + }); + + case 'rerank': + return startInactiveSpan({ + name: modelId ? `${GEN_AI_RERANK_OPERATION} ${modelId}` : GEN_AI_RERANK_OPERATION, + op: `gen_ai.${GEN_AI_RERANK_OPERATION}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_RERANK_OPERATION, + }, + }); + + default: + data._sentrySkip = true; + return getActiveSpan() as Span; + } +} + +/** + * Best-effort enrichment from the resolved value the AI SDK attaches to the channel context. + * Everything here is guarded: when a field is missing or the shape differs across `ai` versions, + * we simply don't set the attribute rather than emit a malformed span. + */ +function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { + const { type, result } = data; + if (!isRecord(result)) { + return; + } + + const { recordOutputs } = getRecordingOptions(); + + if (type === 'executeTool') { + if (recordOutputs) { + span.setAttribute(GEN_AI_TOOL_OUTPUT, safeStringify(result.output ?? result)); + } + return; + } + + const usage = isRecord(result.usage) ? result.usage : undefined; + if (usage) { + const inputTokens = asNumber(usage.inputTokens) ?? asNumber(usage.tokens); + const outputTokens = asNumber(usage.outputTokens); + const totalTokens = asNumber(usage.totalTokens) ?? sum(inputTokens, outputTokens); + if (inputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_INPUT_TOKENS, inputTokens); + } + if (outputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens); + } + if (totalTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_TOTAL_TOKENS, totalTokens); + } + } + + const finishReason = asString(result.finishReason); + if (finishReason) { + span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, safeStringify([finishReason])); + } + + const response = isRecord(result.response) ? result.response : undefined; + const responseId = asString(response?.id) ?? asString(result.responseId); + if (responseId) { + span.setAttribute(GEN_AI_RESPONSE_ID, responseId); + } + const responseModel = asString(response?.modelId); + if (responseModel) { + span.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); + } + + if (recordOutputs) { + const text = asString(result.text); + if (text) { + span.setAttribute(GEN_AI_RESPONSE_TEXT, text); + } + if (Array.isArray(result.toolCalls) && result.toolCalls.length) { + span.setAttribute(GEN_AI_RESPONSE_TOOL_CALLS, safeStringify(result.toolCalls)); + } + } +} + +function getRecordingOptions(): { recordInputs: boolean; recordOutputs: boolean } { + const client = getClient(); + const genAI = client?.getDataCollectionOptions().genAI; + const options = client?.getIntegrationByName< + Integration & { options?: { recordInputs?: boolean; recordOutputs?: boolean } } + >(INTEGRATION_NAME)?.options; + + return { + recordInputs: options?.recordInputs ?? genAI?.inputs ?? false, + recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? false, + }; +} + +function buildInputMessageAttributes(event: Record): Record { + // The AI SDK start events extend `StandardizedPrompt`; messages live on `messages`, otherwise the + // simpler `prompt` field is used. + const messages = event.messages ?? event.prompt; + if (messages === undefined) { + return {}; + } + return { [GEN_AI_INPUT_MESSAGES]: safeStringify(messages) }; +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function asNumber(value: unknown): number | undefined { + return typeof value === 'number' && !isNaN(value) ? value : undefined; +} + +function sum(a: number | undefined, b: number | undefined): number | undefined { + return a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function safeStringify(value: unknown): string { + if (typeof value === 'string') { + return value; + } + try { + return JSON.stringify(value); + } catch { + return '[unserializable]'; + } +} + +/** Test-only: reset module-local subscribe state. */ +export function _resetVercelAiTracingChannelForTesting(): void { + subscribed = false; +} From 3b1ac572768e71751e3d44500f41f7410fd7a53f Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 17 Jun 2026 15:37:22 +0200 Subject: [PATCH 2/5] WIP --- .../vercelai/v7/instrument-with-pii.mjs | 12 + .../suites/tracing/vercelai/v7/instrument.mjs | 11 + .../vercelai/v7/scenario-error-in-tool.mjs | 41 +++ .../vercelai/v7/scenario-tool-loop-agent.mjs | 61 ++++ .../suites/tracing/vercelai/v7/scenario.mjs | 93 +++++ .../suites/tracing/vercelai/v7/test.ts | 323 +++++++++++++++++ .../tracing/vercelai/channel.test.ts | 12 +- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 328 +++++++++++++----- 8 files changed, 795 insertions(+), 86 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs new file mode 100644 index 000000000000..d15e81cf6d2b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs @@ -0,0 +1,12 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: { genAI: { inputs: true, outputs: true } }, + transport: loggingTransport, + integrations: [Sentry.vercelAIIntegration()], + streamGenAiSpans: true, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs new file mode 100644 index 000000000000..a76d206a0b61 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + integrations: [Sentry.vercelAIIntegration()], + streamGenAiSpans: true, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs new file mode 100644 index 000000000000..9ea18401ac35 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs @@ -0,0 +1,41 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + inputSchema: z.object({ location: z.string() }), + execute: async () => { + throw new Error('Error in tool'); + }, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs new file mode 100644 index 000000000000..6967ec2efe94 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs @@ -0,0 +1,61 @@ +import * as Sentry from '@sentry/node'; +import { ToolLoopAgent, stepCountIs, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + let callCount = 0; + + const agent = new ToolLoopAgent({ + experimental_telemetry: { isEnabled: true, functionId: 'weather_agent' }, + model: new MockLanguageModelV3({ + doGenerate: async () => { + if (callCount++ === 0) { + return { + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }; + } + return { + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [{ type: 'text', text: 'The weather in San Francisco is sunny, 72°F.' }], + warnings: [], + }; + }, + }), + tools: { + getWeather: tool({ + description: 'Get the current weather for a location', + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, + }), + }, + stopWhen: stepCountIs(3), + }); + + await agent.generate({ + prompt: 'What is the weather in San Francisco?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs new file mode 100644 index 000000000000..ee2dc802cd9c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs @@ -0,0 +1,93 @@ +import * as Sentry from '@sentry/node'; +import { generateText, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'First span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the first span?', + }); + + // This span should have input and output prompts attached because telemetry is explicitly enabled. + await generateText({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Second span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the second span?', + }); + + // This span should include tool calls and tool results + await generateText({ + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }), + }), + tools: { + getWeather: tool({ + description: 'Get the current weather for a location', + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, + }), + }, + prompt: 'What is the weather in San Francisco?', + }); + + // This span should not be captured because we've disabled telemetry + await generateText({ + experimental_telemetry: { isEnabled: false }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [{ type: 'text', text: 'Third span here!' }], + warnings: [], + }), + }), + prompt: 'Where is the third span?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts new file mode 100644 index 000000000000..a0d9d0460a6f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts @@ -0,0 +1,323 @@ +import type { Event } from '@sentry/node'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_SYSTEM_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +const vercelAiVersion = '7.0.0-beta.179'; + +describe('Vercel AI integration (V7)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with genAI recording disabled', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + // 3 generateText calls produce spans (the 4th call disables telemetry, so the channel + // emits nothing): each is invoke_agent + generate_content, plus one execute_tool. + expect(container.items).toHaveLength(7); + + const invokeAgentSpans = container.items.filter(span => span.name === 'invoke_agent'); + expect(invokeAgentSpans).toHaveLength(3); + for (const span of invokeAgentSpans) { + expect(span.status).toBe('ok'); + expect(span.attributes['sentry.origin'].value).toBe('auto.vercelai.channel'); + expect(span.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(span.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); + expect(span.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); + expect(span.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + expect(span.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + // Recording disabled: no input/output messages captured. + expect(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + expect(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + } + + const textInvokeAgent = invokeAgentSpans.find( + span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 10, + ); + expect(textInvokeAgent).toBeDefined(); + expect(textInvokeAgent!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); + expect(textInvokeAgent!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); + expect(textInvokeAgent!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["stop"]'); + + const toolInvokeAgent = invokeAgentSpans.find( + span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 15, + ); + expect(toolInvokeAgent).toBeDefined(); + expect(toolInvokeAgent!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe( + '["tool-calls"]', + ); + + const generateContentSpans = container.items.filter( + span => span.name === 'generate_content mock-model-id', + ); + expect(generateContentSpans).toHaveLength(3); + for (const span of generateContentSpans) { + expect(span.status).toBe('ok'); + expect(span.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(span.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); + expect(span.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); + expect(span.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + // Per-call usage is recorded on the model-call span. + expect(span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBeGreaterThan(0); + expect(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + } + + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); + expect(toolExecutionSpan).toBeDefined(); + expect(toolExecutionSpan!.status).toBe('ok'); + expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); + // Recording disabled: no tool input/output captured. + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeUndefined(); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates ai related spans with genAI recording enabled', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + + const firstInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the first span?"}]', + ); + expect(firstInvokeAgentSpan).toBeDefined(); + expect(firstInvokeAgentSpan!.status).toBe('ok'); + expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ); + + const firstGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('First span here!'), + ); + expect(firstGenerateContentSpan).toBeDefined(); + expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + + const toolInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ); + expect(toolInvokeAgentSpan).toBeDefined(); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('tool_call'); + + // The available tools are recorded on the model-call span for the tool generation. + const toolGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] !== undefined, + ); + expect(toolGenerateContentSpan).toBeDefined(); + expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toContain( + 'getWeather', + ); + expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); + expect(toolExecutionSpan).toBeDefined(); + expect(toolExecutionSpan!.status).toBe('ok'); + expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( + 'Get the current weather for a location', + ); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); + expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-error-in-tool.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('captures error in tool', async () => { + let transactionEvent: Event | undefined; + let errorEvent: Event | undefined; + + await createRunner() + .expect({ + transaction: transaction => { + transactionEvent = transaction; + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); + expect(generateContentSpan).toBeDefined(); + expect(generateContentSpan!.status).toBe('ok'); + expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); + expect(toolSpan).toBeDefined(); + // The failing tool surfaces as `tool-error` content; the integration marks the span errored. + expect(toolSpan!.status).toBe('error'); + expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + }, + }) + .expect({ + event: event => { + errorEvent = event; + }, + }) + .start() + .completed(); + + expect(transactionEvent).toBeDefined(); + expect(transactionEvent!.transaction).toBe('main'); + + expect(errorEvent).toBeDefined(); + expect(errorEvent!.level).toBe('error'); + expect(errorEvent!.tags).toEqual( + expect.objectContaining({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }), + ); + + // Trace id should be the same for the transaction and error event + expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-tool-loop-agent.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates spans for ToolLoopAgent with tool calls', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + // ToolLoopAgent loops: one invoke_agent (named after the functionId), two model calls + // (tool-call step + final step), and one tool execution. + expect(container.items).toHaveLength(4); + + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent'); + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan!.status).toBe('ok'); + expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); + // Usage is aggregated across both steps on the invoke_agent span (10+15, 20+25). + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(25); + expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(45); + + const generateContentSpans = container.items.filter( + span => span.name === 'generate_content mock-model-id', + ); + expect(generateContentSpans).toHaveLength(2); + for (const span of generateContentSpans) { + expect(span.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); + } + + // The first model call reports the tool-call finish reason, the final one stops. + const toolCallsGenerateContentSpan = generateContentSpans.find( + span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["tool-calls"]', + ); + expect(toolCallsGenerateContentSpan).toBeDefined(); + expect(toolCallsGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); + + const finalGenerateContentSpan = generateContentSpans.find( + span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["stop"]', + ); + expect(finalGenerateContentSpan).toBeDefined(); + expect(finalGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); + + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); + expect(toolSpan).toBeDefined(); + expect(toolSpan!.status).toBe('ok'); + expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); + expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); + expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); + expect(toolSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); +}); diff --git a/packages/node/test/integrations/tracing/vercelai/channel.test.ts b/packages/node/test/integrations/tracing/vercelai/channel.test.ts index 82bf7def37c5..27d28b2ec1de 100644 --- a/packages/node/test/integrations/tracing/vercelai/channel.test.ts +++ b/packages/node/test/integrations/tracing/vercelai/channel.test.ts @@ -91,16 +91,17 @@ describe('Vercel AI telemetry tracing channel', () => { const rootSpanId = transaction.contexts?.trace?.span_id; const spans = transaction.spans ?? []; - const chatSpan = spans.find((span: SpanJSON) => span.op === 'gen_ai.chat'); + const modelCallSpan = spans.find((span: SpanJSON) => span.op === 'gen_ai.generate_content'); const toolSpan = spans.find((span: SpanJSON) => span.op === 'gen_ai.execute_tool'); // Model call and tool call are siblings parented under the invoke_agent span. - expect(chatSpan?.parent_span_id).toBe(rootSpanId); + expect(modelCallSpan?.parent_span_id).toBe(rootSpanId); expect(toolSpan?.parent_span_id).toBe(rootSpanId); - expect(chatSpan?.data).toEqual( + expect(modelCallSpan?.description).toBe('generate_content gpt-4o'); + expect(modelCallSpan?.data).toEqual( expect.objectContaining({ - 'gen_ai.operation.name': 'chat', + 'gen_ai.operation.name': 'generate_content', 'gen_ai.request.model': 'gpt-4o', 'gen_ai.usage.input_tokens': 10, 'gen_ai.usage.output_tokens': 20, @@ -108,7 +109,8 @@ describe('Vercel AI telemetry tracing channel', () => { 'gen_ai.response.finish_reasons': JSON.stringify(['stop']), 'gen_ai.response.id': 'resp-1', 'gen_ai.response.model': 'gpt-4o-2024', - 'gen_ai.response.text': 'Hi there', + 'gen_ai.output.messages': + '[{"role":"assistant","parts":[{"type":"text","content":"Hi there"}],"finish_reason":"stop"}]', }), ); diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index d49da66b08fe..a4ba1a2d67b7 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -1,16 +1,16 @@ +/* eslint-disable max-lines */ import { GEN_AI_EMBEDDINGS_INPUT, GEN_AI_FUNCTION_ID, GEN_AI_INPUT_MESSAGES, GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, GEN_AI_REQUEST_AVAILABLE_TOOLS, GEN_AI_REQUEST_MODEL, GEN_AI_RESPONSE_FINISH_REASONS, GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_MODEL, GEN_AI_RESPONSE_STREAMING, - GEN_AI_RESPONSE_TEXT, - GEN_AI_RESPONSE_TOOL_CALLS, GEN_AI_SYSTEM, GEN_AI_TOOL_INPUT, GEN_AI_TOOL_NAME, @@ -20,15 +20,18 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, } from '@sentry/conventions/attributes'; -import { GEN_AI_CHAT_SPAN_OP, GEN_AI_EXECUTE_TOOL_SPAN_OP, GEN_AI_INVOKE_AGENT_SPAN_OP } from '@sentry/conventions/op'; +import { GEN_AI_EXECUTE_TOOL_SPAN_OP, GEN_AI_INVOKE_AGENT_SPAN_OP } from '@sentry/conventions/op'; import type { Integration, Span } from '@sentry/core'; import { + captureException, debug, getActiveSpan, getClient, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, SPAN_STATUS_ERROR, + spanToTraceContext, startInactiveSpan, + withScope, } from '@sentry/core'; import { DEBUG_BUILD } from '../debug-build'; @@ -46,6 +49,18 @@ const INTEGRATION_NAME = 'VercelAI'; const GEN_AI_TOOL_CALL_ID_ATTRIBUTE = 'gen_ai.tool.call.id'; const GEN_AI_EMBEDDINGS_OPERATION = 'embeddings'; const GEN_AI_RERANK_OPERATION = 'rerank'; +// The model-call op matches the Vercel AI OTel integration (`gen_ai.generate_content`) rather than +// the generic `gen_ai.chat`, so v6 (OTel) and v7 (channel) produce the same spans. +const GEN_AI_GENERATE_CONTENT_OPERATION = 'generate_content'; + +// Subset of the `vercel.ai.*` passthrough attributes the OTel integration emits that we reproduce. +const VERCEL_AI_OPERATION_ID_ATTRIBUTE = 'vercel.ai.operationId'; +const VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE = 'vercel.ai.model.provider'; +const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries'; + +// Tracks the top-level operationId per `callId` so a model-call span can name its `doGenerate`/ +// `doStream` operation the same way the OTel integration does. Cleared when the top-level span ends. +const operationIdByCallId = new Map(); const NOOP = (): void => {}; @@ -183,92 +198,126 @@ function createSpanFromMessage(data: VercelAiTracingChannelContextWithSpan = { [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, - ...(provider ? { [GEN_AI_SYSTEM]: provider } : {}), + ...(provider ? { [GEN_AI_SYSTEM]: provider, [VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider } : {}), ...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}), + ...(maxRetries !== undefined ? { [VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE]: maxRetries } : {}), }; switch (type) { case 'generateText': - case 'streamText': { - const functionId = asString(event.functionId); - return startInactiveSpan({ - name: functionId ? `${GEN_AI_INVOKE_AGENT_SPAN_OP} ${functionId}` : GEN_AI_INVOKE_AGENT_SPAN_OP, - op: `gen_ai.${GEN_AI_INVOKE_AGENT_SPAN_OP}`, - attributes: { - ...baseAttributes, - [GEN_AI_OPERATION_NAME]: GEN_AI_INVOKE_AGENT_SPAN_OP, - [GEN_AI_RESPONSE_STREAMING]: type === 'streamText', - ...(functionId ? { [GEN_AI_FUNCTION_ID]: functionId } : {}), - ...(recordInputs ? buildInputMessageAttributes(event) : {}), - }, - }); - } - + case 'streamText': + return buildInvokeAgentSpan(event, baseAttributes, recordInputs, callId, type === 'streamText'); case 'languageModelCall': - return startInactiveSpan({ - name: modelId ? `${GEN_AI_CHAT_SPAN_OP} ${modelId}` : GEN_AI_CHAT_SPAN_OP, - op: `gen_ai.${GEN_AI_CHAT_SPAN_OP}`, - attributes: { - ...baseAttributes, - [GEN_AI_OPERATION_NAME]: GEN_AI_CHAT_SPAN_OP, - ...(recordInputs ? buildInputMessageAttributes(event) : {}), - ...(recordInputs && Array.isArray(event.tools) - ? { [GEN_AI_REQUEST_AVAILABLE_TOOLS]: safeStringify(event.tools) } - : {}), - }, - }); - - case 'executeTool': { - const toolCall = isRecord(event.toolCall) ? event.toolCall : {}; - const toolName = asString(toolCall.toolName); - const toolCallId = asString(event.toolCallId) ?? asString(toolCall.toolCallId); - const toolInput = toolCall.input ?? toolCall.args; - return startInactiveSpan({ - name: toolName ? `${GEN_AI_EXECUTE_TOOL_SPAN_OP} ${toolName}` : GEN_AI_EXECUTE_TOOL_SPAN_OP, - op: `gen_ai.${GEN_AI_EXECUTE_TOOL_SPAN_OP}`, - attributes: { - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, - [GEN_AI_OPERATION_NAME]: GEN_AI_EXECUTE_TOOL_SPAN_OP, - [GEN_AI_TOOL_TYPE]: 'function', - ...(toolName ? { [GEN_AI_TOOL_NAME]: toolName } : {}), - ...(toolCallId ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId } : {}), - ...(recordInputs && toolInput !== undefined ? { [GEN_AI_TOOL_INPUT]: safeStringify(toolInput) } : {}), - }, - }); - } - + return buildModelCallSpan(event, baseAttributes, recordInputs, callId, modelId); + case 'executeTool': + return buildToolSpan(event, recordInputs); case 'embed': - return startInactiveSpan({ - name: modelId ? `${GEN_AI_EMBEDDINGS_OPERATION} ${modelId}` : GEN_AI_EMBEDDINGS_OPERATION, - op: `gen_ai.${GEN_AI_EMBEDDINGS_OPERATION}`, - attributes: { - ...baseAttributes, - [GEN_AI_OPERATION_NAME]: GEN_AI_EMBEDDINGS_OPERATION, - ...(recordInputs && event.value !== undefined - ? { [GEN_AI_EMBEDDINGS_INPUT]: safeStringify(event.value) } - : {}), - }, + return buildSimpleModelSpan(GEN_AI_EMBEDDINGS_OPERATION, baseAttributes, modelId, { + ...(recordInputs && event.value !== undefined ? { [GEN_AI_EMBEDDINGS_INPUT]: safeStringify(event.value) } : {}), }); - case 'rerank': - return startInactiveSpan({ - name: modelId ? `${GEN_AI_RERANK_OPERATION} ${modelId}` : GEN_AI_RERANK_OPERATION, - op: `gen_ai.${GEN_AI_RERANK_OPERATION}`, - attributes: { - ...baseAttributes, - [GEN_AI_OPERATION_NAME]: GEN_AI_RERANK_OPERATION, - }, - }); - + return buildSimpleModelSpan(GEN_AI_RERANK_OPERATION, baseAttributes, modelId, {}); default: data._sentrySkip = true; return getActiveSpan() as Span; } } +type Attributes = Record; + +function buildInvokeAgentSpan( + event: Record, + baseAttributes: Attributes, + recordInputs: boolean, + callId: string | undefined, + isStream: boolean, +): Span { + const functionId = asString(event.functionId); + const operationId = asString(event.operationId) ?? (isStream ? 'ai.streamText' : 'ai.generateText'); + if (callId) { + operationIdByCallId.set(callId, operationId); + } + return startInactiveSpan({ + name: functionId ? `${GEN_AI_INVOKE_AGENT_SPAN_OP} ${functionId}` : GEN_AI_INVOKE_AGENT_SPAN_OP, + op: `gen_ai.${GEN_AI_INVOKE_AGENT_SPAN_OP}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_INVOKE_AGENT_SPAN_OP, + [VERCEL_AI_OPERATION_ID_ATTRIBUTE]: operationId, + [GEN_AI_RESPONSE_STREAMING]: isStream, + ...(functionId ? { [GEN_AI_FUNCTION_ID]: functionId } : {}), + ...(recordInputs ? buildInputMessageAttributes(event) : {}), + }, + }); +} + +function buildModelCallSpan( + event: Record, + baseAttributes: Attributes, + recordInputs: boolean, + callId: string | undefined, + modelId: string | undefined, +): Span { + const parentOperationId = callId ? operationIdByCallId.get(callId) : undefined; + const operationId = parentOperationId + ? `${parentOperationId}.${parentOperationId.includes('stream') ? 'doStream' : 'doGenerate'}` + : 'ai.generateText.doGenerate'; + return startInactiveSpan({ + name: modelId ? `${GEN_AI_GENERATE_CONTENT_OPERATION} ${modelId}` : GEN_AI_GENERATE_CONTENT_OPERATION, + op: `gen_ai.${GEN_AI_GENERATE_CONTENT_OPERATION}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: GEN_AI_GENERATE_CONTENT_OPERATION, + [VERCEL_AI_OPERATION_ID_ATTRIBUTE]: operationId, + ...(recordInputs ? buildInputMessageAttributes(event) : {}), + ...(recordInputs && Array.isArray(event.tools) + ? { [GEN_AI_REQUEST_AVAILABLE_TOOLS]: safeStringify(event.tools) } + : {}), + }, + }); +} + +function buildToolSpan(event: Record, recordInputs: boolean): Span { + const toolCall = isRecord(event.toolCall) ? event.toolCall : {}; + const toolName = asString(toolCall.toolName); + const toolCallId = asString(event.toolCallId) ?? asString(toolCall.toolCallId); + const toolInput = toolCall.input ?? toolCall.args; + return startInactiveSpan({ + name: toolName ? `${GEN_AI_EXECUTE_TOOL_SPAN_OP} ${toolName}` : GEN_AI_EXECUTE_TOOL_SPAN_OP, + op: `gen_ai.${GEN_AI_EXECUTE_TOOL_SPAN_OP}`, + attributes: { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, + [GEN_AI_OPERATION_NAME]: GEN_AI_EXECUTE_TOOL_SPAN_OP, + [GEN_AI_TOOL_TYPE]: 'function', + ...(toolName ? { [GEN_AI_TOOL_NAME]: toolName } : {}), + ...(toolCallId ? { [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: toolCallId } : {}), + ...(recordInputs && toolInput !== undefined ? { [GEN_AI_TOOL_INPUT]: safeStringify(toolInput) } : {}), + }, + }); +} + +function buildSimpleModelSpan( + operation: string, + baseAttributes: Attributes, + modelId: string | undefined, + extraAttributes: Attributes, +): Span { + return startInactiveSpan({ + name: modelId ? `${operation} ${modelId}` : operation, + op: `gen_ai.${operation}`, + attributes: { + ...baseAttributes, + [GEN_AI_OPERATION_NAME]: operation, + ...extraAttributes, + }, + }); +} + /** * Best-effort enrichment from the resolved value the AI SDK attaches to the channel context. * Everything here is guarded: when a field is missing or the shape differs across `ai` versions, @@ -286,14 +335,31 @@ function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { if (recordOutputs) { span.setAttribute(GEN_AI_TOOL_OUTPUT, safeStringify(result.output ?? result)); } + // From V5 on, tool errors are not rejected (so the `error` channel verb never fires) — they + // surface as `tool-error` content on the resolved result. Mirror the OTel path by marking the + // span and capturing the error. + const output = isRecord(result.output) ? result.output : undefined; + if (output?.type === 'tool-error') { + captureToolError(span, data, output.error); + } return; } + // Top-level call finished — drop its operationId mapping. + if (type !== 'languageModelCall') { + const callId = asString(data.event.callId); + if (callId) { + operationIdByCallId.delete(callId); + } + } + + // `languageModelCall` results report usage as `{ total }` objects; top-level/step results report + // flat numbers. `tokenCount` handles both. const usage = isRecord(result.usage) ? result.usage : undefined; if (usage) { - const inputTokens = asNumber(usage.inputTokens) ?? asNumber(usage.tokens); - const outputTokens = asNumber(usage.outputTokens); - const totalTokens = asNumber(usage.totalTokens) ?? sum(inputTokens, outputTokens); + const inputTokens = tokenCount(usage.inputTokens) ?? tokenCount(usage.tokens); + const outputTokens = tokenCount(usage.outputTokens); + const totalTokens = tokenCount(usage.totalTokens) ?? sum(inputTokens, outputTokens); if (inputTokens !== undefined) { span.setAttribute(GEN_AI_USAGE_INPUT_TOKENS, inputTokens); } @@ -305,7 +371,7 @@ function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { } } - const finishReason = asString(result.finishReason); + const finishReason = getFinishReason(result); if (finishReason) { span.setAttribute(GEN_AI_RESPONSE_FINISH_REASONS, safeStringify([finishReason])); } @@ -315,20 +381,120 @@ function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { if (responseId) { span.setAttribute(GEN_AI_RESPONSE_ID, responseId); } - const responseModel = asString(response?.modelId); + const responseModel = asString(response?.modelId) ?? asString(data.event.modelId); if (responseModel) { span.setAttribute(GEN_AI_RESPONSE_MODEL, responseModel); } if (recordOutputs) { - const text = asString(result.text); - if (text) { - span.setAttribute(GEN_AI_RESPONSE_TEXT, text); + // `languageModelCall` exposes the response as a `content` parts array; top-level results expose + // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. + const parts = + type === 'languageModelCall' && Array.isArray(result.content) + ? partsFromContent(result.content) + : partsFromTextAndToolCalls(result.text, result.toolCalls); + const outputMessages = buildOutputMessages(parts, finishReason); + if (outputMessages) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, outputMessages); + } + } +} + +/** Maps a Vercel AI finish reason to the OTel `gen_ai.output.messages` form (`tool-calls` → `tool_call`). */ +function normalizeFinishReason(finishReason: string | undefined): string { + return finishReason === 'tool-calls' ? 'tool_call' : (finishReason ?? 'stop'); +} + +/** Reads the finish reason from a result — a string on top-level results, `{ unified }` on model calls. */ +function getFinishReason(result: Record): string | undefined { + const finishReason = result.finishReason; + if (typeof finishReason === 'string') { + return finishReason; + } + return isRecord(finishReason) ? asString(finishReason.unified) : undefined; +} + +/** Reads a token count that may be a plain number or a `{ total }` object (model-call usage). */ +function tokenCount(value: unknown): number | undefined { + return asNumber(value) ?? (isRecord(value) ? asNumber(value.total) : undefined); +} + +function buildOutputMessages( + parts: Array>, + finishReason: string | undefined, +): string | undefined { + if (!parts.length) { + return undefined; + } + return safeStringify([{ role: 'assistant', parts, finish_reason: normalizeFinishReason(finishReason) }]); +} + +function toolCallPart(toolCall: Record): Record { + const args = toolCall.input ?? toolCall.args; + return { + type: 'tool_call', + id: asString(toolCall.toolCallId), + name: asString(toolCall.toolName), + arguments: typeof args === 'string' ? args : safeStringify(args ?? {}), + }; +} + +function partsFromContent(content: unknown[]): Array> { + const parts: Array> = []; + for (const item of content) { + if (!isRecord(item)) { + continue; } - if (Array.isArray(result.toolCalls) && result.toolCalls.length) { - span.setAttribute(GEN_AI_RESPONSE_TOOL_CALLS, safeStringify(result.toolCalls)); + if (item.type === 'text' && typeof item.text === 'string') { + parts.push({ type: 'text', content: item.text }); + } else if (item.type === 'tool-call') { + parts.push(toolCallPart(item)); + } + } + return parts; +} + +function partsFromTextAndToolCalls(text: unknown, toolCalls: unknown): Array> { + const parts: Array> = []; + if (typeof text === 'string' && text.length) { + parts.push({ type: 'text', content: text }); + } + if (Array.isArray(toolCalls)) { + for (const toolCall of toolCalls) { + if (isRecord(toolCall)) { + parts.push(toolCallPart(toolCall)); + } } } + return parts; +} + +function captureToolError(span: Span, data: VercelAiChannelMessage, error: unknown): void { + span.setStatus({ + code: SPAN_STATUS_ERROR, + message: error instanceof Error ? error.message : 'tool_error', + }); + + const toolCall = isRecord(data.event.toolCall) ? data.event.toolCall : {}; + const toolName = asString(toolCall.toolName); + const toolCallId = asString(data.event.toolCallId) ?? asString(toolCall.toolCallId); + + withScope(scope => { + scope.setContext('trace', spanToTraceContext(span)); + if (toolName) { + scope.setTag('vercel.ai.tool.name', toolName); + } + if (toolCallId) { + scope.setTag('vercel.ai.tool.callId', toolCallId); + } + scope.setLevel('error'); + captureException( + error instanceof Error ? error : new Error(typeof error === 'string' ? error : 'Tool execution failed'), + { + mechanism: { type: 'auto.vercelai.channel', handled: false }, + }, + ); + }); } function getRecordingOptions(): { recordInputs: boolean; recordOutputs: boolean } { From a9d0f0746dffe9d71dbd1c817b0432ddb76e14b6 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 17 Jun 2026 16:03:20 +0200 Subject: [PATCH 3/5] unify test --- .../suites/tracing/vercelai/v6/test.ts | 421 ----------------- .../{v6 => v6_v7}/instrument-with-pii.mjs | 0 .../vercelai/{v6 => v6_v7}/instrument.mjs | 0 .../{v6 => v6_v7}/scenario-error-in-tool.mjs | 0 .../scenario-tool-loop-agent.mjs | 0 .../vercelai/{v6 => v6_v7}/scenario.mjs | 6 +- .../suites/tracing/vercelai/v6_v7/test.ts | 430 ++++++++++++++++++ .../vercelai/v7/instrument-with-pii.mjs | 12 - .../suites/tracing/vercelai/v7/instrument.mjs | 11 - .../vercelai/v7/scenario-error-in-tool.mjs | 41 -- .../vercelai/v7/scenario-tool-loop-agent.mjs | 61 --- .../suites/tracing/vercelai/v7/scenario.mjs | 93 ---- .../suites/tracing/vercelai/v7/test.ts | 323 ------------- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 20 +- 14 files changed, 448 insertions(+), 970 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts rename dev-packages/node-integration-tests/suites/tracing/vercelai/{v6 => v6_v7}/instrument-with-pii.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{v6 => v6_v7}/instrument.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{v6 => v6_v7}/scenario-error-in-tool.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{v6 => v6_v7}/scenario-tool-loop-agent.mjs (100%) rename dev-packages/node-integration-tests/suites/tracing/vercelai/{v6 => v6_v7}/scenario.mjs (90%) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts deleted file mode 100644 index 7f008db7e654..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts +++ /dev/null @@ -1,421 +0,0 @@ -import type { Event } from '@sentry/node'; -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES_ATTRIBUTE, - GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, - GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, - GEN_AI_REQUEST_MODEL_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, - GEN_AI_RESPONSE_MODEL_ATTRIBUTE, - GEN_AI_SYSTEM_ATTRIBUTE, - GEN_AI_TOOL_CALL_ID_ATTRIBUTE, - GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, - GEN_AI_TOOL_INPUT_ATTRIBUTE, - GEN_AI_TOOL_NAME_ATTRIBUTE, - GEN_AI_TOOL_OUTPUT_ATTRIBUTE, - GEN_AI_TOOL_TYPE_ATTRIBUTE, - GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, - GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, - GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, -} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -describe('Vercel AI integration (V6)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans with genAI recording disabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes['vercel.ai.operationId'].value === 'ai.generateText' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && - span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 10, - ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - - const firstGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes['vercel.ai.operationId'].value === 'ai.generateText.doGenerate' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && - span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 10, - ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - - const secondInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the second span?"}]', - ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"user","content":"Where is the second span?"}]', - ); - expect(secondInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', - ); - - const secondGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), - ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, - ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(toolInvokeAgentSpan!.status).toBe('ok'); - - const toolGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, - ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(toolGenerateContentSpan!.status).toBe('ok'); - - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); - expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('creates ai related spans with genAI recording enabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - const firstInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the first span?"}]', - ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"user","content":"Where is the first span?"}]', - ); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ); - - const firstGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('First span here!'), - ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(firstGenerateContentSpan!.status).toBe('ok'); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes['vercel.ai.operationId'].value).toBe( - 'ai.generateText.doGenerate', - ); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); - expect(firstGenerateContentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain( - 'First span here!', - ); - - const secondInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the second span?"}]', - ); - expect(secondInvokeAgentSpan).toBeDefined(); - expect(secondInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(secondInvokeAgentSpan!.status).toBe('ok'); - expect(secondInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const secondGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('Second span here!'), - ); - expect(secondGenerateContentSpan).toBeDefined(); - expect(secondGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(secondGenerateContentSpan!.status).toBe('ok'); - expect(secondGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.name).toBe('invoke_agent'); - expect(toolInvokeAgentSpan!.status).toBe('ok'); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - - const toolGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] !== undefined, - ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(toolGenerateContentSpan!.status).toBe('ok'); - expect(toolGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeDefined(); - expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.name).toBe('execute_tool getWeather'); - expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( - 'Get the current weather for a location', - ); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-error-in-tool.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('captures error in tool', async () => { - let transactionEvent: Event | undefined; - let errorEvent: Event | undefined; - - await createRunner() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); - expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - }, - }) - .expect({ - event: event => { - errorEvent = event; - }, - }) - .start() - .completed(); - - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); - - expect(errorEvent).toBeDefined(); - expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); - - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans with v6', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - const invokeAgentSpans = container.items.filter( - span => span.attributes['sentry.op'].value === 'gen_ai.invoke_agent', - ); - expect(invokeAgentSpans).toHaveLength(3); - - const generateContentSpans = container.items.filter( - span => span.attributes['sentry.op'].value === 'gen_ai.generate_content', - ); - expect(generateContentSpans).toHaveLength(3); - - const toolSpan = container.items.find( - span => span.attributes['sentry.op'].value === 'gen_ai.execute_tool', - ); - expect(toolSpan).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-tool-loop-agent.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates spans for ToolLoopAgent with tool calls', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(4); - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.name).toBe('invoke_agent weather_agent'); - expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - - const toolCallsGenerateContentSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["tool-calls"]', - ); - expect(toolCallsGenerateContentSpan).toBeDefined(); - expect(toolCallsGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(toolCallsGenerateContentSpan!.status).toBe('ok'); - expect(toolCallsGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(toolCallsGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - expect(toolCallsGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.name).toBe('execute_tool getWeather'); - expect(toolSpan!.status).toBe('ok'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); - expect(toolSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); - - const finalGenerateContentSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["stop"]', - ); - expect(finalGenerateContentSpan).toBeDefined(); - expect(finalGenerateContentSpan!.name).toBe('generate_content mock-model-id'); - expect(finalGenerateContentSpan!.status).toBe('ok'); - expect(finalGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(finalGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - expect(finalGenerateContentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(25); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: '^6.0.0', - }, - }, - ); -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument-with-pii.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-with-pii.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/instrument.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-error-in-tool.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-error-in-tool.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-error-in-tool.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-tool-loop-agent.mjs similarity index 100% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-tool-loop-agent.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs similarity index 90% rename from dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs rename to dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs index ee2dc802cd9c..5c7ca973cd27 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario.mjs @@ -21,9 +21,11 @@ async function run() { prompt: 'Where is the first span?', }); - // This span should have input and output prompts attached because telemetry is explicitly enabled. + // This span should have input and output prompts attached because recording is explicitly enabled. + // (v6 enables recording implicitly from `isEnabled: true`; v7's diagnostics channel only carries + // the explicit `recordInputs`/`recordOutputs` flags, so we set them here to exercise both.) await generateText({ - experimental_telemetry: { isEnabled: true }, + experimental_telemetry: { isEnabled: true, recordInputs: true, recordOutputs: true }, model: new MockLanguageModelV3({ doGenerate: async () => ({ finishReason: { unified: 'stop', raw: 'stop' }, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts new file mode 100644 index 000000000000..8afd4b1f8559 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -0,0 +1,430 @@ +import type { Event } from '@sentry/node'; +import { afterAll, describe, expect } from 'vitest'; +import { + GEN_AI_INPUT_MESSAGES_ATTRIBUTE, + GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_SYSTEM_ATTRIBUTE, + GEN_AI_TOOL_CALL_ID_ATTRIBUTE, + GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, + GEN_AI_TOOL_INPUT_ATTRIBUTE, + GEN_AI_TOOL_NAME_ATTRIBUTE, + GEN_AI_TOOL_OUTPUT_ATTRIBUTE, + GEN_AI_TOOL_TYPE_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +describe.each([ + ['6', '^6.0.0'], + ['7', '7.0.0-beta.179'], +])('Vercel AI integration (version %s)', (_, vercelAiVersion) => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with genAI recording disabled', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const firstInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.['vercel.ai.operationId']?.value === 'ai.generateText' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && + span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 10, + )!; + expect(firstInvokeAgentSpan).toBeDefined(); + expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan.status).toBe('ok'); + expect(firstInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes?.['vercel.ai.operationId']?.value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_REQUEST_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(10); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(20); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]?.value).toBe(30); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + const firstGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes?.['vercel.ai.operationId']?.value === 'ai.generateText.doGenerate' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE] === undefined && + span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 10, + )!; + expect(firstGenerateContentSpan).toBeDefined(); + expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan.status).toBe('ok'); + expect(firstGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes?.['vercel.ai.operationId']?.value).toBe( + 'ai.generateText.doGenerate', + ); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_SYSTEM_ATTRIBUTE]?.value).toBe('mock-provider'); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(10); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); + + const secondInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the second span?"}]', + )!; + expect(secondInvokeAgentSpan).toBeDefined(); + expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan.status).toBe('ok'); + expect(secondInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(secondInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"user","content":"Where is the second span?"}]', + )!; + expect(secondInvokeAgentSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"Second span here!"}],"finish_reason":"stop"}]', + ); + + const secondGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + (span.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value as string | undefined)?.includes( + 'Second span here!', + ), + )!; + expect(secondGenerateContentSpan).toBeDefined(); + expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(secondGenerateContentSpan.status).toBe('ok'); + expect(secondGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + + const toolInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, + )!; + expect(toolInvokeAgentSpan).toBeDefined(); + expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan.status).toBe('ok'); + + const toolGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value === 15, + )!; + expect(toolGenerateContentSpan).toBeDefined(); + expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan.status).toBe('ok'); + + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan).toBeDefined(); + expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan.status).toBe('ok'); + expect(toolExecutionSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_TYPE_ATTRIBUTE]?.value).toBe('function'); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates ai related spans with genAI recording enabled', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const firstInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the first span?"}]', + )!; + expect(firstInvokeAgentSpan).toBeDefined(); + expect(firstInvokeAgentSpan.name).toBe('invoke_agent'); + expect(firstInvokeAgentSpan.status).toBe('ok'); + expect(firstInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(firstInvokeAgentSpan.attributes?.['vercel.ai.operationId']?.value).toBe('ai.generateText'); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"user","content":"Where is the first span?"}]', + ); + expect(firstInvokeAgentSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', + ); + + const firstGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + (span.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value as string | undefined)?.includes( + 'First span here!', + ), + )!; + expect(firstGenerateContentSpan).toBeDefined(); + expect(firstGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(firstGenerateContentSpan.status).toBe('ok'); + expect(firstGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(firstGenerateContentSpan.attributes?.['vercel.ai.operationId']?.value).toBe( + 'ai.generateText.doGenerate', + ); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); + expect(firstGenerateContentSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value).toContain( + 'First span here!', + ); + + const secondInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"Where is the second span?"}]', + )!; + expect(secondInvokeAgentSpan).toBeDefined(); + expect(secondInvokeAgentSpan.name).toBe('invoke_agent'); + expect(secondInvokeAgentSpan.status).toBe('ok'); + expect(secondInvokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + + const secondGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + (span.attributes?.[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value as string | undefined)?.includes( + 'Second span here!', + ), + )!; + expect(secondGenerateContentSpan).toBeDefined(); + expect(secondGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(secondGenerateContentSpan.status).toBe('ok'); + expect(secondGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + + const toolInvokeAgentSpan = container.items.find( + span => + span.name === 'invoke_agent' && + span.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + )!; + expect(toolInvokeAgentSpan).toBeDefined(); + expect(toolInvokeAgentSpan.name).toBe('invoke_agent'); + expect(toolInvokeAgentSpan.status).toBe('ok'); + expect(toolInvokeAgentSpan.attributes?.[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value).toBe( + '[{"role":"user","content":"What is the weather in San Francisco?"}]', + ); + + const toolGenerateContentSpan = container.items.find( + span => + span.name === 'generate_content mock-model-id' && + span.attributes?.[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] !== undefined, + )!; + expect(toolGenerateContentSpan).toBeDefined(); + expect(toolGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolGenerateContentSpan.status).toBe('ok'); + expect(toolGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolGenerateContentSpan.attributes?.[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]).toBeDefined(); + expect(toolGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(15); + + const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolExecutionSpan).toBeDefined(); + expect(toolExecutionSpan.name).toBe('execute_tool getWeather'); + expect(toolExecutionSpan.status).toBe('ok'); + expect(toolExecutionSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE]?.value).toBe( + 'Get the current weather for a location', + ); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); + expect(toolExecutionSpan.attributes?.[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-error-in-tool.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('captures error in tool', async () => { + let transactionEvent: Event | undefined; + let errorEvent: Event | undefined; + + await createRunner() + .expect({ + transaction: transaction => { + transactionEvent = transaction; + }, + }) + .expect({ + span: container => { + expect(container.items).toHaveLength(3); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent')!; + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan.name).toBe('invoke_agent'); + expect(invokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + + const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id')!; + expect(generateContentSpan).toBeDefined(); + expect(generateContentSpan.name).toBe('generate_content mock-model-id'); + expect(generateContentSpan.status).toBe('ok'); + expect(generateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan).toBeDefined(); + expect(toolSpan.name).toBe('execute_tool getWeather'); + expect(toolSpan.status).toBe('error'); + expect(toolSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + }, + }) + .expect({ + event: event => { + errorEvent = event; + }, + }) + .start() + .completed(); + + expect(transactionEvent).toBeDefined(); + expect(transactionEvent!.transaction).toBe('main'); + + expect(errorEvent).toBeDefined(); + expect(errorEvent!.level).toBe('error'); + expect(errorEvent!.tags).toEqual( + expect.objectContaining({ + 'vercel.ai.tool.name': 'getWeather', + 'vercel.ai.tool.callId': 'call-1', + }), + ); + + // Trace id should be the same for the transaction and error event + expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates ai related spans with v6', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(7); + const invokeAgentSpans = container.items.filter( + span => span.attributes?.['sentry.op']?.value === 'gen_ai.invoke_agent', + ); + expect(invokeAgentSpans).toHaveLength(3); + + const generateContentSpans = container.items.filter( + span => span.attributes?.['sentry.op']?.value === 'gen_ai.generate_content', + ); + expect(generateContentSpans).toHaveLength(3); + + const toolSpan = container.items.find( + span => span.attributes?.['sentry.op']?.value === 'gen_ai.execute_tool', + ); + expect(toolSpan).toBeDefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-tool-loop-agent.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates spans for ToolLoopAgent with tool calls', async () => { + await createRunner() + .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + expect(container.items).toHaveLength(4); + const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent')!; + expect(invokeAgentSpan).toBeDefined(); + expect(invokeAgentSpan.name).toBe('invoke_agent weather_agent'); + expect(invokeAgentSpan.status).toBe('ok'); + expect(invokeAgentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.invoke_agent'); + expect(invokeAgentSpan.attributes?.[GEN_AI_REQUEST_MODEL_ATTRIBUTE]?.value).toBe('mock-model-id'); + + const toolCallsGenerateContentSpan = container.items.find( + span => span.attributes?.[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["tool-calls"]', + )!; + expect(toolCallsGenerateContentSpan).toBeDefined(); + expect(toolCallsGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(toolCallsGenerateContentSpan.status).toBe('ok'); + expect(toolCallsGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(toolCallsGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(10); + expect(toolCallsGenerateContentSpan.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(20); + + const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather')!; + expect(toolSpan).toBeDefined(); + expect(toolSpan.name).toBe('execute_tool getWeather'); + expect(toolSpan.status).toBe('ok'); + expect(toolSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.execute_tool'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_NAME_ATTRIBUTE]?.value).toBe('getWeather'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_CALL_ID_ATTRIBUTE]?.value).toBe('call-1'); + expect(toolSpan.attributes?.[GEN_AI_TOOL_TYPE_ATTRIBUTE]?.value).toBe('function'); + + const finalGenerateContentSpan = container.items.find( + span => span.attributes?.[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["stop"]', + )!; + expect(finalGenerateContentSpan).toBeDefined(); + expect(finalGenerateContentSpan.name).toBe('generate_content mock-model-id'); + expect(finalGenerateContentSpan.status).toBe('ok'); + expect(finalGenerateContentSpan.attributes?.['sentry.op']?.value).toBe('gen_ai.generate_content'); + expect(finalGenerateContentSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]?.value).toBe(15); + expect(finalGenerateContentSpan.attributes?.[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]?.value).toBe(25); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: vercelAiVersion, + }, + }, + ); +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs deleted file mode 100644 index d15e81cf6d2b..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument-with-pii.mjs +++ /dev/null @@ -1,12 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - integrations: [Sentry.vercelAIIntegration()], - streamGenAiSpans: true, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs deleted file mode 100644 index a76d206a0b61..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/instrument.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - transport: loggingTransport, - integrations: [Sentry.vercelAIIntegration()], - streamGenAiSpans: true, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs deleted file mode 100644 index 9ea18401ac35..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-error-in-tool.mjs +++ /dev/null @@ -1,41 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - inputSchema: z.object({ location: z.string() }), - execute: async () => { - throw new Error('Error in tool'); - }, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - }); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs deleted file mode 100644 index 6967ec2efe94..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario-tool-loop-agent.mjs +++ /dev/null @@ -1,61 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { ToolLoopAgent, stepCountIs, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - let callCount = 0; - - const agent = new ToolLoopAgent({ - experimental_telemetry: { isEnabled: true, functionId: 'weather_agent' }, - model: new MockLanguageModelV3({ - doGenerate: async () => { - if (callCount++ === 0) { - return { - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }; - } - return { - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [{ type: 'text', text: 'The weather in San Francisco is sunny, 72°F.' }], - warnings: [], - }; - }, - }), - tools: { - getWeather: tool({ - description: 'Get the current weather for a location', - inputSchema: z.object({ location: z.string() }), - execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, - }), - }, - stopWhen: stepCountIs(3), - }); - - await agent.generate({ - prompt: 'What is the weather in San Francisco?', - }); - }); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs deleted file mode 100644 index ee2dc802cd9c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/scenario.mjs +++ /dev/null @@ -1,93 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { generateText, tool } from 'ai'; -import { MockLanguageModelV3 } from 'ai/test'; -import { z } from 'zod'; - -async function run() { - await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'First span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the first span?', - }); - - // This span should have input and output prompts attached because telemetry is explicitly enabled. - await generateText({ - experimental_telemetry: { isEnabled: true }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Second span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the second span?', - }); - - // This span should include tool calls and tool results - await generateText({ - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, - usage: { - inputTokens: { total: 15, noCache: 15, cached: 0 }, - outputTokens: { total: 25, noCache: 25, cached: 0 }, - totalTokens: { total: 40, noCache: 40, cached: 0 }, - }, - content: [ - { - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'getWeather', - input: JSON.stringify({ location: 'San Francisco' }), - }, - ], - warnings: [], - }), - }), - tools: { - getWeather: tool({ - description: 'Get the current weather for a location', - inputSchema: z.object({ location: z.string() }), - execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, - }), - }, - prompt: 'What is the weather in San Francisco?', - }); - - // This span should not be captured because we've disabled telemetry - await generateText({ - experimental_telemetry: { isEnabled: false }, - model: new MockLanguageModelV3({ - doGenerate: async () => ({ - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { - inputTokens: { total: 10, noCache: 10, cached: 0 }, - outputTokens: { total: 20, noCache: 20, cached: 0 }, - totalTokens: { total: 30, noCache: 30, cached: 0 }, - }, - content: [{ type: 'text', text: 'Third span here!' }], - warnings: [], - }), - }), - prompt: 'Where is the third span?', - }); - }); -} - -run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts deleted file mode 100644 index a0d9d0460a6f..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v7/test.ts +++ /dev/null @@ -1,323 +0,0 @@ -import type { Event } from '@sentry/node'; -import { afterAll, describe, expect } from 'vitest'; -import { - GEN_AI_INPUT_MESSAGES_ATTRIBUTE, - GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE, - GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE, - GEN_AI_REQUEST_MODEL_ATTRIBUTE, - GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, - GEN_AI_RESPONSE_MODEL_ATTRIBUTE, - GEN_AI_SYSTEM_ATTRIBUTE, - GEN_AI_TOOL_CALL_ID_ATTRIBUTE, - GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE, - GEN_AI_TOOL_INPUT_ATTRIBUTE, - GEN_AI_TOOL_NAME_ATTRIBUTE, - GEN_AI_TOOL_OUTPUT_ATTRIBUTE, - GEN_AI_TOOL_TYPE_ATTRIBUTE, - GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, - GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, - GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, -} from '../../../../../../packages/core/src/tracing/ai/gen-ai-attributes'; -import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; - -const vercelAiVersion = '7.0.0-beta.179'; - -describe('Vercel AI integration (V7)', () => { - afterAll(() => { - cleanupChildProcesses(); - }); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates ai related spans with genAI recording disabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - // 3 generateText calls produce spans (the 4th call disables telemetry, so the channel - // emits nothing): each is invoke_agent + generate_content, plus one execute_tool. - expect(container.items).toHaveLength(7); - - const invokeAgentSpans = container.items.filter(span => span.name === 'invoke_agent'); - expect(invokeAgentSpans).toHaveLength(3); - for (const span of invokeAgentSpans) { - expect(span.status).toBe('ok'); - expect(span.attributes['sentry.origin'].value).toBe('auto.vercelai.channel'); - expect(span.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(span.attributes['vercel.ai.operationId'].value).toBe('ai.generateText'); - expect(span.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); - expect(span.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - expect(span.attributes[GEN_AI_RESPONSE_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - // Recording disabled: no input/output messages captured. - expect(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - expect(span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - } - - const textInvokeAgent = invokeAgentSpans.find( - span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 10, - ); - expect(textInvokeAgent).toBeDefined(); - expect(textInvokeAgent!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(20); - expect(textInvokeAgent!.attributes[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE].value).toBe(30); - expect(textInvokeAgent!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe('["stop"]'); - - const toolInvokeAgent = invokeAgentSpans.find( - span => span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value === 15, - ); - expect(toolInvokeAgent).toBeDefined(); - expect(toolInvokeAgent!.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE].value).toBe( - '["tool-calls"]', - ); - - const generateContentSpans = container.items.filter( - span => span.name === 'generate_content mock-model-id', - ); - expect(generateContentSpans).toHaveLength(3); - for (const span of generateContentSpans) { - expect(span.status).toBe('ok'); - expect(span.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(span.attributes['vercel.ai.operationId'].value).toBe('ai.generateText.doGenerate'); - expect(span.attributes[GEN_AI_SYSTEM_ATTRIBUTE].value).toBe('mock-provider'); - expect(span.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - // Per-call usage is recorded on the model-call span. - expect(span.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBeGreaterThan(0); - expect(span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeUndefined(); - } - - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); - // Recording disabled: no tool input/output captured. - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeUndefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeUndefined(); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: vercelAiVersion, - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario.mjs', - 'instrument-with-pii.mjs', - (createRunner, test) => { - test('creates ai related spans with genAI recording enabled', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - expect(container.items).toHaveLength(7); - - const firstInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"Where is the first span?"}]', - ); - expect(firstInvokeAgentSpan).toBeDefined(); - expect(firstInvokeAgentSpan!.status).toBe('ok'); - expect(firstInvokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(firstInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toBe( - '[{"role":"assistant","parts":[{"type":"text","content":"First span here!"}],"finish_reason":"stop"}]', - ); - - const firstGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE]?.value?.includes('First span here!'), - ); - expect(firstGenerateContentSpan).toBeDefined(); - expect(firstGenerateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - expect(firstGenerateContentSpan!.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]).toBeDefined(); - expect(firstGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - - const toolInvokeAgentSpan = container.items.find( - span => - span.name === 'invoke_agent' && - span.attributes[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]?.value === - '[{"role":"user","content":"What is the weather in San Francisco?"}]', - ); - expect(toolInvokeAgentSpan).toBeDefined(); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - expect(toolInvokeAgentSpan!.attributes[GEN_AI_OUTPUT_MESSAGES_ATTRIBUTE].value).toContain('tool_call'); - - // The available tools are recorded on the model-call span for the tool generation. - const toolGenerateContentSpan = container.items.find( - span => - span.name === 'generate_content mock-model-id' && - span.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE] !== undefined, - ); - expect(toolGenerateContentSpan).toBeDefined(); - expect(toolGenerateContentSpan!.attributes[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE].value).toContain( - 'getWeather', - ); - expect(toolGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - - const toolExecutionSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolExecutionSpan).toBeDefined(); - expect(toolExecutionSpan!.status).toBe('ok'); - expect(toolExecutionSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_DESCRIPTION_ATTRIBUTE].value).toBe( - 'Get the current weather for a location', - ); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_INPUT_ATTRIBUTE]).toBeDefined(); - expect(toolExecutionSpan!.attributes[GEN_AI_TOOL_OUTPUT_ATTRIBUTE]).toBeDefined(); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: vercelAiVersion, - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-error-in-tool.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('captures error in tool', async () => { - let transactionEvent: Event | undefined; - let errorEvent: Event | undefined; - - await createRunner() - .expect({ - transaction: transaction => { - transactionEvent = transaction; - }, - }) - .expect({ - span: container => { - expect(container.items).toHaveLength(3); - - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - - const generateContentSpan = container.items.find(span => span.name === 'generate_content mock-model-id'); - expect(generateContentSpan).toBeDefined(); - expect(generateContentSpan!.status).toBe('ok'); - expect(generateContentSpan!.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - // The failing tool surfaces as `tool-error` content; the integration marks the span errored. - expect(toolSpan!.status).toBe('error'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - }, - }) - .expect({ - event: event => { - errorEvent = event; - }, - }) - .start() - .completed(); - - expect(transactionEvent).toBeDefined(); - expect(transactionEvent!.transaction).toBe('main'); - - expect(errorEvent).toBeDefined(); - expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toEqual( - expect.objectContaining({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }), - ); - - // Trace id should be the same for the transaction and error event - expect(transactionEvent!.contexts!.trace!.trace_id).toBe(errorEvent!.contexts!.trace!.trace_id); - }); - }, - { - additionalDependencies: { - ai: vercelAiVersion, - }, - }, - ); - - createEsmAndCjsTests( - __dirname, - 'scenario-tool-loop-agent.mjs', - 'instrument.mjs', - (createRunner, test) => { - test('creates spans for ToolLoopAgent with tool calls', async () => { - await createRunner() - .expect({ transaction: { transaction: 'main' } }) - .expect({ - span: container => { - // ToolLoopAgent loops: one invoke_agent (named after the functionId), two model calls - // (tool-call step + final step), and one tool execution. - expect(container.items).toHaveLength(4); - - const invokeAgentSpan = container.items.find(span => span.name === 'invoke_agent weather_agent'); - expect(invokeAgentSpan).toBeDefined(); - expect(invokeAgentSpan!.status).toBe('ok'); - expect(invokeAgentSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent'); - expect(invokeAgentSpan!.attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE].value).toBe('mock-model-id'); - // Usage is aggregated across both steps on the invoke_agent span (10+15, 20+25). - expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(25); - expect(invokeAgentSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE].value).toBe(45); - - const generateContentSpans = container.items.filter( - span => span.name === 'generate_content mock-model-id', - ); - expect(generateContentSpans).toHaveLength(2); - for (const span of generateContentSpans) { - expect(span.attributes['sentry.op'].value).toBe('gen_ai.generate_content'); - } - - // The first model call reports the tool-call finish reason, the final one stops. - const toolCallsGenerateContentSpan = generateContentSpans.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["tool-calls"]', - ); - expect(toolCallsGenerateContentSpan).toBeDefined(); - expect(toolCallsGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(10); - - const finalGenerateContentSpan = generateContentSpans.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]?.value === '["stop"]', - ); - expect(finalGenerateContentSpan).toBeDefined(); - expect(finalGenerateContentSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE].value).toBe(15); - - const toolSpan = container.items.find(span => span.name === 'execute_tool getWeather'); - expect(toolSpan).toBeDefined(); - expect(toolSpan!.status).toBe('ok'); - expect(toolSpan!.attributes['sentry.op'].value).toBe('gen_ai.execute_tool'); - expect(toolSpan!.attributes[GEN_AI_TOOL_NAME_ATTRIBUTE].value).toBe('getWeather'); - expect(toolSpan!.attributes[GEN_AI_TOOL_CALL_ID_ATTRIBUTE].value).toBe('call-1'); - expect(toolSpan!.attributes[GEN_AI_TOOL_TYPE_ATTRIBUTE].value).toBe('function'); - }, - }) - .start() - .completed(); - }); - }, - { - additionalDependencies: { - ai: vercelAiVersion, - }, - }, - ); -}); diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index a4ba1a2d67b7..ebdaeb3565e9 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -195,7 +195,7 @@ function createSpanFromMessage(data: VercelAiTracingChannelContextWithSpan): { recordInputs: boolean; recordOutputs: boolean } { const client = getClient(); const genAI = client?.getDataCollectionOptions().genAI; const options = client?.getIntegrationByName< Integration & { options?: { recordInputs?: boolean; recordOutputs?: boolean } } >(INTEGRATION_NAME)?.options; + // Record when the global config opts in OR when the individual `ai` call enabled recording + // (e.g. via `experimental_telemetry: { isEnabled: true }`). The AI SDK resolves the per-call flags + // onto the channel event, matching the OTel integration's per-call precedence. + const globalInputs = options?.recordInputs ?? genAI?.inputs ?? false; + const globalOutputs = options?.recordOutputs ?? genAI?.outputs ?? false; + return { - recordInputs: options?.recordInputs ?? genAI?.inputs ?? false, - recordOutputs: options?.recordOutputs ?? genAI?.outputs ?? false, + recordInputs: globalInputs || event.recordInputs === true, + recordOutputs: globalOutputs || event.recordOutputs === true, }; } From 1ca0a2a32c4f1460f4d2ae9763dee71dacdd1561 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 17 Jun 2026 16:29:17 +0200 Subject: [PATCH 4/5] cleanup --- packages/server-utils/src/index.ts | 1 - .../server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts | 5 ----- 2 files changed, 6 deletions(-) diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index e9572920be10..ac67454159c8 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -24,7 +24,6 @@ export type { RedisTracingChannelSubscribers, } from './redis/redis-dc-subscriber'; export { - _resetVercelAiTracingChannelForTesting, AI_SDK_TELEMETRY_TRACING_CHANNEL, subscribeVercelAiTracingChannel, } from './vercel-ai/vercel-ai-dc-subscriber'; diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index ebdaeb3565e9..dfb33ceba0cf 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -554,8 +554,3 @@ function safeStringify(value: unknown): string { return '[unserializable]'; } } - -/** Test-only: reset module-local subscribe state. */ -export function _resetVercelAiTracingChannelForTesting(): void { - subscribed = false; -} From 21822cea64c79d7dd11e2bd402bf57e1d9f55795 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 17 Jun 2026 19:18:33 +0200 Subject: [PATCH 5/5] small fixes --- packages/server-utils/src/index.ts | 5 +- .../src/vercel-ai/vercel-ai-dc-subscriber.ts | 62 ++++++++++++++----- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index ac67454159c8..c7bffe405cb3 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -23,10 +23,7 @@ export type { RedisTracingChannelFactory, RedisTracingChannelSubscribers, } from './redis/redis-dc-subscriber'; -export { - AI_SDK_TELEMETRY_TRACING_CHANNEL, - subscribeVercelAiTracingChannel, -} from './vercel-ai/vercel-ai-dc-subscriber'; +export { AI_SDK_TELEMETRY_TRACING_CHANNEL, subscribeVercelAiTracingChannel } from './vercel-ai/vercel-ai-dc-subscriber'; export type { VercelAiChannelMessage, VercelAiTracingChannel, diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts index dfb33ceba0cf..cb6e25505178 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-dc-subscriber.ts @@ -1,4 +1,9 @@ /* eslint-disable max-lines */ +// `@sentry/conventions` marks several gen_ai attributes (e.g. `GEN_AI_SYSTEM`, `GEN_AI_TOOL_*`, +// `GEN_AI_REQUEST_AVAILABLE_TOOLS`) as deprecated in favour of newer semconv names. We intentionally +// keep emitting the current names so these spans match the OTel-based (v6) integration and what the +// Sentry product consumes today; migrating to the new names is a separate, coordinated change. +/* eslint-disable typescript-eslint/no-deprecated */ import { GEN_AI_EMBEDDINGS_INPUT, GEN_AI_FUNCTION_ID, @@ -62,6 +67,21 @@ const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries' // `doStream` operation the same way the OTel integration does. Cleared when the top-level span ends. const operationIdByCallId = new Map(); +// Only top-level operations own the `callId` → operationId mapping; `step`/`languageModelCall`/ +// `executeTool` share the parent's `callId`, so they must not clear it. +const ROOT_OPERATION_TYPES = new Set(['generateText', 'streamText', 'embed', 'rerank']); + +/** Drop the `callId` → operationId mapping once the owning top-level operation settles (success or error). */ +function clearOperationId(data: VercelAiChannelMessage): void { + if (!ROOT_OPERATION_TYPES.has(data.type)) { + return; + } + const callId = asString(data.event.callId); + if (callId) { + operationIdByCallId.delete(callId); + } +} + const NOOP = (): void => {}; /** The lifecycle event types the `ai:telemetry` channel can carry. */ @@ -158,11 +178,15 @@ export function subscribeVercelAiTracingChannel(tracingChannel: VercelAiTracingC } enrichSpanOnEnd(span, data); span.end(); + clearOperationId(data); }, error: data => { if (data._sentrySkip) { return; } + // Always drop the mapping (even if the span is missing) so a rejected call can't leak a stale + // `callId` → operationId entry. + clearOperationId(data); const span = data._sentrySpan; if (!span) { return; @@ -345,14 +369,6 @@ function enrichSpanOnEnd(span: Span, data: VercelAiChannelMessage): void { return; } - // Top-level call finished — drop its operationId mapping. - if (type !== 'languageModelCall') { - const callId = asString(data.event.callId); - if (callId) { - operationIdByCallId.delete(callId); - } - } - // `languageModelCall` results report usage as `{ total }` objects; top-level/step results report // flat numbers. `tokenCount` handles both. const usage = isRecord(result.usage) ? result.usage : undefined; @@ -506,18 +522,32 @@ function getRecordingOptions(event: Record): { recordInputs: bo Integration & { options?: { recordInputs?: boolean; recordOutputs?: boolean } } >(INTEGRATION_NAME)?.options; - // Record when the global config opts in OR when the individual `ai` call enabled recording - // (e.g. via `experimental_telemetry: { isEnabled: true }`). The AI SDK resolves the per-call flags - // onto the channel event, matching the OTel integration's per-call precedence. - const globalInputs = options?.recordInputs ?? genAI?.inputs ?? false; - const globalOutputs = options?.recordOutputs ?? genAI?.outputs ?? false; - return { - recordInputs: globalInputs || event.recordInputs === true, - recordOutputs: globalOutputs || event.recordOutputs === true, + recordInputs: resolveRecording(options?.recordInputs, event.recordInputs, genAI?.inputs), + recordOutputs: resolveRecording(options?.recordOutputs, event.recordOutputs, genAI?.outputs), }; } +/** + * Mirrors the OTel integration's `determineRecordingSettings` precedence: an integration-level option + * wins, then the per-call `experimental_telemetry.recordInputs/recordOutputs` flag the AI SDK forwards + * on the channel event, then the global `dataCollection.genAI` default. + * + * NOTE: the OTel integration also defaults recording to `true` for a call with + * `experimental_telemetry: { isEnabled: true }`. The `ai:telemetry` channel does not expose `isEnabled` + * (nor a resolved recording flag), so that per-call default cannot be reproduced here — v7 users who + * want inputs/outputs recorded must enable `dataCollection.genAI` or set `recordInputs`/`recordOutputs`. + */ +function resolveRecording(integrationOption: unknown, perCallOption: unknown, globalDefault: unknown): boolean { + if (typeof integrationOption === 'boolean') { + return integrationOption; + } + if (typeof perCallOption === 'boolean') { + return perCallOption; + } + return globalDefault === true; +} + function buildInputMessageAttributes(event: Record): Record { // The AI SDK start events extend `StandardizedPrompt`; messages live on `messages`, otherwise the // simpler `prompt` field is used.