diff --git a/CHANGELOG.md b/CHANGELOG.md index 334eadff..ccacef0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to the Agent365 TypeScript SDK will be documented in this fi The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- **OutputScope**: Tracing scope for outgoing agent messages with caller details, conversation ID, source metadata, and parent span linking. +- **BaggageMiddleware**: Middleware for automatic OpenTelemetry baggage propagation from TurnContext. +- **OutputLoggingMiddleware**: Middleware that creates OutputScope spans for outgoing messages with lazy parent span linking via `A365_PARENT_SPAN_KEY`. +- **ObservabilityHostingManager**: Manager for configuring hosting-layer observability middleware with `ObservabilityHostingOptions`. + +### Changed +- `InferenceScope.recordInputMessages()` / `recordOutputMessages()` now use JSON array format instead of comma-separated strings. +- `InvokeAgentScope.recordInputMessages()` / `recordOutputMessages()` now use JSON array format instead of comma-separated strings. + ## [1.1.0] - 2025-12-09 ### Changed diff --git a/packages/agents-a365-observability-hosting/src/index.ts b/packages/agents-a365-observability-hosting/src/index.ts index 2484a5eb..5f7113e5 100644 --- a/packages/agents-a365-observability-hosting/src/index.ts +++ b/packages/agents-a365-observability-hosting/src/index.ts @@ -7,3 +7,7 @@ export * from './utils/BaggageBuilderUtils'; export * from './utils/ScopeUtils'; export * from './utils/TurnContextUtils'; export { AgenticTokenCache, AgenticTokenCacheInstance } from './caching/AgenticTokenCache'; +export { BaggageMiddleware } from './middleware/BaggageMiddleware'; +export { OutputLoggingMiddleware, A365_PARENT_SPAN_KEY } from './middleware/OutputLoggingMiddleware'; +export { ObservabilityHostingManager } from './middleware/ObservabilityHostingManager'; +export type { ObservabilityHostingOptions } from './middleware/ObservabilityHostingManager'; diff --git a/packages/agents-a365-observability-hosting/src/middleware/BaggageMiddleware.ts b/packages/agents-a365-observability-hosting/src/middleware/BaggageMiddleware.ts new file mode 100644 index 00000000..344f67c8 --- /dev/null +++ b/packages/agents-a365-observability-hosting/src/middleware/BaggageMiddleware.ts @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TurnContext, Middleware } from '@microsoft/agents-hosting'; +import { ActivityTypes, ActivityEventNames } from '@microsoft/agents-activity'; +import { BaggageBuilder } from '@microsoft/agents-a365-observability'; +import { + getExecutionTypePair, + getCallerBaggagePairs, + getTargetAgentBaggagePairs, + getTenantIdPair, + getSourceMetadataBaggagePairs, + getConversationIdAndItemLinkPairs, +} from '../utils/TurnContextUtils'; + +/** + * Middleware that propagates OpenTelemetry baggage context derived from TurnContext. + * Async replies (ContinueConversation) are passed through without baggage setup. + */ +export class BaggageMiddleware implements Middleware { + + async onTurn(context: TurnContext, next: () => Promise): Promise { + const isAsyncReply = + context.activity?.type === ActivityTypes.Event && + context.activity?.name === ActivityEventNames.ContinueConversation; + + if (isAsyncReply) { + await next(); + return; + } + + const baggageScope = new BaggageBuilder() + .setPairs(getCallerBaggagePairs(context)) + .setPairs(getTargetAgentBaggagePairs(context)) + .setPairs(getTenantIdPair(context)) + .setPairs(getSourceMetadataBaggagePairs(context)) + .setPairs(getConversationIdAndItemLinkPairs(context)) + .setPairs(getExecutionTypePair(context)) + .build(); + + await baggageScope.run(async () => { + await next(); + }); + } +} diff --git a/packages/agents-a365-observability-hosting/src/middleware/ObservabilityHostingManager.ts b/packages/agents-a365-observability-hosting/src/middleware/ObservabilityHostingManager.ts new file mode 100644 index 00000000..a5f05803 --- /dev/null +++ b/packages/agents-a365-observability-hosting/src/middleware/ObservabilityHostingManager.ts @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Middleware } from '@microsoft/agents-hosting'; +import { logger } from '@microsoft/agents-a365-observability'; +import { BaggageMiddleware } from './BaggageMiddleware'; +import { OutputLoggingMiddleware } from './OutputLoggingMiddleware'; + +/** + * Configuration options for the hosting observability layer. + */ +export interface ObservabilityHostingOptions { + /** Enable baggage propagation middleware. Defaults to true. */ + enableBaggage?: boolean; + + /** Enable output logging middleware for tracing outgoing messages. Defaults to false. */ + enableOutputLogging?: boolean; +} + +/** + * Manager for configuring hosting-layer observability middleware. + * + * @example + * ```typescript + * const manager = new ObservabilityHostingManager(); + * manager.configure(adapter, { enableOutputLogging: true }); + * ``` + */ +export class ObservabilityHostingManager { + private _configured = false; + + /** + * Registers observability middleware on the adapter. + * Subsequent calls are ignored. + */ + configure( + adapter: { use(...middlewares: Array): void }, + options: ObservabilityHostingOptions + ): void { + if (this._configured) { + logger.warn('[ObservabilityHostingManager] Already configured. Subsequent configure() calls are ignored.'); + return; + } + + const enableBaggage = options.enableBaggage !== false; + const enableOutputLogging = options.enableOutputLogging === true; + + if (enableBaggage) { + adapter.use(new BaggageMiddleware()); + logger.info('[ObservabilityHostingManager] BaggageMiddleware registered.'); + } + if (enableOutputLogging) { + adapter.use(new OutputLoggingMiddleware()); + logger.info('[ObservabilityHostingManager] OutputLoggingMiddleware registered.'); + } + + logger.info(`[ObservabilityHostingManager] Configured. Baggage: ${enableBaggage}, OutputLogging: ${enableOutputLogging}.`); + this._configured = true; + } +} diff --git a/packages/agents-a365-observability-hosting/src/middleware/OutputLoggingMiddleware.ts b/packages/agents-a365-observability-hosting/src/middleware/OutputLoggingMiddleware.ts new file mode 100644 index 00000000..bc2225ed --- /dev/null +++ b/packages/agents-a365-observability-hosting/src/middleware/OutputLoggingMiddleware.ts @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { TurnContext, Middleware, SendActivitiesHandler } from '@microsoft/agents-hosting'; +import { + OutputScope, + AgentDetails, + TenantDetails, + CallerDetails, + ParentSpanRef, + logger, +} from '@microsoft/agents-a365-observability'; +import { ScopeUtils } from '../utils/ScopeUtils'; + +/** + * TurnState key for the parent span reference. + * Set this in `turnState` to link OutputScope spans as children of an InvokeAgentScope. + */ +export const A365_PARENT_SPAN_KEY = 'A365ParentSpanId'; + +/** + * Middleware that creates {@link OutputScope} spans for outgoing messages. + * Links to a parent span when {@link A365_PARENT_SPAN_KEY} is set in turnState. + * + * **Privacy note:** Outgoing message content is captured verbatim + * as span attributes and exported to the configured telemetry backend. + */ +export class OutputLoggingMiddleware implements Middleware { + + async onTurn(context: TurnContext, next: () => Promise): Promise { + const agentDetails = ScopeUtils.deriveAgentDetails(context); + const tenantDetails = ScopeUtils.deriveTenantDetails(context); + + if (!agentDetails || !tenantDetails) { + await next(); + return; + } + + const callerDetails = ScopeUtils.deriveCallerDetails(context); + const conversationId = ScopeUtils.deriveConversationId(context); + const sourceMetadata = ScopeUtils.deriveSourceMetadataObject(context); + + context.onSendActivities( + this._createSendHandler(context, agentDetails, tenantDetails, callerDetails, conversationId, sourceMetadata) + ); + + await next(); + } + + /** + * Creates a send handler that wraps outgoing messages in OutputScope spans. + * Reads parent span ref lazily so the agent handler can set it during `next()`. + */ + private _createSendHandler( + turnContext: TurnContext, + agentDetails: AgentDetails, + tenantDetails: TenantDetails, + callerDetails?: CallerDetails, + conversationId?: string, + sourceMetadata?: { name?: string; description?: string }, + ): SendActivitiesHandler { + return async (_ctx, activities, sendNext) => { + const messages = activities + .filter((a) => a.type === 'message' && a.text) + .map((a) => a.text!); + + if (messages.length === 0) { + return await sendNext(); + } + + const parentSpanRef: ParentSpanRef | undefined = turnContext.turnState.get(A365_PARENT_SPAN_KEY); + if (!parentSpanRef) { + logger.warn( + `[OutputLoggingMiddleware] No parent span ref in turnState under '${A365_PARENT_SPAN_KEY}'. OutputScope will not be linked to a parent.` + ); + } + + const outputScope = OutputScope.start( + { messages }, + agentDetails, + tenantDetails, + callerDetails, + conversationId, + sourceMetadata, + undefined, + parentSpanRef, + ); + try { + return await sendNext(); + } catch (error) { + outputScope.recordError( + error instanceof Error ? error : new Error(typeof error === 'string' ? error : JSON.stringify(error)) + ); + throw error; + } finally { + outputScope.dispose(); + } + }; + } +} diff --git a/packages/agents-a365-observability-hosting/src/utils/ScopeUtils.ts b/packages/agents-a365-observability-hosting/src/utils/ScopeUtils.ts index 1f8531c3..e1b46201 100644 --- a/packages/agents-a365-observability-hosting/src/utils/ScopeUtils.ts +++ b/packages/agents-a365-observability-hosting/src/utils/ScopeUtils.ts @@ -15,11 +15,7 @@ import { InferenceDetails, InvokeAgentDetails, ToolCallDetails, - ExecutionType } from '@microsoft/agents-a365-observability'; -import { - getExecutionTypePair, -} from './TurnContextUtils'; /** * Unified utilities to populate scope tags from a TurnContext. @@ -198,7 +194,6 @@ export class ScopeUtils { public static buildInvokeAgentDetails(details: InvokeAgentDetails, turnContext: TurnContext): InvokeAgentDetails { const agent = ScopeUtils.deriveAgentDetails(turnContext); const srcMetaFromContext = ScopeUtils.deriveSourceMetadataObject(turnContext); - const executionTypePair = getExecutionTypePair(turnContext); const baseRequest = details.request ?? {}; const baseSource = baseRequest.sourceMetadata ?? {}; const mergedSourceMetadata = { @@ -212,7 +207,6 @@ export class ScopeUtils { conversationId: ScopeUtils.deriveConversationId(turnContext), request: { ...baseRequest, - executionType: executionTypePair.length > 0 ? (executionTypePair[0][1] as ExecutionType) : baseRequest.executionType, sourceMetadata: mergedSourceMetadata } }; @@ -247,4 +241,5 @@ export class ScopeUtils { const scope = ExecuteToolScope.start(details, agent, tenant, conversationId, sourceMetadata, undefined, startTime, endTime); return scope; } + } diff --git a/packages/agents-a365-observability/src/index.ts b/packages/agents-a365-observability/src/index.ts index b86f9338..b2131b64 100644 --- a/packages/agents-a365-observability/src/index.ts +++ b/packages/agents-a365-observability/src/index.ts @@ -44,7 +44,7 @@ export { InferenceDetails, InferenceOperationType, InferenceResponse, - OutputResponse + OutputResponse, } from './tracing/contracts'; // Scopes diff --git a/packages/agents-a365-observability/src/tracing/contracts.ts b/packages/agents-a365-observability/src/tracing/contracts.ts index bef4b494..2a327312 100644 --- a/packages/agents-a365-observability/src/tracing/contracts.ts +++ b/packages/agents-a365-observability/src/tracing/contracts.ts @@ -19,6 +19,7 @@ export enum ExecutionType { Unknown = 'Unknown' } + /** * Represents different roles that can invoke an agent */ diff --git a/packages/agents-a365-observability/src/tracing/scopes/InferenceScope.ts b/packages/agents-a365-observability/src/tracing/scopes/InferenceScope.ts index 3973cc91..57e1d3ff 100644 --- a/packages/agents-a365-observability/src/tracing/scopes/InferenceScope.ts +++ b/packages/agents-a365-observability/src/tracing/scopes/InferenceScope.ts @@ -81,7 +81,7 @@ export class InferenceScope extends OpenTelemetryScope { * @param messages Array of input messages */ public recordInputMessages(messages: string[]): void { - this.setTagMaybe(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, messages.join(',')); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, JSON.stringify(messages)); } /** @@ -89,7 +89,7 @@ export class InferenceScope extends OpenTelemetryScope { * @param messages Array of output messages */ public recordOutputMessages(messages: string[]): void { - this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, messages.join(',')); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, JSON.stringify(messages)); } /** diff --git a/packages/agents-a365-observability/src/tracing/scopes/InvokeAgentScope.ts b/packages/agents-a365-observability/src/tracing/scopes/InvokeAgentScope.ts index cc161466..3e4b5692 100644 --- a/packages/agents-a365-observability/src/tracing/scopes/InvokeAgentScope.ts +++ b/packages/agents-a365-observability/src/tracing/scopes/InvokeAgentScope.ts @@ -125,7 +125,7 @@ export class InvokeAgentScope extends OpenTelemetryScope { * @param messages Array of input messages */ public recordInputMessages(messages: string[]): void { - this.setTagMaybe(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, messages.join(',')); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, JSON.stringify(messages)); } /** @@ -133,6 +133,6 @@ export class InvokeAgentScope extends OpenTelemetryScope { * @param messages Array of output messages */ public recordOutputMessages(messages: string[]): void { - this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, messages.join(',')); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, JSON.stringify(messages)); } } diff --git a/packages/agents-a365-observability/src/tracing/scopes/OutputScope.ts b/packages/agents-a365-observability/src/tracing/scopes/OutputScope.ts index 9888b456..145e31b5 100644 --- a/packages/agents-a365-observability/src/tracing/scopes/OutputScope.ts +++ b/packages/agents-a365-observability/src/tracing/scopes/OutputScope.ts @@ -3,7 +3,7 @@ import { SpanKind, TimeInput } from '@opentelemetry/api'; import { OpenTelemetryScope } from './OpenTelemetryScope'; -import { AgentDetails, TenantDetails, OutputResponse } from '../contracts'; +import { AgentDetails, TenantDetails, CallerDetails, OutputResponse, SourceMetadata, ExecutionType } from '../contracts'; import { ParentContext } from '../context/trace-context-propagation'; import { OpenTelemetryConstants } from '../constants'; @@ -12,12 +12,17 @@ import { OpenTelemetryConstants } from '../constants'; */ export class OutputScope extends OpenTelemetryScope { private _outputMessages: string[]; + private _outputMessagesDirty = false; /** * Creates and starts a new scope for output message tracing. * @param response The response containing initial output messages. * @param agentDetails The details of the agent producing the output. * @param tenantDetails The tenant details. + * @param callerDetails Optional caller identity details (id, upn, name, tenant, client ip). + * @param conversationId Optional conversation identifier. + * @param sourceMetadata Optional source metadata; only `name` and `description` are used for tagging. + * @param executionType Optional execution type (HumanToAgent, Agent2Agent, etc.). * @param parentContext Optional parent context for cross-async-boundary tracing. * Accepts a ParentSpanRef (manual traceId/spanId) or an OTel Context (e.g. from extractTraceContext). * @param startTime Optional explicit start time (ms epoch, Date, or HrTime). @@ -28,17 +33,25 @@ export class OutputScope extends OpenTelemetryScope { response: OutputResponse, agentDetails: AgentDetails, tenantDetails: TenantDetails, + callerDetails?: CallerDetails, + conversationId?: string, + sourceMetadata?: Pick, + executionType?: ExecutionType, parentContext?: ParentContext, startTime?: TimeInput, endTime?: TimeInput ): OutputScope { - return new OutputScope(response, agentDetails, tenantDetails, parentContext, startTime, endTime); + return new OutputScope(response, agentDetails, tenantDetails, callerDetails, conversationId, sourceMetadata, executionType, parentContext, startTime, endTime); } private constructor( response: OutputResponse, agentDetails: AgentDetails, tenantDetails: TenantDetails, + callerDetails?: CallerDetails, + conversationId?: string, + sourceMetadata?: Pick, + executionType?: ExecutionType, parentContext?: ParentContext, startTime?: TimeInput, endTime?: TimeInput @@ -46,7 +59,9 @@ export class OutputScope extends OpenTelemetryScope { super( SpanKind.CLIENT, OpenTelemetryConstants.OUTPUT_MESSAGES_OPERATION_NAME, - `${OpenTelemetryConstants.OUTPUT_MESSAGES_OPERATION_NAME} ${agentDetails.agentId}`, + agentDetails.agentName + ? `${OpenTelemetryConstants.OUTPUT_MESSAGES_OPERATION_NAME} ${agentDetails.agentName}` + : `${OpenTelemetryConstants.OUTPUT_MESSAGES_OPERATION_NAME} ${agentDetails.agentId}`, agentDetails, tenantDetails, parentContext, @@ -62,18 +77,45 @@ export class OutputScope extends OpenTelemetryScope { OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, JSON.stringify(this._outputMessages) ); + + // Set conversation, execution type, and source metadata + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_CONVERSATION_ID_KEY, conversationId); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_EXECUTION_TYPE_KEY, executionType); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_EXECUTION_SOURCE_NAME_KEY, sourceMetadata?.name); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_EXECUTION_SOURCE_DESCRIPTION_KEY, sourceMetadata?.description); + + // Set caller details if provided + if (callerDetails) { + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_CALLER_ID_KEY, callerDetails.callerId); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_CALLER_UPN_KEY, callerDetails.callerUpn); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_CALLER_NAME_KEY, callerDetails.callerName); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_CALLER_TENANT_ID_KEY, callerDetails.tenantId); + this.setTagMaybe(OpenTelemetryConstants.GEN_AI_CALLER_CLIENT_IP_KEY, callerDetails.callerClientIp); + } } /** * Records the output messages for telemetry tracking. * Appends the provided messages to the accumulated output messages list. + * The updated attribute is flushed when the scope is disposed. * @param messages Array of output messages to append. */ public recordOutputMessages(messages: string[]): void { this._outputMessages.push(...messages); - this.setTagMaybe( - OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, - JSON.stringify(this._outputMessages) - ); + this._outputMessagesDirty = true; + } + + public override [Symbol.dispose](): void { + if (this._outputMessagesDirty) { + this.setTagMaybe( + OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY, + JSON.stringify(this._outputMessages) + ); + } + super[Symbol.dispose](); + } + + public override dispose(): void { + this[Symbol.dispose](); } } diff --git a/tests/observability/core/output-scope.test.ts b/tests/observability/core/output-scope.test.ts index 42d1b484..a706931a 100644 --- a/tests/observability/core/output-scope.test.ts +++ b/tests/observability/core/output-scope.test.ts @@ -72,129 +72,53 @@ describe('OutputScope', () => { return { span, attributes: span.attributes }; } - it('should create scope with output messages', async () => { + it('should create scope with correct span attributes and output messages', async () => { const response: OutputResponse = { messages: ['First message', 'Second message'] }; const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - expect(scope).toBeInstanceOf(OutputScope); scope.dispose(); await flushProvider.forceFlush(); const { span, attributes } = getLastSpan(); - // Verify span name contains operation name and agent id - expect(span.name).toContain('output_messages'); - expect(span.name).toContain(testAgentDetails.agentId); - - // Verify output messages are set as JSON array - const outputValue = attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY] as string; - expect(outputValue).toBeDefined(); - const parsed = JSON.parse(outputValue); + expect(span.name).toBe('output_messages Test Agent'); + expect(attributes[OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY]).toBe('output_messages'); + expect(attributes[OpenTelemetryConstants.GEN_AI_AGENT_ID_KEY]).toBe(testAgentDetails.agentId); + expect(attributes[OpenTelemetryConstants.GEN_AI_AGENT_NAME_KEY]).toBe(testAgentDetails.agentName); + const parsed = JSON.parse(attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY] as string); expect(parsed).toEqual(['First message', 'Second message']); }); - it('should record output messages by appending to accumulated messages', async () => { + it('should append messages with recordOutputMessages and flush on dispose', async () => { const response: OutputResponse = { messages: ['Initial'] }; const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - scope.recordOutputMessages(['Appended 1']); scope.recordOutputMessages(['Appended 2', 'Appended 3']); - scope.dispose(); await flushProvider.forceFlush(); const { attributes } = getLastSpan(); - const outputValue = attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY] as string; - expect(outputValue).toBeDefined(); - // All messages should be present (initial + all appended) as JSON array - const parsed = JSON.parse(outputValue); + const parsed = JSON.parse(attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY] as string); expect(parsed).toEqual(['Initial', 'Appended 1', 'Appended 2', 'Appended 3']); }); - it('should use parent span reference to link span to parent context', async () => { - const response: OutputResponse = { messages: ['Test'] }; + it('should use parent span reference for linking', async () => { const parentTraceId = '1234567890abcdef1234567890abcdef'; const parentSpanId = 'abcdefabcdef1234'; - const parentSpanRef: ParentSpanRef = { - traceId: parentTraceId, - spanId: parentSpanId, - }; - - const scope = OutputScope.start(response, testAgentDetails, testTenantDetails, parentSpanRef); + const scope = OutputScope.start( + { messages: ['Test'] }, testAgentDetails, testTenantDetails, + undefined, undefined, undefined, undefined, + { traceId: parentTraceId, spanId: parentSpanId } as ParentSpanRef + ); scope.dispose(); await flushProvider.forceFlush(); const { span } = getLastSpan(); - - // Verify span inherits parent's trace_id expect(span.spanContext().traceId).toBe(parentTraceId); - - // Verify span's parent_span_id matches expect(span.parentSpanContext?.spanId).toBe(parentSpanId); }); - - it('should end the span on dispose', async () => { - const response: OutputResponse = { messages: ['Test'] }; - - const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - scope.dispose(); - - await flushProvider.forceFlush(); - - // Verify span was created and ended - const spans = exporter.getFinishedSpans(); - expect(spans.length).toBe(1); - }); - - it('should create scope with empty messages', async () => { - const response: OutputResponse = { messages: [] }; - - const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - scope.dispose(); - - await flushProvider.forceFlush(); - const { span } = getLastSpan(); - expect(span.name).toContain('output_messages'); - }); - - it('should set gen_ai.operation.name to output_messages', async () => { - const response: OutputResponse = { messages: ['Test'] }; - - const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - scope.dispose(); - - await flushProvider.forceFlush(); - const { attributes } = getLastSpan(); - expect(attributes[OpenTelemetryConstants.GEN_AI_OPERATION_NAME_KEY]).toBe('output_messages'); - }); - - it('should set agent details on the span', async () => { - const response: OutputResponse = { messages: ['Test'] }; - - const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - scope.dispose(); - - await flushProvider.forceFlush(); - const { attributes } = getLastSpan(); - expect(attributes[OpenTelemetryConstants.GEN_AI_AGENT_ID_KEY]).toBe(testAgentDetails.agentId); - expect(attributes[OpenTelemetryConstants.GEN_AI_AGENT_NAME_KEY]).toBe(testAgentDetails.agentName); - }); - - it('should not throw when recordOutputMessages is called multiple times', () => { - const response: OutputResponse = { messages: ['Initial'] }; - - const scope = OutputScope.start(response, testAgentDetails, testTenantDetails); - - expect(() => { - scope.recordOutputMessages(['Message 1']); - scope.recordOutputMessages(['Message 2']); - scope.recordOutputMessages(['Message 3']); - }).not.toThrow(); - - scope.dispose(); - }); }); diff --git a/tests/observability/extension/hosting/baggage-middleware.test.ts b/tests/observability/extension/hosting/baggage-middleware.test.ts new file mode 100644 index 00000000..f2d1907b --- /dev/null +++ b/tests/observability/extension/hosting/baggage-middleware.test.ts @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, beforeAll, afterAll } from '@jest/globals'; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { trace, context as otelContext, propagation } from '@opentelemetry/api'; +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; +import { RoleTypes, ActivityTypes, ActivityEventNames } from '@microsoft/agents-activity'; +import type { TurnContext } from '@microsoft/agents-hosting'; + +import { BaggageMiddleware } from '../../../../packages/agents-a365-observability-hosting/src/middleware/BaggageMiddleware'; +import { OpenTelemetryConstants } from '@microsoft/agents-a365-observability'; + +function makeMockTurnContext(options?: { + text?: string; + recipientId?: string; + recipientTenantId?: string; + channelId?: string; + conversationId?: string; + fromRole?: string; + activityType?: string; + activityName?: string; +}): TurnContext { + const ctx: any = { + activity: { + type: options?.activityType, + name: options?.activityName, + text: options?.text ?? 'Hello agent', + channelId: options?.channelId ?? 'web', + conversation: { id: options?.conversationId ?? 'conv-001' }, + from: { + role: options?.fromRole ?? RoleTypes.User, + aadObjectId: 'user-oid', + name: 'Test User', + agenticUserId: 'user@contoso.com', + tenantId: 'from-tenant', + }, + recipient: { + agenticAppId: options?.recipientId ?? 'agent-1', + name: 'Agent One', + aadObjectId: 'agent-oid', + agenticAppBlueprintId: 'blueprint-1', + agenticUserId: 'agent@contoso.com', + role: 'assistant', + tenantId: options?.recipientTenantId ?? 'tenant-123', + }, + }, + turnState: new Map(), + }; + + return ctx; +} + +describe('BaggageMiddleware', () => { + let provider: BasicTracerProvider; + let contextManager: AsyncLocalStorageContextManager; + + beforeAll(() => { + contextManager = new AsyncLocalStorageContextManager(); + contextManager.enable(); + otelContext.setGlobalContextManager(contextManager); + + const exporter = new InMemorySpanExporter(); + const processor = new SimpleSpanProcessor(exporter); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const globalProvider: any = trace.getTracerProvider(); + if (globalProvider && typeof globalProvider.addSpanProcessor === 'function') { + globalProvider.addSpanProcessor(processor); + } else { + provider = new BasicTracerProvider({ + spanProcessors: [processor] + }); + trace.setGlobalTracerProvider(provider); + } + }); + + afterAll(async () => { + await provider?.shutdown?.(); + contextManager.disable(); + otelContext.disable(); + }); + + it('should propagate baggage context during turn', async () => { + const middleware = new BaggageMiddleware(); + const ctx = makeMockTurnContext(); + let capturedBaggage: Record = {}; + + await middleware.onTurn(ctx, async () => { + const bag = propagation.getBaggage(otelContext.active()); + if (bag) { + for (const [key, entry] of bag.getAllEntries()) { + capturedBaggage[key] = entry.value; + } + } + }); + + expect(capturedBaggage[OpenTelemetryConstants.GEN_AI_CALLER_ID_KEY]).toBe('user-oid'); + expect(capturedBaggage[OpenTelemetryConstants.TENANT_ID_KEY]).toBe('tenant-123'); + expect(capturedBaggage[OpenTelemetryConstants.GEN_AI_AGENT_ID_KEY]).toBe('agent-1'); + expect(capturedBaggage[OpenTelemetryConstants.GEN_AI_EXECUTION_SOURCE_NAME_KEY]).toBe('web'); + expect(capturedBaggage[OpenTelemetryConstants.GEN_AI_CONVERSATION_ID_KEY]).toBe('conv-001'); + }); + + it('should skip baggage setup for async replies (ContinueConversation)', async () => { + const middleware = new BaggageMiddleware(); + const ctx = makeMockTurnContext({ + activityType: ActivityTypes.Event, + activityName: ActivityEventNames.ContinueConversation, + }); + let capturedBaggage: Record = {}; + + await middleware.onTurn(ctx, async () => { + const bag = propagation.getBaggage(otelContext.active()); + if (bag) { + for (const [key, entry] of bag.getAllEntries()) { + capturedBaggage[key] = entry.value; + } + } + }); + + // No baggage set for async replies + expect(Object.keys(capturedBaggage).length).toBe(0); + }); + + it('should call next() even when baggage setup is skipped', async () => { + const middleware = new BaggageMiddleware(); + const ctx = makeMockTurnContext({ + activityType: ActivityTypes.Event, + activityName: ActivityEventNames.ContinueConversation, + }); + let nextCalled = false; + + await middleware.onTurn(ctx, async () => { nextCalled = true; }); + + expect(nextCalled).toBe(true); + }); +}); diff --git a/tests/observability/extension/hosting/observability-hosting-manager.test.ts b/tests/observability/extension/hosting/observability-hosting-manager.test.ts new file mode 100644 index 00000000..99eae6e3 --- /dev/null +++ b/tests/observability/extension/hosting/observability-hosting-manager.test.ts @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect } from '@jest/globals'; +import { ObservabilityHostingManager } from '../../../../packages/agents-a365-observability-hosting/src/middleware/ObservabilityHostingManager'; +import { BaggageMiddleware } from '../../../../packages/agents-a365-observability-hosting/src/middleware/BaggageMiddleware'; +import { OutputLoggingMiddleware } from '../../../../packages/agents-a365-observability-hosting/src/middleware/OutputLoggingMiddleware'; + +function mockAdapter() { + const registered: any[] = []; + return { use(...mw: any[]) { registered.push(...mw); }, registered }; +} + +describe('ObservabilityHostingManager', () => { + it('registers BaggageMiddleware by default', () => { + const adapter = mockAdapter(); + new ObservabilityHostingManager().configure(adapter, {}); + expect(adapter.registered).toHaveLength(1); + expect(adapter.registered[0]).toBeInstanceOf(BaggageMiddleware); + }); + + it('registers both middleware when enableOutputLogging is true', () => { + const adapter = mockAdapter(); + new ObservabilityHostingManager().configure(adapter, { enableOutputLogging: true }); + expect(adapter.registered).toHaveLength(2); + expect(adapter.registered[0]).toBeInstanceOf(BaggageMiddleware); + expect(adapter.registered[1]).toBeInstanceOf(OutputLoggingMiddleware); + }); + + it('skips BaggageMiddleware when enableBaggage is false', () => { + const adapter = mockAdapter(); + new ObservabilityHostingManager().configure(adapter, { enableBaggage: false, enableOutputLogging: true }); + expect(adapter.registered).toHaveLength(1); + expect(adapter.registered[0]).toBeInstanceOf(OutputLoggingMiddleware); + }); + + it('subsequent configure calls on same instance are no-ops', () => { + const adapter = mockAdapter(); + const manager = new ObservabilityHostingManager(); + manager.configure(adapter, { enableOutputLogging: true }); + manager.configure(adapter, { enableOutputLogging: true }); + expect(adapter.registered).toHaveLength(2); + }); +}); diff --git a/tests/observability/extension/hosting/output-logging-middleware.test.ts b/tests/observability/extension/hosting/output-logging-middleware.test.ts new file mode 100644 index 00000000..448384bb --- /dev/null +++ b/tests/observability/extension/hosting/output-logging-middleware.test.ts @@ -0,0 +1,283 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { describe, it, expect, beforeAll, afterAll, beforeEach } from '@jest/globals'; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { trace, context as otelContext } from '@opentelemetry/api'; +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; +import { RoleTypes, ActivityTypes, ActivityEventNames } from '@microsoft/agents-activity'; +import type { TurnContext, SendActivitiesHandler } from '@microsoft/agents-hosting'; + +import { OutputLoggingMiddleware, A365_PARENT_SPAN_KEY } from '../../../../packages/agents-a365-observability-hosting/src/middleware/OutputLoggingMiddleware'; +import { OpenTelemetryConstants, ParentSpanRef } from '@microsoft/agents-a365-observability'; + +function makeMockTurnContext(options?: { + text?: string; + recipientId?: string; + recipientName?: string; + recipientTenantId?: string; + channelId?: string; + conversationId?: string; + fromRole?: string; + activityType?: string; + activityName?: string; +}): TurnContext & { _sendHandlers: SendActivitiesHandler[]; turnState: Map; simulateSend: (activities: Array<{ type?: string; text?: string }>) => Promise> } { + const sendHandlers: SendActivitiesHandler[] = []; + + const ctx: any = { + activity: { + type: options?.activityType, + name: options?.activityName, + text: options?.text ?? 'Hello agent', + channelId: options?.channelId ?? 'web', + conversation: { id: options?.conversationId ?? 'conv-001' }, + from: { + role: options?.fromRole ?? RoleTypes.User, + aadObjectId: 'user-oid', + name: 'Test User', + agenticUserId: 'user@contoso.com', + tenantId: 'from-tenant', + }, + recipient: { + agenticAppId: options?.recipientId ?? 'agent-1', + name: options?.recipientName ?? 'Agent One', + aadObjectId: 'agent-oid', + agenticAppBlueprintId: 'blueprint-1', + agenticUserId: 'agent@contoso.com', + role: 'assistant', + tenantId: options?.recipientTenantId ?? 'tenant-123', + }, + }, + turnState: new Map(), + onSendActivities(handler: SendActivitiesHandler) { + sendHandlers.push(handler); + return ctx; + }, + _sendHandlers: sendHandlers, + async simulateSend(activities: Array<{ type?: string; text?: string }>) { + const finalSend = async () => activities.map(() => ({ id: 'resp-1' })); + let current = finalSend; + for (let i = sendHandlers.length - 1; i >= 0; i--) { + const handler = sendHandlers[i]; + const prev = current; + current = () => handler(ctx, activities as any, prev as any); + } + return await current(); + }, + }; + + return ctx; +} + +describe('OutputLoggingMiddleware', () => { + let exporter: InMemorySpanExporter; + let provider: BasicTracerProvider; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let flushProvider: any; + let contextManager: AsyncLocalStorageContextManager; + + beforeAll(() => { + contextManager = new AsyncLocalStorageContextManager(); + contextManager.enable(); + otelContext.setGlobalContextManager(contextManager); + + exporter = new InMemorySpanExporter(); + const processor = new SimpleSpanProcessor(exporter); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const globalProvider: any = trace.getTracerProvider(); + if (globalProvider && typeof globalProvider.addSpanProcessor === 'function') { + globalProvider.addSpanProcessor(processor); + flushProvider = globalProvider; + } else { + provider = new BasicTracerProvider({ + spanProcessors: [processor] + }); + trace.setGlobalTracerProvider(provider); + flushProvider = provider; + } + }); + + beforeEach(() => { + exporter.reset(); + }); + + afterAll(async () => { + exporter.reset(); + await provider?.shutdown?.(); + contextManager.disable(); + otelContext.disable(); + }); + + it('should create OutputScope for outgoing messages', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ text: 'Hello' }); + + await middleware.onTurn(ctx, async () => { + ctx.turnState.set(A365_PARENT_SPAN_KEY, { traceId: '0af7651916cd43dd8448eb211c80319c', spanId: 'b7ad6b7169203331', traceFlags: 1 }); + await ctx.simulateSend([{ type: 'message', text: 'Hi there!' }]); + }); + + await flushProvider.forceFlush(); + const spans = exporter.getFinishedSpans(); + const outputSpan = spans.find(s => s.name.includes('output_messages')); + + expect(outputSpan).toBeDefined(); + expect(outputSpan!.attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY]).toBe(JSON.stringify(['Hi there!'])); + }); + + it('should skip non-message activities in OutputScope', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ text: 'Hello' }); + + await middleware.onTurn(ctx, async () => { + await ctx.simulateSend([ + { type: 'typing' }, + { type: 'event', text: 'some event data' }, + ]); + }); + + await flushProvider.forceFlush(); + expect(exporter.getFinishedSpans().find(s => s.name.includes('output_messages'))).toBeUndefined(); + }); + + it('should pass through without tracing for skip conditions', async () => { + const middleware = new OutputLoggingMiddleware(); + + // Missing agent details (no recipient) + const ctx1: any = { activity: { text: 'Hello' }, onSendActivities: jest.fn() }; + let nextCalled = false; + await middleware.onTurn(ctx1, async () => { nextCalled = true; }); + expect(nextCalled).toBe(true); + + // Missing tenant details (recipient but no tenantId) + const ctx2: any = { + activity: { text: 'Hello', recipient: { agenticAppId: 'agent-1', name: 'Agent' } }, + onSendActivities: jest.fn(), + }; + nextCalled = false; + await middleware.onTurn(ctx2, async () => { nextCalled = true; }); + expect(nextCalled).toBe(true); + }); + + it('should set caller details and enrichment on OutputScope span', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ text: 'Hello', channelId: 'teams' }); + + await middleware.onTurn(ctx, async () => { + ctx.turnState.set(A365_PARENT_SPAN_KEY, { traceId: '0af7651916cd43dd8448eb211c80319c', spanId: 'b7ad6b7169203331', traceFlags: 1 }); + await ctx.simulateSend([{ type: 'message', text: 'Reply' }]); + }); + + await flushProvider.forceFlush(); + const outputSpan = exporter.getFinishedSpans().find(s => s.name.includes('output_messages')); + expect(outputSpan).toBeDefined(); + + expect(outputSpan!.attributes[OpenTelemetryConstants.GEN_AI_CALLER_ID_KEY]).toBe('user-oid'); + expect(outputSpan!.attributes[OpenTelemetryConstants.GEN_AI_CALLER_NAME_KEY]).toBe('Test User'); + expect(outputSpan!.attributes[OpenTelemetryConstants.GEN_AI_EXECUTION_SOURCE_NAME_KEY]).toBe('teams'); + }); + + it('should link OutputScope to parent when parentSpanRef is set in turnState', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ text: 'Hello' }); + + const parentSpanRef: ParentSpanRef = { + traceId: '0af7651916cd43dd8448eb211c80319c', + spanId: 'b7ad6b7169203331', + traceFlags: 1, + }; + + await middleware.onTurn(ctx, async () => { + ctx.turnState.set(A365_PARENT_SPAN_KEY, parentSpanRef); + await ctx.simulateSend([{ type: 'message', text: 'Reply' }]); + }); + + await flushProvider.forceFlush(); + const outputSpan = exporter.getFinishedSpans().find(s => s.name.includes('output_messages')); + expect(outputSpan).toBeDefined(); + expect(outputSpan!.parentSpanContext?.traceId).toBe(parentSpanRef.traceId); + expect(outputSpan!.parentSpanContext?.spanId).toBe(parentSpanRef.spanId); + }); + + it('should create OutputScope for async reply (ContinueConversation) when messages are sent', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ + text: 'Hello', + activityType: ActivityTypes.Event, + activityName: ActivityEventNames.ContinueConversation, + }); + + await middleware.onTurn(ctx, async () => { + await ctx.simulateSend([{ type: 'message', text: 'Async reply' }]); + }); + + await flushProvider.forceFlush(); + const outputSpan = exporter.getFinishedSpans().find(s => s.name.includes('output_messages')); + expect(outputSpan).toBeDefined(); + expect(outputSpan!.attributes[OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY]).toBe(JSON.stringify(['Async reply'])); + }); + + it('should link async reply OutputScope to parent when parentSpanRef is set', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ + text: 'Hello', + activityType: ActivityTypes.Event, + activityName: ActivityEventNames.ContinueConversation, + }); + + const parentSpanRef: ParentSpanRef = { + traceId: '1af7651916cd43dd8448eb211c80319c', + spanId: 'c7ad6b7169203331', + traceFlags: 1, + }; + ctx.turnState.set(A365_PARENT_SPAN_KEY, parentSpanRef); + + await middleware.onTurn(ctx, async () => { + await ctx.simulateSend([{ type: 'message', text: 'Async reply' }]); + }); + + await flushProvider.forceFlush(); + const outputSpan = exporter.getFinishedSpans().find(s => s.name.includes('output_messages')); + expect(outputSpan).toBeDefined(); + expect(outputSpan!.parentSpanContext?.traceId).toBe(parentSpanRef.traceId); + expect(outputSpan!.parentSpanContext?.spanId).toBe(parentSpanRef.spanId); + }); + + it('should not create spans when no messages are sent', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ + text: 'Hello', + activityType: ActivityTypes.Event, + activityName: ActivityEventNames.ContinueConversation, + }); + + await middleware.onTurn(ctx, async () => { + // next() without sending any messages + }); + + await flushProvider.forceFlush(); + expect(exporter.getFinishedSpans().length).toBe(0); + }); + + it('should re-throw errors from sendNext after recording on OutputScope', async () => { + const middleware = new OutputLoggingMiddleware(); + const ctx = makeMockTurnContext({ text: 'Hello' }); + const sendError = new Error('send pipeline failed'); + + // Override simulateSend to make the send pipeline throw + ctx.simulateSend = async () => { throw sendError; }; + + // Register handler manually since simulateSend is overridden + await middleware.onTurn(ctx, async () => { + ctx.turnState.set(A365_PARENT_SPAN_KEY, { traceId: '0af7651916cd43dd8448eb211c80319c', spanId: 'b7ad6b7169203331', traceFlags: 1 }); + // Simulate the handler being called with message activities + const handler = ctx._sendHandlers[0]; + if (handler) { + await expect( + handler(ctx, [{ type: 'message', text: 'Reply' }] as any, async () => { throw sendError; }) + ).rejects.toThrow('send pipeline failed'); + } + }); + }); +}); diff --git a/tests/observability/extension/hosting/scope-utils.test.ts b/tests/observability/extension/hosting/scope-utils.test.ts index a5ad82f7..73b5e69d 100644 --- a/tests/observability/extension/hosting/scope-utils.test.ts +++ b/tests/observability/extension/hosting/scope-utils.test.ts @@ -73,7 +73,7 @@ describe('ScopeUtils.populateFromTurnContext', () => { [OpenTelemetryConstants.GEN_AI_AGENT_UPN_KEY, 'agent-upn@contoso.com'], [OpenTelemetryConstants.GEN_AI_AGENT_DESCRIPTION_KEY, 'assistant'], [OpenTelemetryConstants.TENANT_ID_KEY, 'tenant-123'], - [OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, 'input text'] + [OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, JSON.stringify(['input text'])] ]) ); scope?.dispose(); @@ -138,8 +138,7 @@ describe('ScopeUtils.populateFromTurnContext', () => { [OpenTelemetryConstants.GEN_AI_CALLER_AGENT_ID_KEY, 'callerAgent-1'], [OpenTelemetryConstants.GEN_AI_CALLER_AGENT_APPLICATION_ID_KEY, 'caller-agentBlueprintId'], [OpenTelemetryConstants.TENANT_ID_KEY, 'tenant-123'], - [OpenTelemetryConstants.GEN_AI_EXECUTION_TYPE_KEY, ExecutionType.Agent2Agent.toString()], - [OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, 'invoke message'], + [OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY, JSON.stringify(['invoke message'])], [OpenTelemetryConstants.GEN_AI_AGENT_ID_KEY, 'agent-1'], [OpenTelemetryConstants.GEN_AI_AGENT_NAME_KEY, 'Agent One'], [OpenTelemetryConstants.GEN_AI_AGENT_DESCRIPTION_KEY, 'assistant']