diff --git a/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts b/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts index fb616327ca4e4..57d4bcd91dad4 100644 --- a/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts +++ b/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts @@ -233,6 +233,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'quotaExhausted', severity: ChatInputNotificationSeverity.Info, message: localize('quota.exhausted.title', "Credit Limit Reached"), description, @@ -249,6 +250,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'overageActivation', severity: ChatInputNotificationSeverity.Info, message: localize('quota.overage.title', "Credit Limit Reached"), description: localize('quota.overage.desc', "Additional budget is now covering extra usage."), @@ -285,6 +287,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'quotaApproaching', severity: ChatInputNotificationSeverity.Info, message: localize('quota.approaching.title', "Credits at {0}%", warning.percentUsed), description, @@ -344,6 +347,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: warning.type === 'session' ? 'sessionRateLimitWarning' : 'weeklyRateLimitWarning', severity: ChatInputNotificationSeverity.Info, message, description, @@ -378,6 +382,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'managedPlanBlocked', severity: ChatInputNotificationSeverity.Info, message: localize('quota.blocked.managed.title', "Usage Blocked"), description: localize('quota.blocked.managed', "Your organization or enterprise has exceeded its Copilot budget. Contact your admin to resume usage."), diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts index 30ff5919e6aa9..bc4f1c2791b95 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts @@ -33,6 +33,7 @@ export interface IChatInputNotificationMuteAction { export interface IChatInputNotification { readonly id: string; + readonly telemetryId?: string; readonly severity: ChatInputNotificationSeverity; readonly message: string | IMarkdownString; readonly description: string | undefined; diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts index d95a03a7d6c15..57e1488dece95 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts @@ -22,6 +22,18 @@ import './media/chatInputNotificationWidget.css'; const $ = dom.$; +type ChatInputNotificationTelemetryEvent = { + id: string; + telemetryId?: string; +}; + +type ChatInputNotificationTelemetryClassification = { + id: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The identifier of the chat input notification.' }; + telemetryId?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The feature-provided identifier for the notification message that was shown or dismissed.' }; + owner: 'rfeltis'; + comment: 'Tracks chat input notification visibility and user dismissals.'; +}; + const severityToClass: Record = { [ChatInputNotificationSeverity.Info]: 'severity-info', [ChatInputNotificationSeverity.Warning]: 'severity-warning', @@ -44,6 +56,7 @@ export class ChatInputNotificationWidget extends Disposable { readonly domNode: HTMLElement; private readonly _contentDisposables = this._register(new DisposableStore()); + private _lastShownTelemetryData: ChatInputNotificationTelemetryEvent | undefined; /** * Optional provider that returns the current session type of the owning @@ -82,11 +95,13 @@ export class ChatInputNotificationWidget extends Disposable { const notification = this._notificationService.getActiveNotification(n => this._matchesSession(n)); if (!notification) { this.domNode.parentElement?.classList.remove('has-notification'); + this._lastShownTelemetryData = undefined; return; } this.domNode.parentElement?.classList.add('has-notification'); this._renderNotification(notification); + this._logShownTelemetry(notification); } private _matchesSession(notification: IChatInputNotification): boolean { @@ -162,7 +177,10 @@ export class ChatInputNotificationWidget extends Disposable { // browser has finished propagating the click event. Otherwise // blur handlers fired by removing the button from focus can // move/remove nodes that `clearNode` then trips over. - const dismiss = () => queueMicrotask(() => this._notificationService.dismissNotification(notification.id)); + const dismiss = () => queueMicrotask(() => { + this._telemetryService.publicLog2('chatInputNotificationDismissed', this._getTelemetryData(notification)); + this._notificationService.dismissNotification(notification.id); + }); this._contentDisposables.add(dom.addDisposableListener(dismissButton, dom.EventType.CLICK, dismiss)); this._contentDisposables.add(dom.addDisposableListener(dismissButton, dom.EventType.KEY_DOWN, (e: KeyboardEvent) => { if (e.key === 'Enter' || e.key === ' ') { @@ -218,4 +236,20 @@ export class ChatInputNotificationWidget extends Disposable { } } } + + private _logShownTelemetry(notification: IChatInputNotification): void { + const data = this._getTelemetryData(notification); + if (this._lastShownTelemetryData?.id === data.id && this._lastShownTelemetryData.telemetryId === data.telemetryId) { + return; + } + this._lastShownTelemetryData = data; + this._telemetryService.publicLog2('chatInputNotificationShown', data); + } + + private _getTelemetryData(notification: IChatInputNotification): ChatInputNotificationTelemetryEvent { + return { + id: notification.id, + telemetryId: notification.telemetryId, + }; + } } diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts new file mode 100644 index 0000000000000..3a2aeaa9a8145 --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts @@ -0,0 +1,396 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { timeout } from '../../../../../base/common/async.js'; +import { Emitter, Event } from '../../../../../base/common/event.js'; +import type { IDisposable } from '../../../../../base/common/lifecycle.js'; +import { observableValue } from '../../../../../base/common/observable.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js'; +import { ICommandEvent, ICommandService } from '../../../../../platform/commands/common/commands.js'; +import { ExtensionIdentifier } from '../../../../../platform/extensions/common/extensions.js'; +import { SyncDescriptor } from '../../../../../platform/instantiation/common/descriptors.js'; +import { getSingletonServiceDescriptors } from '../../../../../platform/instantiation/common/extensions.js'; +import { ServiceCollection } from '../../../../../platform/instantiation/common/serviceCollection.js'; +import { IStorageService, StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js'; +import { ITelemetryData, ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; +import { TelemetryService } from '../../../../../platform/telemetry/common/telemetryService.js'; +import { ITelemetryAppender } from '../../../../../platform/telemetry/common/telemetryUtils.js'; +import { ChatEntitlement, IChatEntitlementService, IChatSentiment, IQuotaSnapshot } from '../../../../services/chat/common/chatEntitlementService.js'; +import { workbenchInstantiationService } from '../../../../test/browser/workbenchTestServices.js'; +import { ChatQuotaNotificationContribution } from '../../browser/chatQuotaNotification.js'; +import { ChatInputNotificationSeverity, IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; +import { ChatInputNotificationWidget } from '../../browser/widget/input/chatInputNotificationWidget.js'; +import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../common/languageModels.js'; + +interface ILoggedTelemetryEvent { + readonly eventName: string; + readonly data: ITelemetryData; +} + +class TestTelemetryAppender implements ITelemetryAppender { + readonly events: ILoggedTelemetryEvent[] = []; + + log(eventName: string, data: ITelemetryData): void { + this.events.push({ eventName, data }); + } + + flush(): Promise { + return Promise.resolve(); + } +} + +class TestCommandService implements ICommandService { + declare readonly _serviceBrand: undefined; + + readonly onWillExecuteCommand: Event = Event.None; + readonly onDidExecuteCommand: Event = Event.None; + readonly executedCommands: string[] = []; + + async executeCommand(commandId: string): Promise { + this.executedCommands.push(commandId); + return undefined; + } +} + +function createQuotaSnapshot(percentRemaining: number, opts?: Partial): IQuotaSnapshot { + return { + percentRemaining, + unlimited: false, + ...opts, + }; +} + +function createEntitlementService(opts?: { + entitlement?: ChatEntitlement; + quotas?: Partial; +}) { + const onDidChangeQuotaRemaining = new Emitter(); + const onDidChangeQuotaExceeded = new Emitter(); + const onDidChangeEntitlement = new Emitter(); + const sentiment: IChatSentiment = {}; + + const service: IChatEntitlementService = { + _serviceBrand: undefined, + entitlement: opts?.entitlement ?? ChatEntitlement.Pro, + entitlementObs: observableValue({}, opts?.entitlement ?? ChatEntitlement.Pro), + onDidChangeEntitlement: onDidChangeEntitlement.event, + onDidChangeQuotaExceeded: onDidChangeQuotaExceeded.event, + onDidChangeQuotaRemaining: onDidChangeQuotaRemaining.event, + onDidChangeUsageBasedBilling: Event.None, + quotas: { + usageBasedBilling: true, + premiumChat: createQuotaSnapshot(0), + additionalUsageEnabled: false, + ...opts?.quotas, + }, + organisations: undefined, + isInternal: false, + sku: undefined, + copilotTrackingId: undefined, + previewFeaturesDisabled: false, + clientByokEnabled: false, + hasByokModels: false, + onDidChangeSentiment: Event.None, + sentiment, + sentimentObs: observableValue({}, sentiment), + onDidChangeAnonymous: Event.None, + anonymous: false, + anonymousObs: observableValue({}, false), + acceptQuotas() { }, + clearQuotas() { }, + markAnonymousRateLimited() { }, + markSetupCompleted() { }, + setForceHidden() { }, + update() { return Promise.resolve(); }, + }; + + return { service, onDidChangeQuotaRemaining, onDidChangeQuotaExceeded, onDidChangeEntitlement }; +} + +suite('ChatQuotaNotificationContribution integration', () => { + const store = ensureNoDisposablesAreLeakedInTestSuite(); + + function createHarness(opts?: Parameters[0]) { + const instantiationService = store.add(workbenchInstantiationService(undefined, store)); + const telemetryAppender = new TestTelemetryAppender(); + const telemetryService = store.add(instantiationService.createInstance(TelemetryService, { appenders: [telemetryAppender] })); + const entitlementService = createEntitlementService(opts); + const commandService = new TestCommandService(); + const storageService = instantiationService.get(IStorageService); + storageService.store('chat.currentLanguageModel.panel', 'copilot/test-model', StorageScope.APPLICATION, StorageTarget.USER); + + instantiationService.stub(ITelemetryService, telemetryService); + instantiationService.stub(IChatEntitlementService, entitlementService.service); + instantiationService.stub(ICommandService, commandService); + instantiationService.stub(ILanguageModelsService, { + _serviceBrand: undefined, + onDidChangeLanguageModelVendors: Event.None, + onDidChangeLanguageModels: Event.None, + getLanguageModelIds: () => ['test-model'], + getVendors: () => [], + lookupLanguageModel: (_id: string): ILanguageModelChatMetadata => ({ + id: 'test-model', + name: 'Test Model', + vendor: 'copilot', + version: '1.0', + family: 'test', + extension: new ExtensionIdentifier('test.extension'), + maxInputTokens: 1, + maxOutputTokens: 1, + isDefaultForLocation: {}, + } satisfies ILanguageModelChatMetadata), + lookupLanguageModelByQualifiedName: () => undefined, + }); + + store.add(entitlementService.onDidChangeQuotaRemaining); + store.add(entitlementService.onDidChangeQuotaExceeded); + store.add(entitlementService.onDidChangeEntitlement); + + const notificationDescriptor = getSingletonServiceDescriptors().find(([id]) => id === IChatInputNotificationService)?.[1]; + assert.ok(notificationDescriptor); + const eagerNotificationDescriptor = new SyncDescriptor(notificationDescriptor.ctor, notificationDescriptor.staticArguments); + const childInstantiationService = store.add(instantiationService.createChild(new ServiceCollection([IChatInputNotificationService, eagerNotificationDescriptor]))); + const contribution = store.add(childInstantiationService.createInstance(ChatQuotaNotificationContribution)); + store.add(childInstantiationService.get(IChatInputNotificationService) as IChatInputNotificationService & IDisposable); + + return { instantiationService: childInstantiationService, telemetryAppender, commandService, contribution, entitlementService }; + } + + function getNotificationTelemetryEvents(telemetryAppender: TestTelemetryAppender): ILoggedTelemetryEvent[] { + return telemetryAppender.events.filter(e => e.eventName === 'chatInputNotificationShown' || e.eventName === 'chatInputNotificationDismissed'); + } + + function getRenderedText(widget: ChatInputNotificationWidget): string { + return widget.domNode.querySelector('.chat-input-notification')?.textContent ?? ''; + } + + function assertShownTelemetry(telemetryAppender: TestTelemetryAppender, telemetryId: string): void { + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), [{ + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId, + }, + }]); + } + + test('emits generic shown telemetry through the real widget render path', () => { + const { instantiationService, telemetryAppender } = createHarness(); + + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), []); + + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + assert.ok(widget.domNode.querySelector('.chat-input-notification')); + + assertShownTelemetry(telemetryAppender, 'quotaExhausted'); + }); + + test('emits shown telemetry for each quota notification variant through the real widget render path', () => { + const cases: readonly { + readonly name: string; + readonly setup: () => ReturnType; + readonly expectedText: string; + readonly expectedTelemetryId: string; + readonly afterCreate?: (harness: ReturnType) => void; + }[] = [ + { + name: 'quota exhausted', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(0), + additionalUsageEnabled: false, + }, + }), + expectedText: 'Credit Limit Reached', + expectedTelemetryId: 'quotaExhausted', + }, + { + name: 'overage activation', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(50), + additionalUsageEnabled: true, + }, + }), + expectedText: 'Additional budget is now covering extra usage.', + expectedTelemetryId: 'overageActivation', + afterCreate: ({ entitlementService }) => { + (entitlementService.service as IChatEntitlementService & { quotas: IChatEntitlementService['quotas'] }).quotas = { + ...entitlementService.service.quotas, + premiumChat: createQuotaSnapshot(0), + }; + entitlementService.onDidChangeQuotaRemaining.fire(); + }, + }, + { + name: 'quota approaching', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(50), + }, + }), + expectedText: 'Credits at 75%', + expectedTelemetryId: 'quotaApproaching', + afterCreate: ({ entitlementService }) => { + (entitlementService.service as IChatEntitlementService & { quotas: IChatEntitlementService['quotas'] }).quotas = { + ...entitlementService.service.quotas, + premiumChat: createQuotaSnapshot(25), + }; + entitlementService.onDidChangeQuotaRemaining.fire(); + }, + }, + { + name: 'rate limit warning', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(50), + sessionRateLimit: { percentRemaining: 50, unlimited: false, resetDate: '2026-06-01T00:00:00Z' }, + }, + }), + expectedText: 'You\'ve used 75% of your session rate limit.', + expectedTelemetryId: 'sessionRateLimitWarning', + afterCreate: ({ entitlementService }) => { + (entitlementService.service as IChatEntitlementService & { quotas: IChatEntitlementService['quotas'] }).quotas = { + ...entitlementService.service.quotas, + sessionRateLimit: { percentRemaining: 25, unlimited: false, resetDate: '2026-06-01T00:00:00Z' }, + }; + entitlementService.onDidChangeQuotaRemaining.fire(); + }, + }, + { + name: 'managed plan blocked', + setup: () => createHarness({ + entitlement: ChatEntitlement.Business, + quotas: { + premiumChat: createQuotaSnapshot(0, { + unlimited: true, + hasQuota: false, + }), + }, + }), + expectedText: 'Usage Blocked', + expectedTelemetryId: 'managedPlanBlocked', + }, + ]; + + const results = cases.map(testCase => { + const harness = testCase.setup(); + testCase.afterCreate?.(harness); + + const widget = store.add(harness.instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + const renderedText = getRenderedText(widget); + assert.ok(renderedText.includes(testCase.expectedText), `${testCase.name} did not render expected text`); + assertShownTelemetry(harness.telemetryAppender, testCase.expectedTelemetryId); + + return { + name: testCase.name, + renderedText: testCase.expectedText, + telemetry: getNotificationTelemetryEvents(harness.telemetryAppender), + }; + }); + + assert.deepStrictEqual(results.map(result => result.name), [ + 'quota exhausted', + 'overage activation', + 'quota approaching', + 'rate limit warning', + 'managed plan blocked', + ]); + }); + + test('emits shown telemetry when the same notification id changes telemetry context', () => { + const { instantiationService, telemetryAppender } = createHarness(); + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + const notificationService = instantiationService.get(IChatInputNotificationService); + + notificationService.setNotification({ + id: 'copilot.quotaStatus', + telemetryId: 'quotaApproaching', + severity: ChatInputNotificationSeverity.Info, + message: 'Credits at 75%', + description: undefined, + actions: [], + dismissible: true, + autoDismissOnMessage: true, + }); + + assert.ok(widget.domNode.querySelector('.chat-input-notification')); + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), [ + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', + }, + }, + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaApproaching', + }, + }, + ]); + }); + + test('does not emit duplicate shown telemetry when the notification rerenders unchanged', () => { + const { instantiationService, telemetryAppender } = createHarness(); + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + + widget.rerender(); + widget.rerender(); + + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), [ + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', + }, + }, + ]); + }); + + test('emits existing action telemetry and generic dismissed telemetry from real DOM interaction', async () => { + const { instantiationService, telemetryAppender, commandService } = createHarness(); + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + + const actionButton = widget.domNode.querySelector('.chat-input-notification-action-button'); + assert.ok(actionButton); + actionButton.click(); + await timeout(0); + + const dismissButton = widget.domNode.querySelector('.chat-input-notification-dismiss'); + assert.ok(dismissButton); + dismissButton.click(); + await timeout(0); + + assert.deepStrictEqual(commandService.executedCommands, ['workbench.action.chat.manageAdditionalSpend']); + assert.deepStrictEqual(telemetryAppender.events.filter(e => e.eventName === 'workbenchActionExecuted' || e.eventName === 'chatInputNotificationShown' || e.eventName === 'chatInputNotificationDismissed'), [ + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', + }, + }, + { + eventName: 'workbenchActionExecuted', + data: { + id: 'workbench.action.chat.manageAdditionalSpend', + from: 'chatInputNotification', + }, + }, + { + eventName: 'chatInputNotificationDismissed', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', + }, + }, + ]); + }); +}); diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts index 3467b84e4b53d..81108bd53e280 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts @@ -83,6 +83,7 @@ function createMockEntitlementService(opts?: { function createMockNotificationService() { let lastNotification: IChatInputNotification | undefined = undefined; let deleted = false; + let dismissed = false; let setCount = 0; const onDidChange = new Emitter(); @@ -95,22 +96,38 @@ function createMockNotificationService() { setNotification(notification: IChatInputNotification) { lastNotification = notification; deleted = false; + dismissed = false; setCount++; + onDidChange.fire(); }, deleteNotification(_id: string) { deleted = true; + dismissed = false; + onDidChange.fire(); + }, + dismissNotification(id: string) { + if (!lastNotification || lastNotification.id !== id || dismissed) { + return; + } + dismissed = true; + onDidDismiss.fire(id); + onDidChange.fire(); + }, + getActiveNotification(filter?: (notification: IChatInputNotification) => boolean) { + if (deleted || dismissed || !lastNotification) { + return undefined; + } + return !filter || filter(lastNotification) ? lastNotification : undefined; }, - dismissNotification() { }, - getActiveNotification() { return deleted ? undefined : lastNotification; }, handleMessageSent() { }, }; return { service, - getNotification(): IChatInputNotification | undefined { return deleted ? undefined : lastNotification; }, + getNotification(): IChatInputNotification | undefined { return deleted || dismissed ? undefined : lastNotification; }, get wasDeleted() { return deleted; }, get setCount() { return setCount; }, - reset() { lastNotification = undefined; deleted = false; setCount = 0; }, + reset() { lastNotification = undefined; deleted = false; dismissed = false; setCount = 0; }, }; }