From f1db8e5923bd086cb353ac12457b17ef50df54b7 Mon Sep 17 00:00:00 2001 From: Ralph Feltis Date: Wed, 17 Jun 2026 09:33:27 -0700 Subject: [PATCH 1/3] Add telemetry for chat quota notification banners Closes #320996 Adds telemetry events to track when users see and interact with consumption quota checkpoint notification banners: - chat.quotaNotificationShown: Emitted when banner renders - chat.quotaNotificationDismissed: Emitted when user dismisses - chat.quotaNotificationActionInvoked: Emitted when user clicks action Implementation adds lifecycle events (onDidShow, onDidAction, onDidDismiss) to ChatInputNotificationService and wires telemetry through the real widget render path with deduplication logic. Includes integration test using production services and DOM interactions to verify telemetry flows through the real pipeline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../chat/browser/chatQuotaNotification.ts | 125 +++++++++- .../input/chatInputNotificationService.ts | 43 ++++ .../input/chatInputNotificationWidget.ts | 2 + .../agentHostPermissionUiContribution.test.ts | 4 + .../chatQuotaNotification.integrationTest.ts | 234 ++++++++++++++++++ .../browser/chatQuotaNotification.test.ts | 210 +++++++++++++++- 6 files changed, 606 insertions(+), 12 deletions(-) create mode 100644 src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts diff --git a/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts b/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts index fb616327ca4e4..6373095cdca34 100644 --- a/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts +++ b/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts @@ -8,6 +8,7 @@ import { safeIntl } from '../../../../base/common/date.js'; import { localize } from '../../../../nls.js'; import { IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js'; import { IStorageService, StorageScope } from '../../../../platform/storage/common/storage.js'; +import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js'; import { IWorkbenchContribution } from '../../../common/contributions.js'; import { ChatEntitlement, IChatEntitlementService, IQuotaSnapshot, IRateLimitSnapshot } from '../../../services/chat/common/chatEntitlementService.js'; import { isSelectedModelCopilot, SELECTED_MODEL_STORAGE_KEY_PREFIX } from '../common/chatSelectedModel.js'; @@ -17,6 +18,42 @@ import { ChatInputNotificationSeverity, IChatInputNotification, IChatInputNotifi const QUOTA_NOTIFICATION_ID = 'copilot.quotaStatus'; const THRESHOLDS = [50, 75, 90, 95]; +type ChatQuotaNotificationType = 'quotaExhausted' | 'quotaApproaching' | 'overageActivation' | 'rateLimitWarning' | 'managedPlanBlocked'; +type ChatQuotaNotificationLimitType = 'quota' | 'sessionRateLimit' | 'weeklyRateLimit' | 'managedPlan'; + +type ChatQuotaNotificationTelemetryEvent = { + notificationType: ChatQuotaNotificationType; + limitType: ChatQuotaNotificationLimitType; + entitlement: string; + additionalUsageEnabled: boolean; + hasActions: boolean; + percentUsed?: number; +}; + +type ChatQuotaNotificationActionTelemetryEvent = ChatQuotaNotificationTelemetryEvent & { + commandId: string; +}; + +type ChatQuotaNotificationTelemetryClassificationProperties = { + notificationType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The quota notification variant associated with the event.' }; + limitType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The type of quota or rate limit represented by the notification.' }; + entitlement: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The current Copilot entitlement associated with the notification.' }; + additionalUsageEnabled: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether additional usage is enabled for the notification.' }; + hasActions: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the notification includes one or more action buttons.' }; + percentUsed?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The percentage of the quota or rate limit used, if available.' }; +}; + +type ChatQuotaNotificationTelemetryClassification = ChatQuotaNotificationTelemetryClassificationProperties & { + owner: 'rfeltis'; + comment: 'Tracks Copilot quota notification visibility and user dismissals.'; +}; + +type ChatQuotaNotificationActionTelemetryClassification = ChatQuotaNotificationTelemetryClassificationProperties & { + commandId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The command invoked from the quota notification action.' }; + owner: 'rfeltis'; + comment: 'Tracks actions invoked from Copilot quota notifications.'; +}; + /** * Core-side workbench contribution that shows chat input notifications for * quota exhaustion and quota-approaching thresholds. @@ -45,6 +82,8 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo private _prevAdditionalUsageEnabled: boolean | undefined; private _prevSessionPercentUsed: number | undefined; private _prevWeeklyPercentUsed: number | undefined; + private _activeNotificationTelemetryData: ChatQuotaNotificationTelemetryEvent | undefined; + private _lastShownTelemetrySignature: string | undefined; constructor( @IChatEntitlementService private readonly _chatEntitlementService: IChatEntitlementService, @@ -52,12 +91,21 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo @IContextKeyService private readonly _contextKeyService: IContextKeyService, @ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService, @IStorageService private readonly _storageService: IStorageService, + @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); this._register(this._chatEntitlementService.onDidChangeQuotaRemaining(() => this._update())); this._register(this._chatEntitlementService.onDidChangeQuotaExceeded(() => this._update())); this._register(this._chatEntitlementService.onDidChangeEntitlement(() => this._update())); + this._register(this._chatInputNotificationService.onDidShow(id => this._onDidShowNotification(id))); + this._register(this._chatInputNotificationService.onDidDismiss(id => this._onDidDismissNotification(id))); + this._register(this._chatInputNotificationService.onDidAction(e => this._onDidActionNotification(e.notificationId, e.commandId))); + this._register(this._chatInputNotificationService.onDidChange(() => { + if (!this._chatInputNotificationService.getActiveNotification(notification => notification.id === QUOTA_NOTIFICATION_ID)) { + this._clearTelemetryState(); + } + })); // Re-evaluate when the selected model changes (e.g. switching between Copilot and BYOK). // The chatModelId context key is widget-scoped and may not bubble to the global @@ -239,7 +287,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo actions, dismissible: true, autoDismissOnMessage: true, - }); + }, this._createTelemetryData('quotaExhausted', 'quota', 100, actions)); } // --- Overage notification ----------------------------------------------- @@ -255,7 +303,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo actions: [], dismissible: true, autoDismissOnMessage: true, - }); + }, this._createTelemetryData('overageActivation', 'quota', 100, [])); } // --- Quota approaching -------------------------------------------------- @@ -291,7 +339,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo actions, dismissible: true, autoDismissOnMessage: true, - }); + }, this._createTelemetryData('quotaApproaching', 'quota', warning.percentUsed, actions)); } // --- Rate-limit warning ------------------------------------------------- @@ -350,7 +398,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo actions: [], dismissible: true, autoDismissOnMessage: true, - }); + }, this._createTelemetryData('rateLimitWarning', warning.type === 'session' ? 'sessionRateLimit' : 'weeklyRateLimit', warning.percentUsed, [])); } // --- Helpers ------------------------------------------------------------ @@ -384,7 +432,7 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo actions: [], dismissible: true, autoDismissOnMessage: true, - }); + }, this._createTelemetryData('managedPlanBlocked', 'managedPlan', undefined, [])); } private _formatResetDate(isoDate: string): string { @@ -397,12 +445,77 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo ).value.format(resetDate); } - private _setNotification(notification: IChatInputNotification): void { + private _setNotification(notification: IChatInputNotification, telemetryData: ChatQuotaNotificationTelemetryEvent): void { + this._activeNotificationTelemetryData = telemetryData; this._chatInputNotificationService.setNotification(notification); } private _hideNotification(): void { this._showingExhausted = false; + this._clearTelemetryState(); this._chatInputNotificationService.deleteNotification(QUOTA_NOTIFICATION_ID); } + + private _createTelemetryData( + notificationType: ChatQuotaNotificationType, + limitType: ChatQuotaNotificationLimitType, + percentUsed: number | undefined, + actions: IChatInputNotification['actions'], + ): ChatQuotaNotificationTelemetryEvent { + return { + notificationType, + limitType, + entitlement: this._getEntitlementTelemetryValue(), + additionalUsageEnabled: this._chatEntitlementService.quotas.additionalUsageEnabled === true, + hasActions: actions.length > 0, + percentUsed, + }; + } + + private _getEntitlementTelemetryValue(): string { + return ChatEntitlement[this._chatEntitlementService.entitlement] ?? String(this._chatEntitlementService.entitlement); + } + + private _logShownTelemetry(data: ChatQuotaNotificationTelemetryEvent): void { + const signature = this._getTelemetrySignature(data); + if (signature === this._lastShownTelemetrySignature) { + return; + } + this._lastShownTelemetrySignature = signature; + this._telemetryService.publicLog2('chat.quotaNotificationShown', data); + } + + private _onDidShowNotification(id: string): void { + if (id !== QUOTA_NOTIFICATION_ID || !this._activeNotificationTelemetryData) { + return; + } + this._logShownTelemetry(this._activeNotificationTelemetryData); + } + + private _onDidActionNotification(id: string, commandId: string): void { + if (id !== QUOTA_NOTIFICATION_ID || !this._activeNotificationTelemetryData) { + return; + } + this._telemetryService.publicLog2('chat.quotaNotificationActionInvoked', { + ...this._activeNotificationTelemetryData, + commandId, + }); + } + + private _onDidDismissNotification(id: string): void { + if (id !== QUOTA_NOTIFICATION_ID || !this._activeNotificationTelemetryData) { + return; + } + this._telemetryService.publicLog2('chat.quotaNotificationDismissed', this._activeNotificationTelemetryData); + this._clearTelemetryState(); + } + + private _clearTelemetryState(): void { + this._activeNotificationTelemetryData = undefined; + this._lastShownTelemetrySignature = undefined; + } + + private _getTelemetrySignature(data: ChatQuotaNotificationTelemetryEvent): string { + return `${data.notificationType}\u0000${data.limitType}\u0000${data.entitlement}\u0000${data.additionalUsageEnabled}\u0000${data.hasActions}\u0000${data.percentUsed ?? ''}`; + } } diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts index 30ff5919e6aa9..5b28ba52ffdaa 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts @@ -54,6 +54,11 @@ export interface IChatInputNotification { readonly mute?: IChatInputNotificationMuteAction; } +export interface IChatInputNotificationActionEvent { + readonly notificationId: string; + readonly commandId: string; +} + export const IChatInputNotificationService = createDecorator('chatInputNotificationService'); export interface IChatInputNotificationService { @@ -64,6 +69,12 @@ export interface IChatInputNotificationService { /** Fires when a notification is dismissed by the user (via the X button). */ readonly onDidDismiss: Event; + /** Fires when a notification is rendered by a chat input widget. */ + readonly onDidShow: Event; + + /** Fires when a notification action is invoked by the user. */ + readonly onDidAction: Event; + /** * Set or update a notification. If a notification with the same ID already * exists, its content is replaced and any previous user dismissal is cleared. @@ -81,6 +92,16 @@ export interface IChatInputNotificationService { */ dismissNotification(id: string): void; + /** + * Mark a notification as rendered by a chat input widget. + */ + showNotification(id: string): void; + + /** + * Mark a notification action as invoked by the user. + */ + actionNotification(id: string, commandId: string): void; + /** * Get the single active notification to display. Returns the highest-severity * notification that has not been dismissed. Ties are broken by most-recent insertion. @@ -101,6 +122,7 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif private readonly _notifications = new Map(); private readonly _dismissed = new Set(); + private readonly _shown = new Set(); /** Insertion order tracking — higher index = more recently set. */ private readonly _insertionOrder = new Map(); @@ -112,6 +134,12 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif private readonly _onDidDismiss = this._register(new Emitter()); readonly onDidDismiss = this._onDidDismiss.event; + private readonly _onDidShow = this._register(new Emitter()); + readonly onDidShow = this._onDidShow.event; + + private readonly _onDidAction = this._register(new Emitter()); + readonly onDidAction = this._onDidAction.event; + /** * Signature of the last active notification we announced via ARIA, so we * don't re-announce the same content when the model fires `onDidChange` @@ -122,6 +150,7 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif setNotification(notification: IChatInputNotification): void { this._notifications.set(notification.id, notification); this._dismissed.delete(notification.id); + this._shown.delete(notification.id); this._insertionOrder.set(notification.id, this._insertionCounter++); this._fireDidChange(); } @@ -129,6 +158,7 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif deleteNotification(id: string): void { if (this._notifications.delete(id)) { this._dismissed.delete(id); + this._shown.delete(id); this._insertionOrder.delete(id); this._fireDidChange(); } @@ -142,6 +172,19 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif } } + showNotification(id: string): void { + if (this._notifications.has(id) && !this._dismissed.has(id) && !this._shown.has(id)) { + this._shown.add(id); + this._onDidShow.fire(id); + } + } + + actionNotification(id: string, commandId: string): void { + if (this._notifications.has(id) && !this._dismissed.has(id)) { + this._onDidAction.fire({ notificationId: id, commandId }); + } + } + getActiveNotification(filter?: (notification: IChatInputNotification) => boolean): IChatInputNotification | undefined { let best: IChatInputNotification | undefined; let bestOrder = -1; diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts index d95a03a7d6c15..b327b668f1253 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts @@ -87,6 +87,7 @@ export class ChatInputNotificationWidget extends Disposable { this.domNode.parentElement?.classList.add('has-notification'); this._renderNotification(notification); + this._notificationService.showNotification(notification.id); } private _matchesSession(notification: IChatInputNotification): boolean { @@ -208,6 +209,7 @@ export class ChatInputNotificationWidget extends Disposable { button.element.ariaLabel = `${ariaTitle} ${action.label}`; this._contentDisposables.add(button.onDidClick(async () => { + this._notificationService.actionNotification(notification.id, action.commandId); this._telemetryService.publicLog2('workbenchActionExecuted', { id: action.commandId, from: 'chatInputNotification', diff --git a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts index 087fac8c787e7..2aca5b89ef139 100644 --- a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts @@ -50,6 +50,8 @@ class FakeNotificationService implements IChatInputNotificationService { declare readonly _serviceBrand: undefined; readonly onDidChange: Event = Event.None; readonly onDidDismiss: Event = Event.None; + readonly onDidShow: Event = Event.None; + readonly onDidAction: Event<{ notificationId: string; commandId: string }> = Event.None; readonly setCalls: IChatInputNotification[] = []; readonly deleteCalls: string[] = []; @@ -60,6 +62,8 @@ class FakeNotificationService implements IChatInputNotificationService { this.deleteCalls.push(id); } dismissNotification(_id: string): void { /* */ } + showNotification(_id: string): void { /* */ } + actionNotification(_id: string, _commandId: string): void { /* */ } getActiveNotification(): IChatInputNotification | undefined { return undefined; } handleMessageSent(): void { /* */ } } diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts new file mode 100644 index 0000000000000..73ffd4716cdae --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts @@ -0,0 +1,234 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { timeout } from '../../../../../base/common/async.js'; +import { Emitter, Event } from '../../../../../base/common/event.js'; +import type { IDisposable } from '../../../../../base/common/lifecycle.js'; +import { observableValue } from '../../../../../base/common/observable.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js'; +import { ICommandEvent, ICommandService } from '../../../../../platform/commands/common/commands.js'; +import { ExtensionIdentifier } from '../../../../../platform/extensions/common/extensions.js'; +import { SyncDescriptor } from '../../../../../platform/instantiation/common/descriptors.js'; +import { getSingletonServiceDescriptors } from '../../../../../platform/instantiation/common/extensions.js'; +import { ServiceCollection } from '../../../../../platform/instantiation/common/serviceCollection.js'; +import { IStorageService, StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js'; +import { ITelemetryData, ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; +import { TelemetryService } from '../../../../../platform/telemetry/common/telemetryService.js'; +import { ITelemetryAppender } from '../../../../../platform/telemetry/common/telemetryUtils.js'; +import { ChatEntitlement, IChatEntitlementService, IChatSentiment, IQuotaSnapshot } from '../../../../services/chat/common/chatEntitlementService.js'; +import { workbenchInstantiationService } from '../../../../test/browser/workbenchTestServices.js'; +import { ChatQuotaNotificationContribution } from '../../browser/chatQuotaNotification.js'; +import { IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; +import { ChatInputNotificationWidget } from '../../browser/widget/input/chatInputNotificationWidget.js'; +import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../common/languageModels.js'; + +interface ILoggedTelemetryEvent { + readonly eventName: string; + readonly data: ITelemetryData; +} + +class TestTelemetryAppender implements ITelemetryAppender { + readonly events: ILoggedTelemetryEvent[] = []; + + log(eventName: string, data: ITelemetryData): void { + this.events.push({ eventName, data }); + } + + flush(): Promise { + return Promise.resolve(); + } +} + +class TestCommandService implements ICommandService { + declare readonly _serviceBrand: undefined; + + readonly onWillExecuteCommand: Event = Event.None; + readonly onDidExecuteCommand: Event = Event.None; + readonly executedCommands: string[] = []; + + async executeCommand(commandId: string): Promise { + this.executedCommands.push(commandId); + return undefined; + } +} + +function createQuotaSnapshot(percentRemaining: number): IQuotaSnapshot { + return { + percentRemaining, + unlimited: false, + }; +} + +function createEntitlementService() { + const onDidChangeQuotaRemaining = new Emitter(); + const onDidChangeQuotaExceeded = new Emitter(); + const onDidChangeEntitlement = new Emitter(); + const sentiment: IChatSentiment = {}; + + const service: IChatEntitlementService = { + _serviceBrand: undefined, + entitlement: ChatEntitlement.Pro, + entitlementObs: observableValue({}, ChatEntitlement.Pro), + onDidChangeEntitlement: onDidChangeEntitlement.event, + onDidChangeQuotaExceeded: onDidChangeQuotaExceeded.event, + onDidChangeQuotaRemaining: onDidChangeQuotaRemaining.event, + onDidChangeUsageBasedBilling: Event.None, + quotas: { + usageBasedBilling: true, + premiumChat: createQuotaSnapshot(0), + additionalUsageEnabled: false, + }, + organisations: undefined, + isInternal: false, + sku: undefined, + copilotTrackingId: undefined, + previewFeaturesDisabled: false, + clientByokEnabled: false, + hasByokModels: false, + onDidChangeSentiment: Event.None, + sentiment, + sentimentObs: observableValue({}, sentiment), + onDidChangeAnonymous: Event.None, + anonymous: false, + anonymousObs: observableValue({}, false), + acceptQuotas() { }, + clearQuotas() { }, + markAnonymousRateLimited() { }, + markSetupCompleted() { }, + setForceHidden() { }, + update() { return Promise.resolve(); }, + }; + + return { service, onDidChangeQuotaRemaining, onDidChangeQuotaExceeded, onDidChangeEntitlement }; +} + +suite('ChatQuotaNotificationContribution integration', () => { + const store = ensureNoDisposablesAreLeakedInTestSuite(); + + function createHarness() { + const instantiationService = store.add(workbenchInstantiationService(undefined, store)); + const telemetryAppender = new TestTelemetryAppender(); + const telemetryService = store.add(instantiationService.createInstance(TelemetryService, { appenders: [telemetryAppender] })); + const entitlementService = createEntitlementService(); + const commandService = new TestCommandService(); + const storageService = instantiationService.get(IStorageService); + storageService.store('chat.currentLanguageModel.panel', 'copilot/test-model', StorageScope.APPLICATION, StorageTarget.USER); + + instantiationService.stub(ITelemetryService, telemetryService); + instantiationService.stub(IChatEntitlementService, entitlementService.service); + instantiationService.stub(ICommandService, commandService); + instantiationService.stub(ILanguageModelsService, { + _serviceBrand: undefined, + onDidChangeLanguageModelVendors: Event.None, + onDidChangeLanguageModels: Event.None, + getLanguageModelIds: () => ['test-model'], + getVendors: () => [], + lookupLanguageModel: (_id: string): ILanguageModelChatMetadata => ({ + id: 'test-model', + name: 'Test Model', + vendor: 'copilot', + version: '1.0', + family: 'test', + extension: new ExtensionIdentifier('test.extension'), + maxInputTokens: 1, + maxOutputTokens: 1, + isDefaultForLocation: {}, + } satisfies ILanguageModelChatMetadata), + lookupLanguageModelByQualifiedName: () => undefined, + }); + + store.add(entitlementService.onDidChangeQuotaRemaining); + store.add(entitlementService.onDidChangeQuotaExceeded); + store.add(entitlementService.onDidChangeEntitlement); + + const notificationDescriptor = getSingletonServiceDescriptors().find(([id]) => id === IChatInputNotificationService)?.[1]; + assert.ok(notificationDescriptor); + const eagerNotificationDescriptor = new SyncDescriptor(notificationDescriptor.ctor, notificationDescriptor.staticArguments); + const childInstantiationService = store.add(instantiationService.createChild(new ServiceCollection([IChatInputNotificationService, eagerNotificationDescriptor]))); + const contribution = store.add(childInstantiationService.createInstance(ChatQuotaNotificationContribution)); + store.add(childInstantiationService.get(IChatInputNotificationService) as IChatInputNotificationService & IDisposable); + + return { instantiationService: childInstantiationService, telemetryAppender, commandService, contribution }; + } + + function getQuotaTelemetryEvents(telemetryAppender: TestTelemetryAppender): ILoggedTelemetryEvent[] { + return telemetryAppender.events.filter(e => e.eventName.startsWith('chat.quotaNotification')); + } + + test('emits shown telemetry through the real widget render path', () => { + const { instantiationService, telemetryAppender } = createHarness(); + + assert.deepStrictEqual(getQuotaTelemetryEvents(telemetryAppender), []); + + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + assert.ok(widget.domNode.querySelector('.chat-input-notification')); + + assert.deepStrictEqual(getQuotaTelemetryEvents(telemetryAppender), [{ + eventName: 'chat.quotaNotificationShown', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }, + }]); + }); + + test('emits action and dismissed telemetry from real DOM interaction', async () => { + const { instantiationService, telemetryAppender, commandService } = createHarness(); + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + + const actionButton = widget.domNode.querySelector('.chat-input-notification-action-button'); + assert.ok(actionButton); + actionButton.click(); + await timeout(0); + + const dismissButton = widget.domNode.querySelector('.chat-input-notification-dismiss'); + assert.ok(dismissButton); + dismissButton.click(); + await timeout(0); + + assert.deepStrictEqual(commandService.executedCommands, ['workbench.action.chat.manageAdditionalSpend']); + assert.deepStrictEqual(getQuotaTelemetryEvents(telemetryAppender), [ + { + eventName: 'chat.quotaNotificationShown', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }, + }, + { + eventName: 'chat.quotaNotificationActionInvoked', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + commandId: 'workbench.action.chat.manageAdditionalSpend', + }, + }, + { + eventName: 'chat.quotaNotificationDismissed', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }, + }, + ]); + }); +}); diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts index 3467b84e4b53d..4636a80169237 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts @@ -10,10 +10,12 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/tes import { IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js'; import { MockContextKeyService } from '../../../../../platform/keybinding/test/common/mockKeybindingService.js'; import { InMemoryStorageService, StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js'; +import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; +import { NullTelemetryService } from '../../../../../platform/telemetry/common/telemetryUtils.js'; import { ChatEntitlement, IChatEntitlementService, IChatSentiment, IQuotaSnapshot, IRateLimitSnapshot } from '../../../../services/chat/common/chatEntitlementService.js'; import { ChatQuotaNotificationContribution } from '../../browser/chatQuotaNotification.js'; import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../common/languageModels.js'; -import { IChatInputNotification, IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; +import { IChatInputNotification, IChatInputNotificationActionEvent, IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; // --- Mock IChatEntitlementService ------------------------------------------- @@ -83,37 +85,103 @@ function createMockEntitlementService(opts?: { function createMockNotificationService() { let lastNotification: IChatInputNotification | undefined = undefined; let deleted = false; + let dismissed = false; + let shown = false; let setCount = 0; const onDidChange = new Emitter(); const onDidDismiss = new Emitter(); + const onDidShow = new Emitter(); + const onDidAction = new Emitter(); const service: IChatInputNotificationService = { _serviceBrand: undefined, onDidChange: onDidChange.event, onDidDismiss: onDidDismiss.event, + onDidShow: onDidShow.event, + onDidAction: onDidAction.event, setNotification(notification: IChatInputNotification) { lastNotification = notification; deleted = false; + dismissed = false; + shown = false; setCount++; + onDidChange.fire(); }, deleteNotification(_id: string) { deleted = true; + dismissed = false; + shown = false; + onDidChange.fire(); + }, + dismissNotification(id: string) { + if (!lastNotification || lastNotification.id !== id || dismissed) { + return; + } + dismissed = true; + onDidDismiss.fire(id); + onDidChange.fire(); + }, + showNotification(id: string) { + if (!lastNotification || lastNotification.id !== id || dismissed || shown) { + return; + } + shown = true; + onDidShow.fire(id); + }, + actionNotification(id: string, commandId: string) { + if (!lastNotification || lastNotification.id !== id || dismissed) { + return; + } + onDidAction.fire({ notificationId: id, commandId }); + }, + getActiveNotification(filter?: (notification: IChatInputNotification) => boolean) { + if (deleted || dismissed || !lastNotification) { + return undefined; + } + return !filter || filter(lastNotification) ? lastNotification : undefined; }, - dismissNotification() { }, - getActiveNotification() { return deleted ? undefined : lastNotification; }, handleMessageSent() { }, }; return { service, - getNotification(): IChatInputNotification | undefined { return deleted ? undefined : lastNotification; }, + getNotification(): IChatInputNotification | undefined { return deleted || dismissed ? undefined : lastNotification; }, get wasDeleted() { return deleted; }, get setCount() { return setCount; }, - reset() { lastNotification = undefined; deleted = false; setCount = 0; }, + reset() { lastNotification = undefined; deleted = false; dismissed = false; shown = false; setCount = 0; }, }; } +// --- Mock ITelemetryService ------------------------------------------------- + +interface ILoggedTelemetryData { + readonly notificationType?: string; + readonly limitType?: string; + readonly entitlement?: string; + readonly additionalUsageEnabled?: boolean; + readonly hasActions?: boolean; + readonly percentUsed?: number; + readonly commandId?: string; +} + +interface ILoggedTelemetryEvent { + readonly eventName: string; + readonly data: ILoggedTelemetryData | undefined; +} + +function createMockTelemetryService() { + const events: ILoggedTelemetryEvent[] = []; + const service = { + ...NullTelemetryService, + publicLog2(eventName: string, data?: ILoggedTelemetryData): void { + events.push({ eventName, data }); + }, + } as ITelemetryService; + + return { service, events }; +} + // --- Helpers --------------------------------------------------------------- function makeQuotaSnapshot(percentRemaining: number, opts?: Partial): IQuotaSnapshot { @@ -142,6 +210,7 @@ suite('ChatQuotaNotificationContribution', () => { function createContribution(entitlementOpts?: Parameters[0], modelOpts?: { vendor?: string }) { const entitlementMock = createMockEntitlementService(entitlementOpts); const notificationMock = createMockNotificationService(); + const telemetryMock = createMockTelemetryService(); const contextKeyService = store.add(new MockContextKeyService()); const storageService = store.add(new InMemoryStorageService()); const vendor = modelOpts?.vendor ?? 'copilot'; @@ -169,9 +238,10 @@ suite('ChatQuotaNotificationContribution', () => { contextKeyService as IContextKeyService, languageModelsService, storageService, + telemetryMock.service, )); - return { contribution, entitlementMock, notificationMock, storageService }; + return { contribution, entitlementMock, notificationMock, storageService, telemetryMock }; } function updateQuotas( @@ -187,6 +257,132 @@ suite('ChatQuotaNotificationContribution', () => { entitlementMock.onDidChangeQuotaRemaining.fire(); } + // --- Telemetry ----------------------------------------------------------- + + suite('telemetry', () => { + test('logs shown telemetry when quota notification is rendered', () => { + const { notificationMock, telemetryMock } = createContribution({ + quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, + }); + + assert.deepStrictEqual(telemetryMock.events, []); + + notificationMock.service.showNotification('copilot.quotaStatus'); + + assert.deepStrictEqual(telemetryMock.events, [{ + eventName: 'chat.quotaNotificationShown', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }, + }]); + }); + + test('does not duplicate shown telemetry while the same notification remains active', () => { + const { entitlementMock, notificationMock, telemetryMock } = createContribution({ + quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, + }); + + notificationMock.service.showNotification('copilot.quotaStatus'); + updateQuotas(entitlementMock, { premiumChat: makeQuotaSnapshot(0) }); + notificationMock.service.showNotification('copilot.quotaStatus'); + + assert.deepStrictEqual(telemetryMock.events, [{ + eventName: 'chat.quotaNotificationShown', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }, + }]); + }); + + test('logs dismissed telemetry when quota notification is dismissed', () => { + const { notificationMock, telemetryMock } = createContribution({ + quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, + }); + + notificationMock.service.showNotification('copilot.quotaStatus'); + notificationMock.service.dismissNotification('copilot.quotaStatus'); + + const expectedData = { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }; + assert.deepStrictEqual(telemetryMock.events, [ + { eventName: 'chat.quotaNotificationShown', data: expectedData }, + { eventName: 'chat.quotaNotificationDismissed', data: expectedData }, + ]); + }); + + test('logs action telemetry when quota notification action is invoked', () => { + const { notificationMock, telemetryMock } = createContribution({ + quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, + }); + + notificationMock.service.showNotification('copilot.quotaStatus'); + notificationMock.service.actionNotification('copilot.quotaStatus', 'workbench.action.chat.manageAdditionalSpend'); + + assert.deepStrictEqual(telemetryMock.events, [ + { + eventName: 'chat.quotaNotificationShown', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + }, + }, + { + eventName: 'chat.quotaNotificationActionInvoked', + data: { + notificationType: 'quotaExhausted', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 100, + commandId: 'workbench.action.chat.manageAdditionalSpend', + }, + }, + ]); + }); + + test('logs approaching notification details', () => { + const { entitlementMock, notificationMock, telemetryMock } = createContribution({ + quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(60) }, + }); + + updateQuotas(entitlementMock, { premiumChat: makeQuotaSnapshot(50) }); + notificationMock.service.showNotification('copilot.quotaStatus'); + + assert.deepStrictEqual(telemetryMock.events, [{ + eventName: 'chat.quotaNotificationShown', + data: { + notificationType: 'quotaApproaching', + limitType: 'quota', + entitlement: 'Pro', + additionalUsageEnabled: false, + hasActions: true, + percentUsed: 50, + }, + }]); + }); + }); + // --- Quota exhausted --------------------------------------------------- suite('quota exhausted', () => { @@ -607,6 +803,7 @@ suite('ChatQuotaNotificationContribution', () => { quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, }); const notificationMock = createMockNotificationService(); + const telemetryMock = createMockTelemetryService(); const contextKeyService = store.add(new MockContextKeyService()); const storageService = store.add(new InMemoryStorageService()); // Start with BYOK model @@ -632,6 +829,7 @@ suite('ChatQuotaNotificationContribution', () => { contextKeyService as IContextKeyService, languageModelsService, storageService, + telemetryMock.service, )); // Initially deferred — BYOK model From 7b366deb4542ac9293c8a5504621ca30090a086c Mon Sep 17 00:00:00 2001 From: Ralph Feltis Date: Wed, 17 Jun 2026 11:41:40 -0700 Subject: [PATCH 2/3] Address PR review comments - Add validation that commandId exists in notification actions before emitting event - Guard mock service methods against deleted state to match real service behavior Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../browser/widget/input/chatInputNotificationService.ts | 9 +++++++-- .../chat/test/browser/chatQuotaNotification.test.ts | 8 ++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts index 5b28ba52ffdaa..903f3d51a81ec 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts @@ -180,8 +180,13 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif } actionNotification(id: string, commandId: string): void { - if (this._notifications.has(id) && !this._dismissed.has(id)) { - this._onDidAction.fire({ notificationId: id, commandId }); + const notification = this._notifications.get(id); + if (notification && !this._dismissed.has(id)) { + // Validate that the commandId exists in the notification's actions + const hasAction = notification.actions.some(a => a.commandId === commandId); + if (hasAction) { + this._onDidAction.fire({ notificationId: id, commandId }); + } } } diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts index 4636a80169237..db4bd992bbc96 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts @@ -130,10 +130,14 @@ function createMockNotificationService() { onDidShow.fire(id); }, actionNotification(id: string, commandId: string) { - if (!lastNotification || lastNotification.id !== id || dismissed) { + if (!lastNotification || lastNotification.id !== id || dismissed || deleted) { return; } - onDidAction.fire({ notificationId: id, commandId }); + // Validate that the commandId exists in the notification's actions + const hasAction = lastNotification.actions.some(a => a.commandId === commandId); + if (hasAction) { + onDidAction.fire({ notificationId: id, commandId }); + } }, getActiveNotification(filter?: (notification: IChatInputNotification) => boolean) { if (deleted || dismissed || !lastNotification) { From c51cb1897e74b9215da53a12585d38e1321a17ae Mon Sep 17 00:00:00 2001 From: Ralph Feltis Date: Fri, 19 Jun 2026 09:25:34 -0700 Subject: [PATCH 3/3] Move chat notification telemetry to widget Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../chat/browser/chatQuotaNotification.ts | 130 +-------- .../input/chatInputNotificationService.ts | 49 +--- .../input/chatInputNotificationWidget.ts | 38 ++- .../agentHostPermissionUiContribution.test.ts | 4 - .../chatQuotaNotification.integrationTest.ts | 254 ++++++++++++++---- .../browser/chatQuotaNotification.test.ts | 191 +------------ 6 files changed, 258 insertions(+), 408 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts b/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts index 6373095cdca34..57d4bcd91dad4 100644 --- a/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts +++ b/src/vs/workbench/contrib/chat/browser/chatQuotaNotification.ts @@ -8,7 +8,6 @@ import { safeIntl } from '../../../../base/common/date.js'; import { localize } from '../../../../nls.js'; import { IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js'; import { IStorageService, StorageScope } from '../../../../platform/storage/common/storage.js'; -import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js'; import { IWorkbenchContribution } from '../../../common/contributions.js'; import { ChatEntitlement, IChatEntitlementService, IQuotaSnapshot, IRateLimitSnapshot } from '../../../services/chat/common/chatEntitlementService.js'; import { isSelectedModelCopilot, SELECTED_MODEL_STORAGE_KEY_PREFIX } from '../common/chatSelectedModel.js'; @@ -18,42 +17,6 @@ import { ChatInputNotificationSeverity, IChatInputNotification, IChatInputNotifi const QUOTA_NOTIFICATION_ID = 'copilot.quotaStatus'; const THRESHOLDS = [50, 75, 90, 95]; -type ChatQuotaNotificationType = 'quotaExhausted' | 'quotaApproaching' | 'overageActivation' | 'rateLimitWarning' | 'managedPlanBlocked'; -type ChatQuotaNotificationLimitType = 'quota' | 'sessionRateLimit' | 'weeklyRateLimit' | 'managedPlan'; - -type ChatQuotaNotificationTelemetryEvent = { - notificationType: ChatQuotaNotificationType; - limitType: ChatQuotaNotificationLimitType; - entitlement: string; - additionalUsageEnabled: boolean; - hasActions: boolean; - percentUsed?: number; -}; - -type ChatQuotaNotificationActionTelemetryEvent = ChatQuotaNotificationTelemetryEvent & { - commandId: string; -}; - -type ChatQuotaNotificationTelemetryClassificationProperties = { - notificationType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The quota notification variant associated with the event.' }; - limitType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The type of quota or rate limit represented by the notification.' }; - entitlement: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The current Copilot entitlement associated with the notification.' }; - additionalUsageEnabled: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether additional usage is enabled for the notification.' }; - hasActions: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the notification includes one or more action buttons.' }; - percentUsed?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The percentage of the quota or rate limit used, if available.' }; -}; - -type ChatQuotaNotificationTelemetryClassification = ChatQuotaNotificationTelemetryClassificationProperties & { - owner: 'rfeltis'; - comment: 'Tracks Copilot quota notification visibility and user dismissals.'; -}; - -type ChatQuotaNotificationActionTelemetryClassification = ChatQuotaNotificationTelemetryClassificationProperties & { - commandId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The command invoked from the quota notification action.' }; - owner: 'rfeltis'; - comment: 'Tracks actions invoked from Copilot quota notifications.'; -}; - /** * Core-side workbench contribution that shows chat input notifications for * quota exhaustion and quota-approaching thresholds. @@ -82,8 +45,6 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo private _prevAdditionalUsageEnabled: boolean | undefined; private _prevSessionPercentUsed: number | undefined; private _prevWeeklyPercentUsed: number | undefined; - private _activeNotificationTelemetryData: ChatQuotaNotificationTelemetryEvent | undefined; - private _lastShownTelemetrySignature: string | undefined; constructor( @IChatEntitlementService private readonly _chatEntitlementService: IChatEntitlementService, @@ -91,21 +52,12 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo @IContextKeyService private readonly _contextKeyService: IContextKeyService, @ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService, @IStorageService private readonly _storageService: IStorageService, - @ITelemetryService private readonly _telemetryService: ITelemetryService, ) { super(); this._register(this._chatEntitlementService.onDidChangeQuotaRemaining(() => this._update())); this._register(this._chatEntitlementService.onDidChangeQuotaExceeded(() => this._update())); this._register(this._chatEntitlementService.onDidChangeEntitlement(() => this._update())); - this._register(this._chatInputNotificationService.onDidShow(id => this._onDidShowNotification(id))); - this._register(this._chatInputNotificationService.onDidDismiss(id => this._onDidDismissNotification(id))); - this._register(this._chatInputNotificationService.onDidAction(e => this._onDidActionNotification(e.notificationId, e.commandId))); - this._register(this._chatInputNotificationService.onDidChange(() => { - if (!this._chatInputNotificationService.getActiveNotification(notification => notification.id === QUOTA_NOTIFICATION_ID)) { - this._clearTelemetryState(); - } - })); // Re-evaluate when the selected model changes (e.g. switching between Copilot and BYOK). // The chatModelId context key is widget-scoped and may not bubble to the global @@ -281,13 +233,14 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'quotaExhausted', severity: ChatInputNotificationSeverity.Info, message: localize('quota.exhausted.title', "Credit Limit Reached"), description, actions, dismissible: true, autoDismissOnMessage: true, - }, this._createTelemetryData('quotaExhausted', 'quota', 100, actions)); + }); } // --- Overage notification ----------------------------------------------- @@ -297,13 +250,14 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'overageActivation', severity: ChatInputNotificationSeverity.Info, message: localize('quota.overage.title', "Credit Limit Reached"), description: localize('quota.overage.desc', "Additional budget is now covering extra usage."), actions: [], dismissible: true, autoDismissOnMessage: true, - }, this._createTelemetryData('overageActivation', 'quota', 100, [])); + }); } // --- Quota approaching -------------------------------------------------- @@ -333,13 +287,14 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'quotaApproaching', severity: ChatInputNotificationSeverity.Info, message: localize('quota.approaching.title', "Credits at {0}%", warning.percentUsed), description, actions, dismissible: true, autoDismissOnMessage: true, - }, this._createTelemetryData('quotaApproaching', 'quota', warning.percentUsed, actions)); + }); } // --- Rate-limit warning ------------------------------------------------- @@ -392,13 +347,14 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: warning.type === 'session' ? 'sessionRateLimitWarning' : 'weeklyRateLimitWarning', severity: ChatInputNotificationSeverity.Info, message, description, actions: [], dismissible: true, autoDismissOnMessage: true, - }, this._createTelemetryData('rateLimitWarning', warning.type === 'session' ? 'sessionRateLimit' : 'weeklyRateLimit', warning.percentUsed, [])); + }); } // --- Helpers ------------------------------------------------------------ @@ -426,13 +382,14 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo this._setNotification({ id: QUOTA_NOTIFICATION_ID, + telemetryId: 'managedPlanBlocked', severity: ChatInputNotificationSeverity.Info, message: localize('quota.blocked.managed.title', "Usage Blocked"), description: localize('quota.blocked.managed', "Your organization or enterprise has exceeded its Copilot budget. Contact your admin to resume usage."), actions: [], dismissible: true, autoDismissOnMessage: true, - }, this._createTelemetryData('managedPlanBlocked', 'managedPlan', undefined, [])); + }); } private _formatResetDate(isoDate: string): string { @@ -445,77 +402,12 @@ export class ChatQuotaNotificationContribution extends Disposable implements IWo ).value.format(resetDate); } - private _setNotification(notification: IChatInputNotification, telemetryData: ChatQuotaNotificationTelemetryEvent): void { - this._activeNotificationTelemetryData = telemetryData; + private _setNotification(notification: IChatInputNotification): void { this._chatInputNotificationService.setNotification(notification); } private _hideNotification(): void { this._showingExhausted = false; - this._clearTelemetryState(); this._chatInputNotificationService.deleteNotification(QUOTA_NOTIFICATION_ID); } - - private _createTelemetryData( - notificationType: ChatQuotaNotificationType, - limitType: ChatQuotaNotificationLimitType, - percentUsed: number | undefined, - actions: IChatInputNotification['actions'], - ): ChatQuotaNotificationTelemetryEvent { - return { - notificationType, - limitType, - entitlement: this._getEntitlementTelemetryValue(), - additionalUsageEnabled: this._chatEntitlementService.quotas.additionalUsageEnabled === true, - hasActions: actions.length > 0, - percentUsed, - }; - } - - private _getEntitlementTelemetryValue(): string { - return ChatEntitlement[this._chatEntitlementService.entitlement] ?? String(this._chatEntitlementService.entitlement); - } - - private _logShownTelemetry(data: ChatQuotaNotificationTelemetryEvent): void { - const signature = this._getTelemetrySignature(data); - if (signature === this._lastShownTelemetrySignature) { - return; - } - this._lastShownTelemetrySignature = signature; - this._telemetryService.publicLog2('chat.quotaNotificationShown', data); - } - - private _onDidShowNotification(id: string): void { - if (id !== QUOTA_NOTIFICATION_ID || !this._activeNotificationTelemetryData) { - return; - } - this._logShownTelemetry(this._activeNotificationTelemetryData); - } - - private _onDidActionNotification(id: string, commandId: string): void { - if (id !== QUOTA_NOTIFICATION_ID || !this._activeNotificationTelemetryData) { - return; - } - this._telemetryService.publicLog2('chat.quotaNotificationActionInvoked', { - ...this._activeNotificationTelemetryData, - commandId, - }); - } - - private _onDidDismissNotification(id: string): void { - if (id !== QUOTA_NOTIFICATION_ID || !this._activeNotificationTelemetryData) { - return; - } - this._telemetryService.publicLog2('chat.quotaNotificationDismissed', this._activeNotificationTelemetryData); - this._clearTelemetryState(); - } - - private _clearTelemetryState(): void { - this._activeNotificationTelemetryData = undefined; - this._lastShownTelemetrySignature = undefined; - } - - private _getTelemetrySignature(data: ChatQuotaNotificationTelemetryEvent): string { - return `${data.notificationType}\u0000${data.limitType}\u0000${data.entitlement}\u0000${data.additionalUsageEnabled}\u0000${data.hasActions}\u0000${data.percentUsed ?? ''}`; - } } diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts index 903f3d51a81ec..bc4f1c2791b95 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationService.ts @@ -33,6 +33,7 @@ export interface IChatInputNotificationMuteAction { export interface IChatInputNotification { readonly id: string; + readonly telemetryId?: string; readonly severity: ChatInputNotificationSeverity; readonly message: string | IMarkdownString; readonly description: string | undefined; @@ -54,11 +55,6 @@ export interface IChatInputNotification { readonly mute?: IChatInputNotificationMuteAction; } -export interface IChatInputNotificationActionEvent { - readonly notificationId: string; - readonly commandId: string; -} - export const IChatInputNotificationService = createDecorator('chatInputNotificationService'); export interface IChatInputNotificationService { @@ -69,12 +65,6 @@ export interface IChatInputNotificationService { /** Fires when a notification is dismissed by the user (via the X button). */ readonly onDidDismiss: Event; - /** Fires when a notification is rendered by a chat input widget. */ - readonly onDidShow: Event; - - /** Fires when a notification action is invoked by the user. */ - readonly onDidAction: Event; - /** * Set or update a notification. If a notification with the same ID already * exists, its content is replaced and any previous user dismissal is cleared. @@ -92,16 +82,6 @@ export interface IChatInputNotificationService { */ dismissNotification(id: string): void; - /** - * Mark a notification as rendered by a chat input widget. - */ - showNotification(id: string): void; - - /** - * Mark a notification action as invoked by the user. - */ - actionNotification(id: string, commandId: string): void; - /** * Get the single active notification to display. Returns the highest-severity * notification that has not been dismissed. Ties are broken by most-recent insertion. @@ -122,7 +102,6 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif private readonly _notifications = new Map(); private readonly _dismissed = new Set(); - private readonly _shown = new Set(); /** Insertion order tracking — higher index = more recently set. */ private readonly _insertionOrder = new Map(); @@ -134,12 +113,6 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif private readonly _onDidDismiss = this._register(new Emitter()); readonly onDidDismiss = this._onDidDismiss.event; - private readonly _onDidShow = this._register(new Emitter()); - readonly onDidShow = this._onDidShow.event; - - private readonly _onDidAction = this._register(new Emitter()); - readonly onDidAction = this._onDidAction.event; - /** * Signature of the last active notification we announced via ARIA, so we * don't re-announce the same content when the model fires `onDidChange` @@ -150,7 +123,6 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif setNotification(notification: IChatInputNotification): void { this._notifications.set(notification.id, notification); this._dismissed.delete(notification.id); - this._shown.delete(notification.id); this._insertionOrder.set(notification.id, this._insertionCounter++); this._fireDidChange(); } @@ -158,7 +130,6 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif deleteNotification(id: string): void { if (this._notifications.delete(id)) { this._dismissed.delete(id); - this._shown.delete(id); this._insertionOrder.delete(id); this._fireDidChange(); } @@ -172,24 +143,6 @@ class ChatInputNotificationService extends Disposable implements IChatInputNotif } } - showNotification(id: string): void { - if (this._notifications.has(id) && !this._dismissed.has(id) && !this._shown.has(id)) { - this._shown.add(id); - this._onDidShow.fire(id); - } - } - - actionNotification(id: string, commandId: string): void { - const notification = this._notifications.get(id); - if (notification && !this._dismissed.has(id)) { - // Validate that the commandId exists in the notification's actions - const hasAction = notification.actions.some(a => a.commandId === commandId); - if (hasAction) { - this._onDidAction.fire({ notificationId: id, commandId }); - } - } - } - getActiveNotification(filter?: (notification: IChatInputNotification) => boolean): IChatInputNotification | undefined { let best: IChatInputNotification | undefined; let bestOrder = -1; diff --git a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts index b327b668f1253..57e1488dece95 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/input/chatInputNotificationWidget.ts @@ -22,6 +22,18 @@ import './media/chatInputNotificationWidget.css'; const $ = dom.$; +type ChatInputNotificationTelemetryEvent = { + id: string; + telemetryId?: string; +}; + +type ChatInputNotificationTelemetryClassification = { + id: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The identifier of the chat input notification.' }; + telemetryId?: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The feature-provided identifier for the notification message that was shown or dismissed.' }; + owner: 'rfeltis'; + comment: 'Tracks chat input notification visibility and user dismissals.'; +}; + const severityToClass: Record = { [ChatInputNotificationSeverity.Info]: 'severity-info', [ChatInputNotificationSeverity.Warning]: 'severity-warning', @@ -44,6 +56,7 @@ export class ChatInputNotificationWidget extends Disposable { readonly domNode: HTMLElement; private readonly _contentDisposables = this._register(new DisposableStore()); + private _lastShownTelemetryData: ChatInputNotificationTelemetryEvent | undefined; /** * Optional provider that returns the current session type of the owning @@ -82,12 +95,13 @@ export class ChatInputNotificationWidget extends Disposable { const notification = this._notificationService.getActiveNotification(n => this._matchesSession(n)); if (!notification) { this.domNode.parentElement?.classList.remove('has-notification'); + this._lastShownTelemetryData = undefined; return; } this.domNode.parentElement?.classList.add('has-notification'); this._renderNotification(notification); - this._notificationService.showNotification(notification.id); + this._logShownTelemetry(notification); } private _matchesSession(notification: IChatInputNotification): boolean { @@ -163,7 +177,10 @@ export class ChatInputNotificationWidget extends Disposable { // browser has finished propagating the click event. Otherwise // blur handlers fired by removing the button from focus can // move/remove nodes that `clearNode` then trips over. - const dismiss = () => queueMicrotask(() => this._notificationService.dismissNotification(notification.id)); + const dismiss = () => queueMicrotask(() => { + this._telemetryService.publicLog2('chatInputNotificationDismissed', this._getTelemetryData(notification)); + this._notificationService.dismissNotification(notification.id); + }); this._contentDisposables.add(dom.addDisposableListener(dismissButton, dom.EventType.CLICK, dismiss)); this._contentDisposables.add(dom.addDisposableListener(dismissButton, dom.EventType.KEY_DOWN, (e: KeyboardEvent) => { if (e.key === 'Enter' || e.key === ' ') { @@ -209,7 +226,6 @@ export class ChatInputNotificationWidget extends Disposable { button.element.ariaLabel = `${ariaTitle} ${action.label}`; this._contentDisposables.add(button.onDidClick(async () => { - this._notificationService.actionNotification(notification.id, action.commandId); this._telemetryService.publicLog2('workbenchActionExecuted', { id: action.commandId, from: 'chatInputNotification', @@ -220,4 +236,20 @@ export class ChatInputNotificationWidget extends Disposable { } } } + + private _logShownTelemetry(notification: IChatInputNotification): void { + const data = this._getTelemetryData(notification); + if (this._lastShownTelemetryData?.id === data.id && this._lastShownTelemetryData.telemetryId === data.telemetryId) { + return; + } + this._lastShownTelemetryData = data; + this._telemetryService.publicLog2('chatInputNotificationShown', data); + } + + private _getTelemetryData(notification: IChatInputNotification): ChatInputNotificationTelemetryEvent { + return { + id: notification.id, + telemetryId: notification.telemetryId, + }; + } } diff --git a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts index 2aca5b89ef139..087fac8c787e7 100644 --- a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostPermissionUiContribution.test.ts @@ -50,8 +50,6 @@ class FakeNotificationService implements IChatInputNotificationService { declare readonly _serviceBrand: undefined; readonly onDidChange: Event = Event.None; readonly onDidDismiss: Event = Event.None; - readonly onDidShow: Event = Event.None; - readonly onDidAction: Event<{ notificationId: string; commandId: string }> = Event.None; readonly setCalls: IChatInputNotification[] = []; readonly deleteCalls: string[] = []; @@ -62,8 +60,6 @@ class FakeNotificationService implements IChatInputNotificationService { this.deleteCalls.push(id); } dismissNotification(_id: string): void { /* */ } - showNotification(_id: string): void { /* */ } - actionNotification(_id: string, _commandId: string): void { /* */ } getActiveNotification(): IChatInputNotification | undefined { return undefined; } handleMessageSent(): void { /* */ } } diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts index 73ffd4716cdae..3a2aeaa9a8145 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.integrationTest.ts @@ -21,7 +21,7 @@ import { ITelemetryAppender } from '../../../../../platform/telemetry/common/tel import { ChatEntitlement, IChatEntitlementService, IChatSentiment, IQuotaSnapshot } from '../../../../services/chat/common/chatEntitlementService.js'; import { workbenchInstantiationService } from '../../../../test/browser/workbenchTestServices.js'; import { ChatQuotaNotificationContribution } from '../../browser/chatQuotaNotification.js'; -import { IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; +import { ChatInputNotificationSeverity, IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; import { ChatInputNotificationWidget } from '../../browser/widget/input/chatInputNotificationWidget.js'; import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../common/languageModels.js'; @@ -55,14 +55,18 @@ class TestCommandService implements ICommandService { } } -function createQuotaSnapshot(percentRemaining: number): IQuotaSnapshot { +function createQuotaSnapshot(percentRemaining: number, opts?: Partial): IQuotaSnapshot { return { percentRemaining, unlimited: false, + ...opts, }; } -function createEntitlementService() { +function createEntitlementService(opts?: { + entitlement?: ChatEntitlement; + quotas?: Partial; +}) { const onDidChangeQuotaRemaining = new Emitter(); const onDidChangeQuotaExceeded = new Emitter(); const onDidChangeEntitlement = new Emitter(); @@ -70,8 +74,8 @@ function createEntitlementService() { const service: IChatEntitlementService = { _serviceBrand: undefined, - entitlement: ChatEntitlement.Pro, - entitlementObs: observableValue({}, ChatEntitlement.Pro), + entitlement: opts?.entitlement ?? ChatEntitlement.Pro, + entitlementObs: observableValue({}, opts?.entitlement ?? ChatEntitlement.Pro), onDidChangeEntitlement: onDidChangeEntitlement.event, onDidChangeQuotaExceeded: onDidChangeQuotaExceeded.event, onDidChangeQuotaRemaining: onDidChangeQuotaRemaining.event, @@ -80,6 +84,7 @@ function createEntitlementService() { usageBasedBilling: true, premiumChat: createQuotaSnapshot(0), additionalUsageEnabled: false, + ...opts?.quotas, }, organisations: undefined, isInternal: false, @@ -108,11 +113,11 @@ function createEntitlementService() { suite('ChatQuotaNotificationContribution integration', () => { const store = ensureNoDisposablesAreLeakedInTestSuite(); - function createHarness() { + function createHarness(opts?: Parameters[0]) { const instantiationService = store.add(workbenchInstantiationService(undefined, store)); const telemetryAppender = new TestTelemetryAppender(); const telemetryService = store.add(instantiationService.createInstance(TelemetryService, { appenders: [telemetryAppender] })); - const entitlementService = createEntitlementService(); + const entitlementService = createEntitlementService(opts); const commandService = new TestCommandService(); const storageService = instantiationService.get(IStorageService); storageService.store('chat.currentLanguageModel.panel', 'copilot/test-model', StorageScope.APPLICATION, StorageTarget.USER); @@ -151,35 +156,205 @@ suite('ChatQuotaNotificationContribution integration', () => { const contribution = store.add(childInstantiationService.createInstance(ChatQuotaNotificationContribution)); store.add(childInstantiationService.get(IChatInputNotificationService) as IChatInputNotificationService & IDisposable); - return { instantiationService: childInstantiationService, telemetryAppender, commandService, contribution }; + return { instantiationService: childInstantiationService, telemetryAppender, commandService, contribution, entitlementService }; } - function getQuotaTelemetryEvents(telemetryAppender: TestTelemetryAppender): ILoggedTelemetryEvent[] { - return telemetryAppender.events.filter(e => e.eventName.startsWith('chat.quotaNotification')); + function getNotificationTelemetryEvents(telemetryAppender: TestTelemetryAppender): ILoggedTelemetryEvent[] { + return telemetryAppender.events.filter(e => e.eventName === 'chatInputNotificationShown' || e.eventName === 'chatInputNotificationDismissed'); } - test('emits shown telemetry through the real widget render path', () => { + function getRenderedText(widget: ChatInputNotificationWidget): string { + return widget.domNode.querySelector('.chat-input-notification')?.textContent ?? ''; + } + + function assertShownTelemetry(telemetryAppender: TestTelemetryAppender, telemetryId: string): void { + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), [{ + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId, + }, + }]); + } + + test('emits generic shown telemetry through the real widget render path', () => { const { instantiationService, telemetryAppender } = createHarness(); - assert.deepStrictEqual(getQuotaTelemetryEvents(telemetryAppender), []); + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), []); const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); assert.ok(widget.domNode.querySelector('.chat-input-notification')); - assert.deepStrictEqual(getQuotaTelemetryEvents(telemetryAppender), [{ - eventName: 'chat.quotaNotificationShown', - data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, + assertShownTelemetry(telemetryAppender, 'quotaExhausted'); + }); + + test('emits shown telemetry for each quota notification variant through the real widget render path', () => { + const cases: readonly { + readonly name: string; + readonly setup: () => ReturnType; + readonly expectedText: string; + readonly expectedTelemetryId: string; + readonly afterCreate?: (harness: ReturnType) => void; + }[] = [ + { + name: 'quota exhausted', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(0), + additionalUsageEnabled: false, + }, + }), + expectedText: 'Credit Limit Reached', + expectedTelemetryId: 'quotaExhausted', + }, + { + name: 'overage activation', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(50), + additionalUsageEnabled: true, + }, + }), + expectedText: 'Additional budget is now covering extra usage.', + expectedTelemetryId: 'overageActivation', + afterCreate: ({ entitlementService }) => { + (entitlementService.service as IChatEntitlementService & { quotas: IChatEntitlementService['quotas'] }).quotas = { + ...entitlementService.service.quotas, + premiumChat: createQuotaSnapshot(0), + }; + entitlementService.onDidChangeQuotaRemaining.fire(); + }, + }, + { + name: 'quota approaching', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(50), + }, + }), + expectedText: 'Credits at 75%', + expectedTelemetryId: 'quotaApproaching', + afterCreate: ({ entitlementService }) => { + (entitlementService.service as IChatEntitlementService & { quotas: IChatEntitlementService['quotas'] }).quotas = { + ...entitlementService.service.quotas, + premiumChat: createQuotaSnapshot(25), + }; + entitlementService.onDidChangeQuotaRemaining.fire(); + }, + }, + { + name: 'rate limit warning', + setup: () => createHarness({ + quotas: { + premiumChat: createQuotaSnapshot(50), + sessionRateLimit: { percentRemaining: 50, unlimited: false, resetDate: '2026-06-01T00:00:00Z' }, + }, + }), + expectedText: 'You\'ve used 75% of your session rate limit.', + expectedTelemetryId: 'sessionRateLimitWarning', + afterCreate: ({ entitlementService }) => { + (entitlementService.service as IChatEntitlementService & { quotas: IChatEntitlementService['quotas'] }).quotas = { + ...entitlementService.service.quotas, + sessionRateLimit: { percentRemaining: 25, unlimited: false, resetDate: '2026-06-01T00:00:00Z' }, + }; + entitlementService.onDidChangeQuotaRemaining.fire(); + }, + }, + { + name: 'managed plan blocked', + setup: () => createHarness({ + entitlement: ChatEntitlement.Business, + quotas: { + premiumChat: createQuotaSnapshot(0, { + unlimited: true, + hasQuota: false, + }), + }, + }), + expectedText: 'Usage Blocked', + expectedTelemetryId: 'managedPlanBlocked', + }, + ]; + + const results = cases.map(testCase => { + const harness = testCase.setup(); + testCase.afterCreate?.(harness); + + const widget = store.add(harness.instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + const renderedText = getRenderedText(widget); + assert.ok(renderedText.includes(testCase.expectedText), `${testCase.name} did not render expected text`); + assertShownTelemetry(harness.telemetryAppender, testCase.expectedTelemetryId); + + return { + name: testCase.name, + renderedText: testCase.expectedText, + telemetry: getNotificationTelemetryEvents(harness.telemetryAppender), + }; + }); + + assert.deepStrictEqual(results.map(result => result.name), [ + 'quota exhausted', + 'overage activation', + 'quota approaching', + 'rate limit warning', + 'managed plan blocked', + ]); + }); + + test('emits shown telemetry when the same notification id changes telemetry context', () => { + const { instantiationService, telemetryAppender } = createHarness(); + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + const notificationService = instantiationService.get(IChatInputNotificationService); + + notificationService.setNotification({ + id: 'copilot.quotaStatus', + telemetryId: 'quotaApproaching', + severity: ChatInputNotificationSeverity.Info, + message: 'Credits at 75%', + description: undefined, + actions: [], + dismissible: true, + autoDismissOnMessage: true, + }); + + assert.ok(widget.domNode.querySelector('.chat-input-notification')); + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), [ + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', + }, }, - }]); + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaApproaching', + }, + }, + ]); + }); + + test('does not emit duplicate shown telemetry when the notification rerenders unchanged', () => { + const { instantiationService, telemetryAppender } = createHarness(); + const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); + + widget.rerender(); + widget.rerender(); + + assert.deepStrictEqual(getNotificationTelemetryEvents(telemetryAppender), [ + { + eventName: 'chatInputNotificationShown', + data: { + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', + }, + }, + ]); }); - test('emits action and dismissed telemetry from real DOM interaction', async () => { + test('emits existing action telemetry and generic dismissed telemetry from real DOM interaction', async () => { const { instantiationService, telemetryAppender, commandService } = createHarness(); const widget = store.add(instantiationService.createInstance(ChatInputNotificationWidget, undefined)); @@ -194,39 +369,26 @@ suite('ChatQuotaNotificationContribution integration', () => { await timeout(0); assert.deepStrictEqual(commandService.executedCommands, ['workbench.action.chat.manageAdditionalSpend']); - assert.deepStrictEqual(getQuotaTelemetryEvents(telemetryAppender), [ + assert.deepStrictEqual(telemetryAppender.events.filter(e => e.eventName === 'workbenchActionExecuted' || e.eventName === 'chatInputNotificationShown' || e.eventName === 'chatInputNotificationDismissed'), [ { - eventName: 'chat.quotaNotificationShown', + eventName: 'chatInputNotificationShown', data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', }, }, { - eventName: 'chat.quotaNotificationActionInvoked', + eventName: 'workbenchActionExecuted', data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, - commandId: 'workbench.action.chat.manageAdditionalSpend', + id: 'workbench.action.chat.manageAdditionalSpend', + from: 'chatInputNotification', }, }, { - eventName: 'chat.quotaNotificationDismissed', + eventName: 'chatInputNotificationDismissed', data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, + id: 'copilot.quotaStatus', + telemetryId: 'quotaExhausted', }, }, ]); diff --git a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts index db4bd992bbc96..81108bd53e280 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatQuotaNotification.test.ts @@ -10,12 +10,10 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/tes import { IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js'; import { MockContextKeyService } from '../../../../../platform/keybinding/test/common/mockKeybindingService.js'; import { InMemoryStorageService, StorageScope, StorageTarget } from '../../../../../platform/storage/common/storage.js'; -import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; -import { NullTelemetryService } from '../../../../../platform/telemetry/common/telemetryUtils.js'; import { ChatEntitlement, IChatEntitlementService, IChatSentiment, IQuotaSnapshot, IRateLimitSnapshot } from '../../../../services/chat/common/chatEntitlementService.js'; import { ChatQuotaNotificationContribution } from '../../browser/chatQuotaNotification.js'; import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../common/languageModels.js'; -import { IChatInputNotification, IChatInputNotificationActionEvent, IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; +import { IChatInputNotification, IChatInputNotificationService } from '../../browser/widget/input/chatInputNotificationService.js'; // --- Mock IChatEntitlementService ------------------------------------------- @@ -86,32 +84,25 @@ function createMockNotificationService() { let lastNotification: IChatInputNotification | undefined = undefined; let deleted = false; let dismissed = false; - let shown = false; let setCount = 0; const onDidChange = new Emitter(); const onDidDismiss = new Emitter(); - const onDidShow = new Emitter(); - const onDidAction = new Emitter(); const service: IChatInputNotificationService = { _serviceBrand: undefined, onDidChange: onDidChange.event, onDidDismiss: onDidDismiss.event, - onDidShow: onDidShow.event, - onDidAction: onDidAction.event, setNotification(notification: IChatInputNotification) { lastNotification = notification; deleted = false; dismissed = false; - shown = false; setCount++; onDidChange.fire(); }, deleteNotification(_id: string) { deleted = true; dismissed = false; - shown = false; onDidChange.fire(); }, dismissNotification(id: string) { @@ -122,23 +113,6 @@ function createMockNotificationService() { onDidDismiss.fire(id); onDidChange.fire(); }, - showNotification(id: string) { - if (!lastNotification || lastNotification.id !== id || dismissed || shown) { - return; - } - shown = true; - onDidShow.fire(id); - }, - actionNotification(id: string, commandId: string) { - if (!lastNotification || lastNotification.id !== id || dismissed || deleted) { - return; - } - // Validate that the commandId exists in the notification's actions - const hasAction = lastNotification.actions.some(a => a.commandId === commandId); - if (hasAction) { - onDidAction.fire({ notificationId: id, commandId }); - } - }, getActiveNotification(filter?: (notification: IChatInputNotification) => boolean) { if (deleted || dismissed || !lastNotification) { return undefined; @@ -153,39 +127,10 @@ function createMockNotificationService() { getNotification(): IChatInputNotification | undefined { return deleted || dismissed ? undefined : lastNotification; }, get wasDeleted() { return deleted; }, get setCount() { return setCount; }, - reset() { lastNotification = undefined; deleted = false; dismissed = false; shown = false; setCount = 0; }, + reset() { lastNotification = undefined; deleted = false; dismissed = false; setCount = 0; }, }; } -// --- Mock ITelemetryService ------------------------------------------------- - -interface ILoggedTelemetryData { - readonly notificationType?: string; - readonly limitType?: string; - readonly entitlement?: string; - readonly additionalUsageEnabled?: boolean; - readonly hasActions?: boolean; - readonly percentUsed?: number; - readonly commandId?: string; -} - -interface ILoggedTelemetryEvent { - readonly eventName: string; - readonly data: ILoggedTelemetryData | undefined; -} - -function createMockTelemetryService() { - const events: ILoggedTelemetryEvent[] = []; - const service = { - ...NullTelemetryService, - publicLog2(eventName: string, data?: ILoggedTelemetryData): void { - events.push({ eventName, data }); - }, - } as ITelemetryService; - - return { service, events }; -} - // --- Helpers --------------------------------------------------------------- function makeQuotaSnapshot(percentRemaining: number, opts?: Partial): IQuotaSnapshot { @@ -214,7 +159,6 @@ suite('ChatQuotaNotificationContribution', () => { function createContribution(entitlementOpts?: Parameters[0], modelOpts?: { vendor?: string }) { const entitlementMock = createMockEntitlementService(entitlementOpts); const notificationMock = createMockNotificationService(); - const telemetryMock = createMockTelemetryService(); const contextKeyService = store.add(new MockContextKeyService()); const storageService = store.add(new InMemoryStorageService()); const vendor = modelOpts?.vendor ?? 'copilot'; @@ -242,10 +186,9 @@ suite('ChatQuotaNotificationContribution', () => { contextKeyService as IContextKeyService, languageModelsService, storageService, - telemetryMock.service, )); - return { contribution, entitlementMock, notificationMock, storageService, telemetryMock }; + return { contribution, entitlementMock, notificationMock, storageService }; } function updateQuotas( @@ -261,132 +204,6 @@ suite('ChatQuotaNotificationContribution', () => { entitlementMock.onDidChangeQuotaRemaining.fire(); } - // --- Telemetry ----------------------------------------------------------- - - suite('telemetry', () => { - test('logs shown telemetry when quota notification is rendered', () => { - const { notificationMock, telemetryMock } = createContribution({ - quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, - }); - - assert.deepStrictEqual(telemetryMock.events, []); - - notificationMock.service.showNotification('copilot.quotaStatus'); - - assert.deepStrictEqual(telemetryMock.events, [{ - eventName: 'chat.quotaNotificationShown', - data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, - }, - }]); - }); - - test('does not duplicate shown telemetry while the same notification remains active', () => { - const { entitlementMock, notificationMock, telemetryMock } = createContribution({ - quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, - }); - - notificationMock.service.showNotification('copilot.quotaStatus'); - updateQuotas(entitlementMock, { premiumChat: makeQuotaSnapshot(0) }); - notificationMock.service.showNotification('copilot.quotaStatus'); - - assert.deepStrictEqual(telemetryMock.events, [{ - eventName: 'chat.quotaNotificationShown', - data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, - }, - }]); - }); - - test('logs dismissed telemetry when quota notification is dismissed', () => { - const { notificationMock, telemetryMock } = createContribution({ - quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, - }); - - notificationMock.service.showNotification('copilot.quotaStatus'); - notificationMock.service.dismissNotification('copilot.quotaStatus'); - - const expectedData = { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, - }; - assert.deepStrictEqual(telemetryMock.events, [ - { eventName: 'chat.quotaNotificationShown', data: expectedData }, - { eventName: 'chat.quotaNotificationDismissed', data: expectedData }, - ]); - }); - - test('logs action telemetry when quota notification action is invoked', () => { - const { notificationMock, telemetryMock } = createContribution({ - quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, - }); - - notificationMock.service.showNotification('copilot.quotaStatus'); - notificationMock.service.actionNotification('copilot.quotaStatus', 'workbench.action.chat.manageAdditionalSpend'); - - assert.deepStrictEqual(telemetryMock.events, [ - { - eventName: 'chat.quotaNotificationShown', - data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, - }, - }, - { - eventName: 'chat.quotaNotificationActionInvoked', - data: { - notificationType: 'quotaExhausted', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 100, - commandId: 'workbench.action.chat.manageAdditionalSpend', - }, - }, - ]); - }); - - test('logs approaching notification details', () => { - const { entitlementMock, notificationMock, telemetryMock } = createContribution({ - quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(60) }, - }); - - updateQuotas(entitlementMock, { premiumChat: makeQuotaSnapshot(50) }); - notificationMock.service.showNotification('copilot.quotaStatus'); - - assert.deepStrictEqual(telemetryMock.events, [{ - eventName: 'chat.quotaNotificationShown', - data: { - notificationType: 'quotaApproaching', - limitType: 'quota', - entitlement: 'Pro', - additionalUsageEnabled: false, - hasActions: true, - percentUsed: 50, - }, - }]); - }); - }); - // --- Quota exhausted --------------------------------------------------- suite('quota exhausted', () => { @@ -807,7 +624,6 @@ suite('ChatQuotaNotificationContribution', () => { quotas: { usageBasedBilling: true, premiumChat: makeQuotaSnapshot(0) }, }); const notificationMock = createMockNotificationService(); - const telemetryMock = createMockTelemetryService(); const contextKeyService = store.add(new MockContextKeyService()); const storageService = store.add(new InMemoryStorageService()); // Start with BYOK model @@ -833,7 +649,6 @@ suite('ChatQuotaNotificationContribution', () => { contextKeyService as IContextKeyService, languageModelsService, storageService, - telemetryMock.service, )); // Initially deferred — BYOK model