diff --git a/src/commands/index.ts b/src/commands/index.ts index d6c25ab6..647298ea 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -85,4 +85,6 @@ enum EXTENSION_COMMANDS { SHOW_EXPORT_TO_LANGUAGE_RESULT = 'mdb.showExportToLanguageResult', } +export type ExtensionCommand = EXTENSION_COMMANDS; + export default EXTENSION_COMMANDS; diff --git a/src/documentSource.ts b/src/documentSource.ts index 4c110ca0..329032a8 100644 --- a/src/documentSource.ts +++ b/src/documentSource.ts @@ -2,5 +2,5 @@ export enum DocumentSource { DOCUMENT_SOURCE_TREEVIEW = 'treeview', DOCUMENT_SOURCE_PLAYGROUND = 'playground', DOCUMENT_SOURCE_COLLECTIONVIEW = 'collectionview', - DOCUMENT_SOURCE_QUERY_WITH_COPILOT_CODELENS = 'query with copilot codelens', + DOCUMENT_SOURCE_CODELENS = 'codelens', } diff --git a/src/editors/queryWithCopilotCodeLensProvider.ts b/src/editors/queryWithCopilotCodeLensProvider.ts index 23b3958a..23e212a3 100644 --- a/src/editors/queryWithCopilotCodeLensProvider.ts +++ b/src/editors/queryWithCopilotCodeLensProvider.ts @@ -31,9 +31,12 @@ export class QueryWithCopilotCodeLensProvider prompt: 'Describe the query you would like to generate', placeHolder: 'e.g. Find the document in sample_mflix.users with the name of Kayden Washington', - messagePrefix: '/query', + command: 'query', isNewChat: true, - source: DocumentSource.DOCUMENT_SOURCE_QUERY_WITH_COPILOT_CODELENS, + telemetry: { + source: DocumentSource.DOCUMENT_SOURCE_CODELENS, + sourceDetails: 'query with copilot code lens', + }, }; return [ diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index 973c2345..cd2f8d4f 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -25,7 +25,7 @@ import { } from './explorer'; import ExportToLanguageCodeLensProvider from './editors/exportToLanguageCodeLensProvider'; import { type ExportToLanguageResult } from './types/playgroundType'; -import EXTENSION_COMMANDS from './commands'; +import type { ExtensionCommand } from './commands'; import type FieldTreeItem from './explorer/fieldTreeItem'; import type IndexListTreeItem from './explorer/indexListTreeItem'; import { LanguageServerController } from './language'; @@ -40,10 +40,7 @@ import WebviewController from './views/webviewController'; import { createIdFactory, generateId } from './utils/objectIdHelper'; import { ConnectionStorage } from './storage/connectionStorage'; import type StreamProcessorTreeItem from './explorer/streamProcessorTreeItem'; -import type { - ParticipantCommand, - RunParticipantCodeCommandArgs, -} from './participant/participant'; +import type { RunParticipantCodeCommandArgs } from './participant/participant'; import ParticipantController from './participant/participant'; import type { OpenSchemaCommandArgs } from './participant/prompts/schema'; import { QueryWithCopilotCodeLensProvider } from './editors/queryWithCopilotCodeLensProvider'; @@ -52,6 +49,8 @@ import type { SendMessageToParticipantFromInputOptions, } from './participant/participantTypes'; import { COPILOT_CHAT_EXTENSION_ID } from './participant/constants'; +import type { ParticipantCommand } from './participant/participantTypes'; +import EXTENSION_COMMANDS from './commands'; // This class is the top-level controller for our extension. // Commands which the extensions handles are defined in the function `activate`. @@ -385,7 +384,7 @@ export default class MDBExtensionController implements vscode.Disposable { }; registerParticipantCommand = ( - command: string, + command: ExtensionCommand, commandHandler: (...args: any[]) => Promise ): void => { const commandHandlerWithTelemetry = (args: any[]): Promise => { @@ -403,7 +402,7 @@ export default class MDBExtensionController implements vscode.Disposable { }; registerCommand = ( - command: string, + command: ExtensionCommand, commandHandler: (...args: any[]) => Promise ): void => { const commandHandlerWithTelemetry = (args: any[]): Promise => { diff --git a/src/participant/constants.ts b/src/participant/constants.ts index 86511f82..f73e3965 100644 --- a/src/participant/constants.ts +++ b/src/participant/constants.ts @@ -1,21 +1,12 @@ import type * as vscode from 'vscode'; import { ChatMetadataStore } from './chatMetadata'; +import type { ParticipantResponseType } from './participantTypes'; export const CHAT_PARTICIPANT_ID = 'mongodb.participant'; export const CHAT_PARTICIPANT_MODEL = 'gpt-4o'; export const COPILOT_EXTENSION_ID = 'GitHub.copilot'; export const COPILOT_CHAT_EXTENSION_ID = 'GitHub.copilot-chat'; -export type ParticipantResponseType = - | 'query' - | 'schema' - | 'docs' - | 'generic' - | 'emptyRequest' - | 'cancelledRequest' - | 'askToConnect' - | 'askForNamespace'; - export const codeBlockIdentifier = { start: '```javascript', end: '```', diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 670636fa..5ffad1b1 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -53,10 +53,13 @@ import { PromptHistory } from './prompts/promptHistory'; import type { SendMessageToParticipantOptions, SendMessageToParticipantFromInputOptions, + ParticipantCommand, + ParticipantCommandType, } from './participantTypes'; import { DEFAULT_EXPORT_TO_LANGUAGE_DRIVER_SYNTAX } from '../editors/exportToLanguageCodeLensProvider'; import { EXPORT_TO_LANGUAGE_ALIASES } from '../editors/playgroundSelectionCodeActionProvider'; import { CollectionTreeItem, DatabaseTreeItem } from '../explorer'; +import { DocumentSource } from '../documentSource'; const log = createLogger('participant'); @@ -73,8 +76,6 @@ export type RunParticipantCodeCommandArgs = { runnableContent: string; }; -export type ParticipantCommand = '/query' | '/schema' | '/docs'; - const MAX_MARKDOWN_LIST_LENGTH = 10; export default class ParticipantController { @@ -179,6 +180,8 @@ export default class ParticipantController { message, isNewChat = false, isPartialQuery = false, + telemetry, + command, ...otherOptions } = options; @@ -188,10 +191,28 @@ export default class ParticipantController { 'workbench.action.chat.clearHistory' ); } + const commandPrefix = command ? `/${command} ` : ''; + const query = `@MongoDB ${commandPrefix}${message}`; + + if (telemetry) { + if (isNewChat) { + this._telemetryService.trackParticipantChatOpenedFromAction({ + ...telemetry, + command, + }); + } + if (!isPartialQuery) { + this._telemetryService.trackParticipantPromptSubmittedFromAction({ + ...telemetry, + command: command ?? 'generic', + input_length: query.length, + }); + } + } return await vscode.commands.executeCommand('workbench.action.chat.open', { ...otherOptions, - query: `@MongoDB ${message}`, + query, isPartialQuery, }); } @@ -200,27 +221,34 @@ export default class ParticipantController { options: SendMessageToParticipantFromInputOptions ): Promise { const { - messagePrefix = '', - isNewChat = false, - isPartialQuery = false, - source, + isNewChat, + isPartialQuery, + telemetry, + command, ...inputBoxOptions } = options; - this._telemetryService.trackCopilotParticipantSubmittedFromInputBox({ - source, - }); - const message = await vscode.window.showInputBox({ ...inputBoxOptions, }); + if (telemetry) { + this._telemetryService.trackParticipantInputBoxSubmitted({ + ...telemetry, + input_length: message?.length, + dismissed: message === undefined, + command, + }); + } + if (message === undefined || message.trim() === '') { return Promise.resolve(); } return this.sendMessageToParticipant({ - message: `${messagePrefix ? `${messagePrefix} ` : ''}${message}`, + message, + telemetry, + command, isNewChat, isPartialQuery, }); @@ -235,6 +263,10 @@ export default class ParticipantController { await this.sendMessageToParticipant({ message: `I want to ask questions about the \`${databaseName}\` database.`, isNewChat: true, + telemetry: { + source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW, + sourceDetails: 'copilot button on database tree item', + }, }); } else if (treeItem instanceof CollectionTreeItem) { const { databaseName, collectionName } = treeItem; @@ -242,6 +274,10 @@ export default class ParticipantController { await this.sendMessageToParticipant({ message: `I want to ask questions about the \`${databaseName}\` database's \`${collectionName}\` collection.`, isNewChat: true, + telemetry: { + source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW, + sourceDetails: 'copilot button on collection tree item', + }, }); } else { throw new Error('Unsupported tree item type'); @@ -276,7 +312,7 @@ export default class ParticipantController { }) ), }); - this._telemetryService.trackCopilotParticipantPrompt(modelInput.stats); + this._telemetryService.trackParticipantPrompt(modelInput.stats); const modelResponse = await model.sendRequest( modelInput.messages, @@ -456,7 +492,7 @@ export default class ParticipantController { stream, }); - this._telemetryService.trackCopilotParticipantResponse({ + this._telemetryService.trackParticipantResponse({ command: 'generic', has_cta: false, found_namespace: false, @@ -1423,7 +1459,7 @@ export default class ParticipantController { ], }); - this._telemetryService.trackCopilotParticipantResponse({ + this._telemetryService.trackParticipantResponse({ command: 'schema', has_cta: true, found_namespace: true, @@ -1534,7 +1570,7 @@ export default class ParticipantController { token, }); - this._telemetryService.trackCopilotParticipantResponse({ + this._telemetryService.trackParticipantResponse({ command: 'query', has_cta: false, found_namespace: true, @@ -1640,7 +1676,7 @@ export default class ParticipantController { this._streamGenericDocsLink(stream); - this._telemetryService.trackCopilotParticipantResponse({ + this._telemetryService.trackParticipantResponse({ command: 'docs/copilot', has_cta: true, found_namespace: false, @@ -1720,7 +1756,7 @@ export default class ParticipantController { } } - this._telemetryService.trackCopilotParticipantResponse({ + this._telemetryService.trackParticipantResponse({ command: 'docs/chatbot', has_cta: !!docsResult.responseReferences, found_namespace: false, @@ -1838,10 +1874,7 @@ export default class ParticipantController { return true; } catch (error) { const message = formatError(error).message; - this._telemetryService.trackCopilotParticipantError( - error, - 'exportToPlayground' - ); + this._telemetryService.trackParticipantError(error, 'exportToPlayground'); void vscode.window.showErrorMessage( `An error occurred exporting to a playground: ${message}` ); @@ -1948,9 +1981,9 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i return await this.handleGenericRequest(...args); } } catch (error) { - this._telemetryService.trackCopilotParticipantError( + this._telemetryService.trackParticipantError( error, - request.command || 'generic' + (request.command as ParticipantCommandType) || 'generic' ); // Re-throw other errors so they show up in the UI. throw error; @@ -1999,7 +2032,7 @@ Please see our [FAQ](https://www.mongodb.com/docs/generative-ai-faq/) for more i 'unhelpfulReason' in feedback ? (feedback.unhelpfulReason as string) : undefined; - this._telemetryService.trackCopilotParticipantFeedback({ + this._telemetryService.trackParticipantFeedback({ feedback: chatResultFeedbackKindToTelemetryValue(feedback.kind), reason: unhelpfulReason, response_type: (feedback.result as ChatResult)?.metadata.intent, diff --git a/src/participant/participantTypes.ts b/src/participant/participantTypes.ts index b9e25838..b82e7050 100644 --- a/src/participant/participantTypes.ts +++ b/src/participant/participantTypes.ts @@ -1,11 +1,36 @@ import type * as vscode from 'vscode'; import type { DocumentSource } from '../documentSource'; +export type ParticipantCommandType = 'query' | 'schema' | 'docs'; +export type ParticipantCommand = `/${ParticipantCommandType}`; + +export type ParticipantRequestType = ParticipantCommandType | 'generic'; + +export type ParticipantResponseType = + | 'query' + | 'schema' + | 'docs' + | 'docs/chatbot' + | 'docs/copilot' + | 'exportToPlayground' + | 'generic' + | 'emptyRequest' + | 'cancelledRequest' + | 'askToConnect' + | 'askForNamespace'; + +type TelemetryMetadata = { + source: DocumentSource; + sourceDetails?: string; +}; + /** Based on options from Copilot's chat open command IChatViewOpenOptions */ export type SendMessageToParticipantOptions = { message: string; + command?: ParticipantCommandType; isNewChat?: boolean; isPartialQuery?: boolean; + telemetry?: TelemetryMetadata; /** * Any previous chat requests and responses that should be shown in the chat view. * Note that currently these requests do not end up included in vscode's context.history. @@ -16,8 +41,8 @@ export type SendMessageToParticipantOptions = { }[]; }; -export type SendMessageToParticipantFromInputOptions = { - messagePrefix?: string; - source?: DocumentSource; -} & Omit & +export type SendMessageToParticipantFromInputOptions = Pick< + SendMessageToParticipantOptions, + 'isNewChat' | 'isPartialQuery' | 'command' | 'telemetry' +> & vscode.InputBoxOptions; diff --git a/src/participant/prompts/promptBase.ts b/src/participant/prompts/promptBase.ts index 0df0d1b3..56ed32f6 100644 --- a/src/participant/prompts/promptBase.ts +++ b/src/participant/prompts/promptBase.ts @@ -5,6 +5,7 @@ import type { ParticipantPromptProperties, } from '../../telemetry/telemetryService'; import { PromptHistory } from './promptHistory'; +import type { ParticipantCommandType } from '../participantTypes'; export interface PromptArgsBase { request: { @@ -175,7 +176,7 @@ export abstract class PromptBase { ), user_input_length: request.prompt.length, has_sample_documents: hasSampleDocs, - command: request.command || 'generic', + command: (request.command as ParticipantCommandType) || 'generic', history_size: context?.history.length || 0, internal_purpose: this.internalPurposeForTelemetry, }; diff --git a/src/participant/prompts/promptHistory.ts b/src/participant/prompts/promptHistory.ts index 6f55e577..fa042d70 100644 --- a/src/participant/prompts/promptHistory.ts +++ b/src/participant/prompts/promptHistory.ts @@ -1,6 +1,7 @@ import * as vscode from 'vscode'; import { ParticipantErrorTypes } from '../participantErrorTypes'; -import type { ChatResult, ParticipantResponseType } from '../constants'; +import type { ChatResult } from '../constants'; +import type { ParticipantResponseType } from '../participantTypes'; export class PromptHistory { private static _handleChatResponseTurn({ diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 609238a9..80819512 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -12,8 +12,13 @@ import { getConnectionTelemetryProperties } from './connectionTelemetry'; import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry'; import type { ShellEvaluateResult } from '../types/playgroundType'; import type { StorageController } from '../storage'; -import type { ParticipantResponseType } from '../participant/constants'; import { ParticipantErrorTypes } from '../participant/participantErrorTypes'; +import type { ExtensionCommand } from '../commands'; +import type { + ParticipantCommandType, + ParticipantRequestType, + ParticipantResponseType, +} from '../participant/participantTypes'; const log = createLogger('telemetry'); // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -37,7 +42,7 @@ type LinkClickedTelemetryEventProperties = { }; type ExtensionCommandRunTelemetryEventProperties = { - command: string; + command: ExtensionCommand; }; type DocumentUpdatedTelemetryEventProperties = { @@ -98,7 +103,7 @@ type ParticipantFeedbackProperties = { }; type ParticipantResponseFailedProperties = { - command: string; + command: ParticipantResponseType; error_code?: string; error_name: ParticipantErrorTypes; }; @@ -106,7 +111,7 @@ type ParticipantResponseFailedProperties = { export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; export type ParticipantPromptProperties = { - command: string; + command: ParticipantCommandType; user_input_length: number; total_message_length: number; has_sample_documents: boolean; @@ -115,7 +120,7 @@ export type ParticipantPromptProperties = { }; export type ParticipantResponseProperties = { - command: string; + command: ParticipantResponseType; has_cta: boolean; has_runnable_content: boolean; found_namespace: boolean; @@ -126,8 +131,22 @@ export type CopilotIntroductionProperties = { is_copilot_active: boolean; }; -export type ParticipantOpenedFromInputBoxProperties = { - source?: DocumentSource; +export type ParticipantPromptSubmittedFromActionProperties = { + source: DocumentSource; + input_length: number; + command: ParticipantRequestType; +}; + +export type ParticipantChatOpenedFromActionProperties = { + source: DocumentSource; + command?: ParticipantCommandType; +}; + +export type ParticipantInputBoxSubmitted = { + source: DocumentSource; + input_length: number | undefined; + dismissed: boolean; + command?: ParticipantCommandType; }; export function chatResultFeedbackKindToTelemetryValue( @@ -160,7 +179,8 @@ type TelemetryEventProperties = | ParticipantFeedbackProperties | ParticipantResponseFailedProperties | ParticipantPromptProperties - | ParticipantOpenedFromInputBoxProperties + | ParticipantPromptSubmittedFromActionProperties + | ParticipantChatOpenedFromActionProperties | ParticipantResponseProperties | CopilotIntroductionProperties; @@ -182,9 +202,16 @@ export enum TelemetryEventTypes { PARTICIPANT_FEEDBACK = 'Participant Feedback', PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed', + /** Tracks all submitted prompts */ PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted', + /** Tracks prompts that were submitted as a result of an action other than + * the user typing the message, such as clicking on an item in tree view or a codelens */ + PARTICIPANT_PROMPT_SUBMITTED_FROM_ACTION = 'Participant Prompt Submitted From Action', + /** Tracks when a new chat was opened from an action such as clicking on a tree view. */ + PARTICIPANT_CHAT_OPENED_FROM_ACTION = 'Participant Chat Opened From Action', + /** Tracks after a participant interacts with the input box we open to let the user write the prompt for participant. */ + PARTICIPANT_INPUT_BOX_SUBMITTED = 'Participant Inbox Box Submitted', PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated', - PARTICIPANT_SUBMITTED_FROM_INPUT_BOX = 'Participant Submitted From Input Box', COPILOT_INTRODUCTION_CLICKED = 'Copilot Introduction Clicked', COPILOT_INTRODUCTION_DISMISSED = 'Copilot Introduction Dismissed', } @@ -326,7 +353,7 @@ export default class TelemetryService { ); } - trackCommandRun(command: string): void { + trackCommandRun(command: ExtensionCommand): void { this.track(TelemetryEventTypes.EXTENSION_COMMAND_RUN, { command }); } @@ -435,17 +462,30 @@ export default class TelemetryService { ); } - trackCopilotParticipantFeedback(props: ParticipantFeedbackProperties): void { + trackParticipantFeedback(props: ParticipantFeedbackProperties): void { this.track(TelemetryEventTypes.PARTICIPANT_FEEDBACK, props); } - trackCopilotParticipantSubmittedFromInputBox( - props: ParticipantOpenedFromInputBoxProperties + trackParticipantPromptSubmittedFromAction( + props: ParticipantPromptSubmittedFromActionProperties ): void { - this.track(TelemetryEventTypes.PARTICIPANT_SUBMITTED_FROM_INPUT_BOX, props); + this.track( + TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED_FROM_ACTION, + props + ); + } + + trackParticipantChatOpenedFromAction( + props: ParticipantChatOpenedFromActionProperties + ): void { + this.track(TelemetryEventTypes.PARTICIPANT_CHAT_OPENED_FROM_ACTION, props); } - trackCopilotParticipantError(err: any, command: string): void { + trackParticipantInputBoxSubmitted(props: ParticipantInputBoxSubmitted): void { + this.track(TelemetryEventTypes.PARTICIPANT_INPUT_BOX_SUBMITTED, props); + } + + trackParticipantError(err: any, command: ParticipantResponseType): void { let errorCode: string | undefined; let errorName: ParticipantErrorTypes; // Making the chat request might fail because @@ -477,14 +517,14 @@ export default class TelemetryService { command, error_code: errorCode, error_name: errorName, - }); + } as ParticipantResponseFailedProperties); } - trackCopilotParticipantPrompt(stats: ParticipantPromptProperties): void { + trackParticipantPrompt(stats: ParticipantPromptProperties): void { this.track(TelemetryEventTypes.PARTICIPANT_PROMPT_SUBMITTED, stats); } - trackCopilotParticipantResponse(props: ParticipantResponseProperties): void { + trackParticipantResponse(props: ParticipantResponseProperties): void { this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_GENERATED, props); } diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 5e4722c2..2bd25349 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -44,6 +44,7 @@ import EditDocumentCodeLensProvider from '../../../editors/editDocumentCodeLensP import PlaygroundResultProvider from '../../../editors/playgroundResultProvider'; import { CollectionTreeItem, DatabaseTreeItem } from '../../../explorer'; import type { SendMessageToParticipantOptions } from '../../../participant/participantTypes'; +import { DocumentSource } from '../../../documentSource'; // The Copilot's model in not available in tests, // therefore we need to mock its methods and returning values. @@ -1838,6 +1839,10 @@ Schema: { message: `I want to ask questions about the \`${mockDatabaseItem.databaseName}\` database.`, isNewChat: true, + telemetry: { + source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW, + sourceDetails: 'copilot button on database tree item', + }, }, ]); @@ -1872,6 +1877,10 @@ Schema: { message: `I want to ask questions about the \`${mockCollectionItem.databaseName}\` database's \`${mockCollectionItem.collectionName}\` collection.`, isNewChat: true, + telemetry: { + source: DocumentSource.DOCUMENT_SOURCE_TREEVIEW, + sourceDetails: 'copilot button on collection tree item', + }, }, ]); @@ -2583,7 +2592,7 @@ Schema: test('reports error', function () { const err = Error('Filtered by Responsible AI Service'); - testParticipantController._telemetryService.trackCopilotParticipantError( + testParticipantController._telemetryService.trackParticipantError( err, 'query' ); @@ -2604,7 +2613,7 @@ Schema: test('reports nested error', function () { const err = new Error('Parent error'); err.cause = Error('This message is flagged as off topic: off_topic.'); - testParticipantController._telemetryService.trackCopilotParticipantError( + testParticipantController._telemetryService.trackParticipantError( err, 'docs' ); @@ -2623,7 +2632,7 @@ Schema: test('Reports error code when available', function () { // eslint-disable-next-line new-cap const err = vscode.LanguageModelError.NotFound('Model not found'); - testParticipantController._telemetryService.trackCopilotParticipantError( + testParticipantController._telemetryService.trackParticipantError( err, 'schema' ); diff --git a/src/test/suite/participant/participantHelpers.ts b/src/test/suite/participant/participantHelpers.ts index 77448c3d..4ff90958 100644 --- a/src/test/suite/participant/participantHelpers.ts +++ b/src/test/suite/participant/participantHelpers.ts @@ -1,6 +1,6 @@ import { CHAT_PARTICIPANT_ID } from '../../../participant/constants'; import * as vscode from 'vscode'; -import type { ParticipantCommand } from '../../../participant/participant'; +import type { ParticipantCommand } from '../../../participant/participantTypes'; export function createChatRequestTurn( command: ParticipantCommand | undefined,