Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add telemetry for followups #224235

Merged
merged 1 commit into from
Jul 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/vs/workbench/contrib/chat/browser/chatFollowups.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export class ChatFollowups<T extends IChatFollowup> extends Disposable {
}

const tooltipPrefix = formatChatQuestion(this.chatAgentService, this.location, '', followup.agentId, followup.subCommand);
if (!tooltipPrefix) {
if (tooltipPrefix === undefined) {
return;
}

Expand Down
4 changes: 3 additions & 1 deletion src/vs/workbench/contrib/chat/common/chatServiceImpl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -544,14 +544,15 @@ export class ChatService extends Disposable implements IChatService {
rawResult.errorDetails && gotProgress ? 'errorWithOutput' :
rawResult.errorDetails ? 'error' :
'success';
const commandForTelemetry = agentSlashCommandPart ? agentSlashCommandPart.command.name : commandPart?.slashCommand.command;
this.telemetryService.publicLog2<ChatProviderInvokedEvent, ChatProviderInvokedClassification>('interactiveSessionProviderInvoked', {
timeToFirstProgress: rawResult.timings?.firstProgress,
totalTime: rawResult.timings?.totalElapsed,
result,
requestType,
agent: agentPart?.agent.id ?? '',
agentExtensionId: agentPart?.agent.extensionId.value ?? '',
slashCommand: agentSlashCommandPart ? agentSlashCommandPart.command.name : commandPart?.slashCommand.command,
slashCommand: commandForTelemetry,
chatSessionId: model.sessionId,
location,
citations: request.response?.codeCitations.length ?? 0
Expand All @@ -564,6 +565,7 @@ export class ChatService extends Disposable implements IChatService {
if (agentOrCommandFollowups) {
agentOrCommandFollowups.then(followups => {
model.setFollowups(request, followups);
this._chatServiceTelemetry.retrievedFollowups(agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0);
});
}
}
Expand Down
39 changes: 39 additions & 0 deletions src/vs/workbench/contrib/chat/common/chatServiceTelemetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,18 @@ type ChatCommandClassification = {
comment: 'Provides insight into the usage of Chat features.';
};

type ChatFollowupEvent = {
agentId: string;
command: string | undefined;
};

type ChatFollowupClassification = {
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
owner: 'roblourens';
comment: 'Provides insight into the usage of Chat features.';
};

type ChatTerminalEvent = {
languageId: string;
agentId: string;
Expand All @@ -77,6 +89,20 @@ type ChatTerminalClassification = {
comment: 'Provides insight into the usage of Chat features.';
};

type ChatFollowupsRetrievedEvent = {
agentId: string;
command: string | undefined;
numFollowups: number;
};

type ChatFollowupsRetrievedClassification = {
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
numFollowups: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of followup prompts returned by the agent.' };
owner: 'roblourens';
comment: 'Provides insight into the usage of Chat features.';
};

export class ChatServiceTelemetry {
constructor(
@ITelemetryService private readonly telemetryService: ITelemetryService,
Expand Down Expand Up @@ -116,6 +142,19 @@ export class ChatServiceTelemetry {
agentId: action.agentId ?? '',
command: action.command,
});
} else if (action.action.kind === 'followUp') {
this.telemetryService.publicLog2<ChatFollowupEvent, ChatFollowupClassification>('chatFollowupClicked', {
agentId: action.agentId ?? '',
command: action.command,
});
}
}

retrievedFollowups(agentId: string, command: string | undefined, numFollowups: number): void {
this.telemetryService.publicLog2<ChatFollowupsRetrievedEvent, ChatFollowupsRetrievedClassification>('chatFollowupsRetrieved', {
agentId,
command,
numFollowups,
});
}
}
Loading