From 3b7adbe6acc381d11d8e17866da4b79444679c08 Mon Sep 17 00:00:00 2001 From: Craig Loewen Date: Thu, 27 Jun 2024 11:42:15 -0400 Subject: [PATCH] [AdvancedPaste]Improved telemetry (#33520) ## Summary of the Pull Request This PR improves advanced paste telemetry. Here's what's changed - Added `AdvancedPasteClipboardItemClicked` event - Changed `CustomFormatEvent` to only fire on successful completion and include the number of tokens used, and the model name Here are the goals of adding this telemtry: - `AdvancedPasteClipboardItemClicked` helps us estimate the total number of user who are using the clipboard history feature, which helps us prioritize future investments and improvements in PowerToys. (This is just regular feature usage data). - `CustomFormatEvent` now includes number of tokens used, and the model name. We are considering using alternative models to power Advanced Paste (as we've heard feedback about this), and these different models could have different token lengths. Understanding the average usage will help us make good decisions that will benefit the most users. As well, the model name is hard coded right now, but in the future if we do add different AI models for completion then this will help us compare their performance. ## PR Checklist - [X] Communication: I've discussed this with core contributors already. If work hasn't been agreed, this work might be rejected ## Detailed Description of the Pull Request / Additional comments The details above are detailed enough since this change is super small. ## Validation Steps Performed Ensured that PowerToys successfully built (Need help verifying the events fire off correctly). --- .../AdvancedPasteXAML/Controls/PromptBox.xaml.cs | 2 -- .../AdvancedPasteXAML/Pages/MainPage.xaml.cs | 1 + .../AdvancedPaste/Helpers/AICompletionsHelper.cs | 16 ++++++++++++---- .../AdvancedPasteClipboardItemClicked.cs | 16 ++++++++++++++++ .../AdvancedPasteGenerateCustomFormatEvent.cs | 13 +++++++++++++ 5 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteClipboardItemClicked.cs diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs index 334d23ebd390..01e4db575f4d 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Controls/PromptBox.xaml.cs @@ -78,8 +78,6 @@ private void GenerateCustom() { Logger.LogTrace(); - PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent()); - VisualStateManager.GoToState(this, "LoadingState", true); string inputInstructions = InputTxtBox.Text; ViewModel.SaveQuery(inputInstructions); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs index 8926cb1820f1..94e81c2add26 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/AdvancedPasteXAML/Pages/MainPage.xaml.cs @@ -231,6 +231,7 @@ private async void ClipboardHistory_ItemClick(object sender, ItemClickEventArgs var item = e.ClickedItem as ClipboardItem; if (item is not null) { + PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteClipboardItemClicked()); if (!string.IsNullOrEmpty(item.Content)) { ClipboardHelper.SetClipboardTextContent(item.Content); diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs index 11c69ff120a7..2dfa803d24fb 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Helpers/AICompletionsHelper.cs @@ -33,6 +33,8 @@ public AICompletionsResponse(string response, int apiRequestStatus) private string _openAIKey; + private string _modelName = "gpt-3.5-turbo-instruct"; + public bool IsAIEnabled => !string.IsNullOrEmpty(this._openAIKey); public AICompletionsHelper() @@ -69,14 +71,14 @@ public static string LoadOpenAIKey() return string.Empty; } - public string GetAICompletion(string systemInstructions, string userMessage) + private Response GetAICompletion(string systemInstructions, string userMessage) { OpenAIClient azureAIClient = new OpenAIClient(_openAIKey); var response = azureAIClient.GetCompletions( new CompletionsOptions() { - DeploymentName = "gpt-3.5-turbo-instruct", + DeploymentName = _modelName, Prompts = { systemInstructions + "\n\n" + userMessage, @@ -90,7 +92,7 @@ public string GetAICompletion(string systemInstructions, string userMessage) Console.WriteLine("Cut off due to length constraints"); } - return response.Value.Choices[0].Text; + return response; } public AICompletionsResponse AIFormatString(string inputInstructions, string inputString) @@ -109,10 +111,16 @@ public AICompletionsResponse AIFormatString(string inputInstructions, string inp "; string aiResponse = null; + Response rawAIResponse = null; int apiRequestStatus = (int)HttpStatusCode.OK; try { - aiResponse = this.GetAICompletion(systemInstructions, userMessage); + rawAIResponse = this.GetAICompletion(systemInstructions, userMessage); + aiResponse = rawAIResponse.Value.Choices[0].Text; + + int promptTokens = rawAIResponse.Value.Usage.PromptTokens; + int completionTokens = rawAIResponse.Value.Usage.CompletionTokens; + PowerToysTelemetry.Log.WriteEvent(new Telemetry.AdvancedPasteGenerateCustomFormatEvent(promptTokens, completionTokens, _modelName)); } catch (Azure.RequestFailedException error) { diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteClipboardItemClicked.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteClipboardItemClicked.cs new file mode 100644 index 000000000000..3c5b63083770 --- /dev/null +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteClipboardItemClicked.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation +// The Microsoft Corporation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.Tracing; +using Microsoft.PowerToys.Telemetry; +using Microsoft.PowerToys.Telemetry.Events; + +namespace AdvancedPaste.Telemetry +{ + [EventData] + public class AdvancedPasteClipboardItemClicked : EventBase, IEvent + { + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; + } +} diff --git a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteGenerateCustomFormatEvent.cs b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteGenerateCustomFormatEvent.cs index a3e874f1f81d..d6f9fed41229 100644 --- a/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteGenerateCustomFormatEvent.cs +++ b/src/modules/AdvancedPaste/AdvancedPaste/Telemetry/AdvancedPasteGenerateCustomFormatEvent.cs @@ -11,6 +11,19 @@ namespace AdvancedPaste.Telemetry [EventData] public class AdvancedPasteGenerateCustomFormatEvent : EventBase, IEvent { + public int PromptTokens { get; set; } + + public int CompletionTokens { get; set; } + + public string ModelName { get; set; } + + public AdvancedPasteGenerateCustomFormatEvent(int promptTokens, int completionTokens, string modelName) + { + this.PromptTokens = promptTokens; + this.CompletionTokens = completionTokens; + ModelName = modelName; + } + public PartA_PrivTags PartA_PrivTags => PartA_PrivTags.ProductAndServiceUsage; } }