From f3c4e1e909508fd3381c124842d89bea0c52c7e9 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Wed, 9 Oct 2024 16:08:35 -0700 Subject: [PATCH] Update tool helper names --- package-lock.json | 4 +- src/base/index.ts | 6 +- src/base/vscodeTypes.d.ts | 205 +++++++++++++++++++++++++++++--------- 3 files changed, 163 insertions(+), 52 deletions(-) diff --git a/package-lock.json b/package-lock.json index 14784c1..3e1740d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@vscode/prompt-tsx", - "version": "0.2.10", + "version": "0.3.0-alpha.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@vscode/prompt-tsx", - "version": "0.2.10", + "version": "0.3.0-alpha.1", "license": "SEE LICENSE IN LICENSE", "devDependencies": { "@microsoft/tiktokenizer": "^1.0.6", diff --git a/src/base/index.ts b/src/base/index.ts index 6351417..ee6c4d1 100644 --- a/src/base/index.ts +++ b/src/base/index.ts @@ -168,19 +168,19 @@ export function toVsCodeChatMessages(messages: ChatMessage[]) { const message: LanguageModelChatMessage = vscode.LanguageModelChatMessage.Assistant(m.content, m.name); if (m.tool_calls) { message.content2 = [m.content]; - message.content2.push(...m.tool_calls.map(tc => new vscode.LanguageModelChatResponseToolCallPart(tc.function.name, tc.function.arguments, tc.id))); + message.content2.push(...m.tool_calls.map(tc => new vscode.LanguageModelToolCallPart(tc.function.name, tc.function.arguments, tc.id))); } return message; case ChatRole.User: return vscode.LanguageModelChatMessage.User(m.content, m.name); case ChatRole.Function: { const message: LanguageModelChatMessage = vscode.LanguageModelChatMessage.User(''); - message.content2 = [new vscode.LanguageModelChatMessageFunctionResultPart(m.name, m.content)]; + message.content2 = [new vscode.LanguageModelToolResultPart(m.name, m.content)]; return message; } case ChatRole.Tool: { const message: LanguageModelChatMessage = vscode.LanguageModelChatMessage.User(m.content); - message.content2 = [new vscode.LanguageModelChatMessageFunctionResultPart(m.tool_call_id, m.content)]; + message.content2 = [new vscode.LanguageModelToolResultPart(m.tool_call_id, m.content)]; return message; } default: diff --git a/src/base/vscodeTypes.d.ts b/src/base/vscodeTypes.d.ts index 1411b3a..c7d4ead 100644 --- a/src/base/vscodeTypes.d.ts +++ b/src/base/vscodeTypes.d.ts @@ -2,7 +2,7 @@ * Copyright (c) Microsoft Corporation and GitHub. All rights reserved. *--------------------------------------------------------------------------------------------*/ -import type { CancellationToken, Command, Location, MarkdownString, ProviderResult, ThemeIcon, Range, Uri } from 'vscode'; +import type { CancellationToken, Command, Location, MarkdownString, ProviderResult, Range, ThemeIcon, Uri } from 'vscode'; /** * Represents a part of a chat response that is formatted as Markdown. @@ -353,11 +353,8 @@ export interface LanguageModelChat { countTokens(text: string | LanguageModelChatMessage, token?: CancellationToken): Thenable; } -// TODO@API capabilities - -// API -> LM: an tool/function that is available to the language model export interface LanguageModelChatTool { - // TODO@API should use "id" here to match vscode tools, or keep name to match OpenAI? + // TODO@API should use "id" here to match vscode tools, or keep name to match OpenAI? Align everything. name: string; description: string; parametersSchema?: JSONSchema; @@ -375,33 +372,32 @@ export interface LanguageModelChatRequestOptions { } // LM -> USER: function that should be used -export class LanguageModelChatResponseToolCallPart { +export class LanguageModelToolCallPart { name: string; toolCallId: string; parameters: any; - constructor(name: string, parameters: any, toolCallId: string); + constructor(name: string, toolCallId: string, parameters: any); } // LM -> USER: text chunk -export class LanguageModelChatResponseTextPart { +export class LanguageModelTextPart { value: string; constructor(value: string); } export interface LanguageModelChatResponse { - stream: AsyncIterable; + stream: AsyncIterable; } // USER -> LM: the result of a function call -export class LanguageModelChatMessageToolResultPart { +export class LanguageModelToolResultPart { toolCallId: string; content: string; - isError: boolean; - constructor(toolCallId: string, content: string, isError?: boolean); + constructor(toolCallId: string, content: string); } export interface LanguageModelChatMessage { @@ -410,31 +406,36 @@ export interface LanguageModelChatMessage { * Some parts would be message-type specific for some models and wouldn't go together, * but it's up to the chat provider to decide what to do about that. * Can drop parts that are not valid for the message type. - * LanguageModelChatMessageToolResultPart: only on User messages - * LanguageModelChatResponseToolCallPart: only on Assistant messages + * LanguageModelToolResultPart: only on User messages + * LanguageModelToolCallPart: only on Assistant messages */ - content2: (string | LanguageModelChatMessageToolResultPart | LanguageModelChatResponseToolCallPart)[]; + content2: (string | LanguageModelToolResultPart | LanguageModelToolCallPart)[]; } +// Tool registration/invoking between extensions + +/** + * A result returned from a tool invocation. + */ +// TODO@API should we align this with NotebookCellOutput and NotebookCellOutputItem export interface LanguageModelToolResult { /** - * The result can contain arbitrary representations of the content. An example might be 'prompt-tsx' to indicate an element that can be rendered with the @vscode/prompt-tsx library. + * The result can contain arbitrary representations of the content. A tool user can set + * {@link LanguageModelToolInvocationOptions.requested} to request particular types, and a tool implementation should only + * compute the types that were requested. `text/plain` is recommended to be supported by all tools, which would indicate + * any text-based content. Another example might be a `PromptElementJSON` from `@vscode/prompt-tsx`, using the + * `contentType` exported by that library. */ [contentType: string]: any; - - /** - * A string representation of the result which can be incorporated back into an LLM prompt without any special handling. - */ - toString(): string; } -// Tool registration/invoking between extensions - export namespace lm { /** - * Register a LanguageModelTool. The tool must also be registered in the package.json `languageModelTools` contribution point. + * Register a LanguageModelTool. The tool must also be registered in the package.json `languageModelTools` contribution + * point. A registered tool is available in the {@link lm.tools} list for any extension to see. But in order for it to + * be seen by a language model, it must be passed in the list of available tools in {@link LanguageModelChatRequestOptions.tools}. */ - export function registerTool(id: string, tool: LanguageModelTool): Disposable; + export function registerTool(id: string, tool: LanguageModelTool): Disposable; /** * A list of all available tools. @@ -443,16 +444,38 @@ export namespace lm { /** * Invoke a tool with the given parameters. - * TODO@API Could request a set of contentTypes to be returned so they don't all need to be computed? */ - export function invokeTool(id: string, options: LanguageModelToolInvocationOptions, token: CancellationToken): Thenable; + export function invokeTool(id: string, options: LanguageModelToolInvocationOptions, token: CancellationToken): Thenable; } -export interface LanguageModelToolInvocationOptions { +/** + * A token that can be passed to {@link lm.invokeTool} when invoking a tool inside the context of handling a chat request. + */ +export type ChatParticipantToolToken = unknown; + +/** + * Options provided for tool invocation. + */ +export interface LanguageModelToolInvocationOptions { + /** + * When this tool is being invoked within the context of a chat request, this token should be passed from + * {@link ChatRequest.toolInvocationToken}. In that case, a progress bar will be automatically shown for the tool + * invocation in the chat response view, and if the tool requires user confirmation, it will show up inline in the chat + * view. If the tool is being invoked outside of a chat request, `undefined` should be passed instead. + */ + toolInvocationToken: ChatParticipantToolToken | undefined; + /** - * Parameters with which to invoke the tool. + * The parameters with which to invoke the tool. The parameters must match the schema defined in + * {@link LanguageModelToolDescription.parametersSchema} */ - parameters: Object; + parameters: T; + + /** + * A tool user can request that particular content types be returned from the tool, depending on what the tool user + * supports. All tools are recommended to support `text/plain`. See {@link LanguageModelToolResult}. + */ + requestedContentTypes: string[]; /** * Options to hint at how many tokens the tool should return in its response. @@ -473,35 +496,118 @@ export interface LanguageModelToolInvocationOptions { }; } -export type JSONSchema = object; +/** + * Represents a JSON Schema. + * TODO@API - is this worth it? + */ +export type JSONSchema = Object; +/** + * A description of an available tool. + */ export interface LanguageModelToolDescription { /** * A unique identifier for the tool. */ - id: string; + readonly id: string; /** * A human-readable name for this tool that may be used to describe it in the UI. + * TODO@API keep? */ - displayName: string | undefined; + readonly displayName: string | undefined; /** * A description of this tool that may be passed to a language model. */ - modelDescription: string; + readonly description: string; /** * A JSON schema for the parameters this tool accepts. */ - parametersSchema?: JSONSchema; + readonly parametersSchema?: JSONSchema; + + /** + * The list of content types that the tool has declared support for. See {@link LanguageModelToolResult}. + */ + readonly supportedContentTypes: string[]; + + /** + * A set of tags, declared by the tool, that roughly describe the tool's capabilities. A tool user may use these to filter + * the set of tools to just ones that are relevant for the task at hand. + */ + readonly tags: string[]; +} + +/** + * Messages shown in the chat view when a tool needs confirmation from the user to run. These messages will be shown with + * buttons that say Continue and Cancel. + */ +export interface LanguageModelToolConfirmationMessages { + /** + * The title of the confirmation message. + */ + title: string; + + /** + * The body of the confirmation message. This should be phrased as an action of the participant that is invoking the tool + * from {@link LanguageModelToolInvocationPrepareOptions.participantName}. An example of a good message would be + * `${participantName} will run the command ${echo 'hello world'} in the terminal.` + * TODO@API keep this? + */ + message: string | MarkdownString; +} + +/** + * Options for {@link LanguageModelTool.prepareToolInvocation}. + */ +export interface LanguageModelToolInvocationPrepareOptions { + /** + * The name of the participant invoking the tool. + * TODO@API keep this? + */ + participantName: string; + + /** + * The parameters that the tool is being invoked with. + */ + parameters: T; +} + +/** + * A tool that can be invoked by a call to a {@link LanguageModelChat}. + */ +export interface LanguageModelTool { + /** + * Invoke the tool with the given parameters and return a result. + */ + invoke(options: LanguageModelToolInvocationOptions, token: CancellationToken): ProviderResult; + + /** + * Called once before a tool is invoked. May be implemented to customize the progress message that appears while the tool + * is running, and the messages that appear when the tool needs confirmation. + */ + prepareToolInvocation?(options: LanguageModelToolInvocationPrepareOptions, token: CancellationToken): ProviderResult; } -export interface LanguageModelTool { - // TODO@API should it be LanguageModelToolResult | string? - invoke(options: LanguageModelToolInvocationOptions, token: CancellationToken): ProviderResult; +/** + * The result of a call to {@link LanguageModelTool.prepareToolInvocation}. + */ +export interface PreparedToolInvocation { + /** + * A customized progress message to show while the tool runs. + */ + invocationMessage?: string; + + /** + * Customized messages to show when asking for user confirmation to run the tool. + */ + confirmationMessages?: LanguageModelToolConfirmationMessages; } +/** + * A reference to a tool attached to a user's request. + */ export interface ChatLanguageModelToolReference { /** * The tool's ID. Refers to a tool listed in {@link lm.tools}. @@ -509,10 +615,11 @@ export interface ChatLanguageModelToolReference { readonly id: string; /** - * The start and end index of the reference in the {@link ChatRequest.prompt prompt}. When undefined, the reference was not part of the prompt text. + * The start and end index of the reference in the {@link ChatRequest.prompt prompt}. When undefined, the reference was + * not part of the prompt text. * - * *Note* that the indices take the leading `#`-character into account which means they can - * used to modify the prompt as-is. + * *Note* that the indices take the leading `#`-character into account which means they can be used to modify the prompt + * as-is. */ readonly range?: [start: number, end: number]; } @@ -521,14 +628,18 @@ export interface ChatRequest { /** * The list of tools that the user attached to their request. * - * *Note* that if tools are referenced in the text of the prompt, using `#`, the prompt contains - * references as authored and that it is up to the participant - * to further modify the prompt, for instance by inlining reference values or creating links to - * headings which contain the resolved values. References are sorted in reverse by their range - * in the prompt. That means the last reference in the prompt is the first in this list. This simplifies - * string-manipulation of the prompt. + * *Note* that if tools are referenced in the text of the prompt, using `#`, the prompt contains references as authored + * and it is up to the participant to further modify the prompt, for instance by inlining reference values or + * creating links to headings which contain the resolved values. References are sorted in reverse by their range in the + * prompt. That means the last reference in the prompt is the first in this list. This simplifies string-manipulation of + * the prompt. */ readonly toolReferences: readonly ChatLanguageModelToolReference[]; + + /** + * A token that can be passed to {@link lm.invokeTool} when invoking a tool inside the context of handling a chat request. + */ + readonly toolInvocationToken: ChatParticipantToolToken; } export interface ChatRequestTurn {