Feature Request: Automatic Tracing?

[TylerLeonhardt]

I wrote this fun helper:

import { BasePromptElementProps, HTMLTracer, IChatEndpointInfo, PromptElementCtor, PromptRenderer, PromptSizing, RenderPromptResult, toVsCodeChatMessages } from "@vscode/prompt-tsx";
import { AnyTokenizer } from "@vscode/prompt-tsx/dist/base/tokenizer/tokenizer";
import { CancellationToken, ExtensionMode, LanguageModelChat, LanguageModelChatMessage, LanguageModelToolInvocationOptions, ProgressLocation, window } from "vscode";

let tracingEnabled = false;
export function toggleTracing(extensionMode: ExtensionMode) {
    if (extensionMode === ExtensionMode.Development) {
        tracingEnabled = !tracingEnabled;
    }
    return tracingEnabled;
}

export async function renderPrompt<P extends BasePromptElementProps>(endpoint: IChatEndpointInfo, ctor: PromptElementCtor<P, any>, props: P, model: LanguageModelChat, token: CancellationToken) {
    const renderer = new PromptRenderer(endpoint, ctor, props, new AnyTokenizer(model.countTokens));
    let result: RenderPromptResult;
    if (tracingEnabled) {
        const tracer = new HTMLTracer();
        renderer.tracer = tracer;
        result = await renderer.render(undefined, token);
        const server = await tracer.serveHTML();
        await window.withProgress({ location: ProgressLocation.Notification, title: `Tracer server running at ${server.address}`, cancellable: true }, async (progress, token) => {
            await new Promise<void>(resolve => {
                const dispose = token.onCancellationRequested(() => {
                    dispose.dispose();
                    server.dispose();
                    resolve();
                });
            });
        });
    } else {
        result = await renderer.render(undefined, token);
    }

    return {
        messages: toVsCodeChatMessages(result.messages) as LanguageModelChatMessage[],
        references: result.references
    };
}

That can be used when debugging my extension and running the Toggle Prompt Tracing command. To start a server every time I render a prompt so I can inspect it.

It would be cool to have something like this baked into prompt-tsx. Maybe it could remember the last X prompts and just kick in automatically when debugging an extension. It could log the address to the debug console or something.

In short, lets make tracing really easy to get so debugging these prompts doesn't have to be so hard. It's a big sell for prompt-tsx.