From 903ba973d288cb73ac4802237058f3dd89c1cfa7 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 19 Nov 2024 19:42:57 +0000 Subject: [PATCH] =?UTF-8?q?refactor:=20=E2=99=BB=EF=B8=8F=20remove=20test-?= =?UTF-8?q?provider=20option=20and=20streamline=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../content/docs/reference/cli/commands.md | 1 - packages/cli/src/cli.ts | 1 - packages/cli/src/test.ts | 21 +--- packages/core/src/test.ts | 106 ++++++++---------- 4 files changed, 47 insertions(+), 82 deletions(-) diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md index f4f8c5df61..c7be2944a3 100644 --- a/docs/src/content/docs/reference/cli/commands.md +++ b/docs/src/content/docs/reference/cli/commands.md @@ -92,7 +92,6 @@ Options: -o, --out output folder -rmo, --remove-out remove output folder if it exists --cli override path to the cli - -tp, --test-provider test provider -td, --test-delay delay between tests in seconds --cache enable LLM result cache -v, --verbose verbose output diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index e8e23f160c..4a26e7cb1a 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -208,7 +208,6 @@ export async function cli() { .option("-o, --out ", "output folder") .option("-rmo, --remove-out", "remove output folder if it exists") .option("--cli ", "override path to the cli") - .option("-tp, --test-provider ", "test provider") .option("-td, --test-delay ", "delay between tests in seconds") .option("--cache", "enable LLM result cache") .option("-v, --verbose", "verbose output") diff --git a/packages/cli/src/test.ts b/packages/cli/src/test.ts index 852819c623..90d5629383 100644 --- a/packages/cli/src/test.ts +++ b/packages/cli/src/test.ts @@ -72,20 +72,6 @@ function parseModelSpec(m: string): ModelOptions { else return { model: m } } -/** - * Resolves the test provider for a given script by determining the language model configuration. - * @param script - The PromptScript object for which to determine the provider. - * @returns The resolved provider or undefined if not found. - */ -async function resolveTestProvider(info: ModelConnectionInfo) { - switch (info?.type) { - case "azure": - case "azure_serverless": - return info?.base - } - return undefined -} - /** * Creates an environment object for execution with defaults and optional overrides. * @returns An environment object with necessary configurations. @@ -160,17 +146,12 @@ export async function runPromptScriptTests( logInfo(` ${fn}`) const { info } = await resolveModelConnectionInfo(script) if (info.error) throw new Error(info.error) - const testProvider = - options?.testProvider || (await resolveTestProvider(info)) const config = generatePromptFooConfiguration(script, { out, cli, - model: info.model, - smallModel: info.smallModel, - visionModel: info.visionModel, models: options.models?.map(parseModelSpec), provider: "provider.mjs", - testProvider, + info, }) const yaml = YAMLStringify(config) await writeFile(fn, yaml) diff --git a/packages/core/src/test.ts b/packages/core/src/test.ts index a15ae4bb72..2cdb2c9284 100644 --- a/packages/core/src/test.ts +++ b/packages/core/src/test.ts @@ -2,31 +2,49 @@ import { HTTPS_REGEX, LARGE_MODEL_ID, + MODEL_PROVIDER_AZURE_OPENAI, + MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI, SMALL_MODEL_ID, VISION_MODEL_ID, } from "./constants" -import { arrayify } from "./util" +import { arrayify, deleteUndefinedValues } from "./util" import { host } from "./host" +import { ModelConnectionInfo } from "./models" /** - * Function to remove properties with undefined values from an object. - * - * @param obj - An object with string keys and any type of values. - * @returns A new object with undefined values removed, or undefined if the input is undefined. + * Convert GenAIScript connection info into prompt foo configuration + * @param info */ -function cleanUndefined(obj: Record) { - // Check if the object is defined - return obj - ? Object.entries(obj) // Convert object to entries - .filter(([_, value]) => value !== undefined) // Filter out undefined values - .reduce( - (newObj, [key, value]) => { - newObj[key] = value // Add key-value pair to new object - return newObj // Return accumulated object - }, - {} as Record // Initialize as empty object - ) - : undefined // Return undefined if input is undefined +function resolveTestProvider(info: ModelConnectionInfo) { + const { provider, model, base } = info + switch (provider) { + case MODEL_PROVIDER_AZURE_OPENAI: + case MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI: + return { + text: { + id: "azureopenai:chat:gpt-4", + config: { + apiHost: base + .replace(HTTPS_REGEX, "") + .replace(/\/openai\/deployments$/i, ""), + }, + }, + embedding: { + id: "azureopenai:embeddings:text-embedding-ada-002", + config: { + apiHost: base + .replace(HTTPS_REGEX, "") + .replace(/\/openai\/deployments$/i, ""), + }, + }, + } + // openai + default: + return { + text: provider + ":chat:" + model, + embedding: provider + ":embeddings:" + base, + } + } } /** @@ -38,19 +56,16 @@ function cleanUndefined(obj: Record) { */ export function generatePromptFooConfiguration( script: PromptScript, - options?: { + options: { + info: ModelConnectionInfo provider?: string - testProvider?: string out?: string cli?: string - model?: string - smallModel?: string - visionModel?: string models?: ModelOptions[] } ) { // Destructure options with default values - const { provider = "provider.mjs", testProvider } = options || {} + const { provider = "provider.mjs", info } = options || {} const { description, title, tests = [], id } = script const models = options?.models || [] @@ -58,9 +73,9 @@ export function generatePromptFooConfiguration( if (!models.length) { models.push({ ...script, - model: options?.model || script.model, - smallModel: options?.smallModel || script.smallModel, - visionModel: options?.visionModel || script.visionModel, + model: info.model, + smallModel: info.smallModel, + visionModel: info.visionModel, }) } @@ -76,6 +91,8 @@ export function generatePromptFooConfiguration( ? host.defaultModelOptions.model : m + const testProvider = resolveTestProvider(info) + // Create configuration object const res = { // Description combining title and description @@ -117,38 +134,7 @@ export function generatePromptFooConfiguration( }, })), // Default test configuration if testProvider is present - defaultTest: testProvider - ? { - options: { - provider: testProvider - ? { - text: { - id: "azureopenai:chat:gpt-4", - config: { - apiHost: testProvider - .replace(HTTPS_REGEX, "") - .replace( - /\/openai\/deployments$/i, - "" - ), - }, - }, - embedding: { - id: "azureopenai:embeddings:text-embedding-ada-002", - config: { - apiHost: testProvider - .replace(HTTPS_REGEX, "") - .replace( - /\/openai\/deployments$/i, - "" - ), - }, - }, - } - : undefined, - }, - } - : undefined, + defaultTest: testProvider ? { provider: testProvider } : undefined, // Map tests to configuration format tests: arrayify(tests).map( ({ @@ -162,7 +148,7 @@ export function generatePromptFooConfiguration( asserts = [], }) => ({ description, - vars: cleanUndefined({ + vars: deleteUndefinedValues({ files, vars, }),