refactor: ♻️ remove test-provider option and streamline config

microsoft · Nov 19, 2024 · 903ba97 · 903ba97
1 parent a5f262f
commit 903ba97
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 82 deletions.
diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md
@@ -92,7 +92,6 @@ Options:
   -o, --out <folder>                  output folder
   -rmo, --remove-out                  remove output folder if it exists
   --cli <string>                      override path to the cli
-  -tp, --test-provider <string>       test provider
   -td, --test-delay <string>          delay between tests in seconds
   --cache                             enable LLM result cache
   -v, --verbose                       verbose output

diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
@@ -208,7 +208,6 @@ export async function cli() {
         .option("-o, --out <folder>", "output folder")
         .option("-rmo, --remove-out", "remove output folder if it exists")
         .option("--cli <string>", "override path to the cli")
-        .option("-tp, --test-provider <string>", "test provider")
         .option("-td, --test-delay <string>", "delay between tests in seconds")
         .option("--cache", "enable LLM result cache")
         .option("-v, --verbose", "verbose output")

diff --git a/packages/cli/src/test.ts b/packages/cli/src/test.ts
@@ -72,20 +72,6 @@ function parseModelSpec(m: string): ModelOptions {
     else return { model: m }
 }
 
-/**
- * Resolves the test provider for a given script by determining the language model configuration.
- * @param script - The PromptScript object for which to determine the provider.
- * @returns The resolved provider or undefined if not found.
- */
-async function resolveTestProvider(info: ModelConnectionInfo) {
-    switch (info?.type) {
-        case "azure":
-        case "azure_serverless":
-            return info?.base
-    }
-    return undefined
-}
-
 /**
  * Creates an environment object for execution with defaults and optional overrides.
  * @returns An environment object with necessary configurations.
@@ -160,17 +146,12 @@ export async function runPromptScriptTests(
         logInfo(`  ${fn}`)
         const { info } = await resolveModelConnectionInfo(script)
         if (info.error) throw new Error(info.error)
-        const testProvider =
-            options?.testProvider || (await resolveTestProvider(info))
         const config = generatePromptFooConfiguration(script, {
             out,
             cli,
-            model: info.model,
-            smallModel: info.smallModel,
-            visionModel: info.visionModel,
             models: options.models?.map(parseModelSpec),
             provider: "provider.mjs",
-            testProvider,
+            info,
         })
         const yaml = YAMLStringify(config)
         await writeFile(fn, yaml)

diff --git a/packages/core/src/test.ts b/packages/core/src/test.ts
@@ -2,31 +2,49 @@
 import {
     HTTPS_REGEX,
     LARGE_MODEL_ID,
+    MODEL_PROVIDER_AZURE_OPENAI,
+    MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI,
     SMALL_MODEL_ID,
     VISION_MODEL_ID,
 } from "./constants"
-import { arrayify } from "./util"
+import { arrayify, deleteUndefinedValues } from "./util"
 import { host } from "./host"
+import { ModelConnectionInfo } from "./models"
 
 /**
- * Function to remove properties with undefined values from an object.
- *
- * @param obj - An object with string keys and any type of values.
- * @returns A new object with undefined values removed, or undefined if the input is undefined.
+ * Convert GenAIScript connection info into prompt foo configuration
+ * @param info
  */
-function cleanUndefined(obj: Record<string, any>) {
-    // Check if the object is defined
-    return obj
-        ? Object.entries(obj) // Convert object to entries
-              .filter(([_, value]) => value !== undefined) // Filter out undefined values
-              .reduce(
-                  (newObj, [key, value]) => {
-                      newObj[key] = value // Add key-value pair to new object
-                      return newObj // Return accumulated object
-                  },
-                  {} as Record<string, any> // Initialize as empty object
-              )
-        : undefined // Return undefined if input is undefined
+function resolveTestProvider(info: ModelConnectionInfo) {
+    const { provider, model, base } = info
+    switch (provider) {
+        case MODEL_PROVIDER_AZURE_OPENAI:
+        case MODEL_PROVIDER_AZURE_SERVERLESS_OPENAI:
+            return {
+                text: {
+                    id: "azureopenai:chat:gpt-4",
+                    config: {
+                        apiHost: base
+                            .replace(HTTPS_REGEX, "")
+                            .replace(/\/openai\/deployments$/i, ""),
+                    },
+                },
+                embedding: {
+                    id: "azureopenai:embeddings:text-embedding-ada-002",
+                    config: {
+                        apiHost: base
+                            .replace(HTTPS_REGEX, "")
+                            .replace(/\/openai\/deployments$/i, ""),
+                    },
+                },
+            }
+        // openai
+        default:
+            return {
+                text: provider + ":chat:" + model,
+                embedding: provider + ":embeddings:" + base,
+            }
+    }
 }
 
 /**
@@ -38,29 +56,26 @@ function cleanUndefined(obj: Record<string, any>) {
  */
 export function generatePromptFooConfiguration(
     script: PromptScript,
-    options?: {
+    options: {
+        info: ModelConnectionInfo
         provider?: string
-        testProvider?: string
         out?: string
         cli?: string
-        model?: string
-        smallModel?: string
-        visionModel?: string
         models?: ModelOptions[]
     }
 ) {
     // Destructure options with default values
-    const { provider = "provider.mjs", testProvider } = options || {}
+    const { provider = "provider.mjs", info } = options || {}
     const { description, title, tests = [], id } = script
     const models = options?.models || []
 
     // Ensure at least one model exists
     if (!models.length) {
         models.push({
             ...script,
-            model: options?.model || script.model,
-            smallModel: options?.smallModel || script.smallModel,
-            visionModel: options?.visionModel || script.visionModel,
+            model: info.model,
+            smallModel: info.smallModel,
+            visionModel: info.visionModel,
         })
     }
 
@@ -76,6 +91,8 @@ export function generatePromptFooConfiguration(
                 ? host.defaultModelOptions.model
                 : m
 
+    const testProvider = resolveTestProvider(info)
+
     // Create configuration object
     const res = {
         // Description combining title and description
@@ -117,38 +134,7 @@ export function generatePromptFooConfiguration(
                 },
             })),
         // Default test configuration if testProvider is present
-        defaultTest: testProvider
-            ? {
-                  options: {
-                      provider: testProvider
-                          ? {
-                                text: {
-                                    id: "azureopenai:chat:gpt-4",
-                                    config: {
-                                        apiHost: testProvider
-                                            .replace(HTTPS_REGEX, "")
-                                            .replace(
-                                                /\/openai\/deployments$/i,
-                                                ""
-                                            ),
-                                    },
-                                },
-                                embedding: {
-                                    id: "azureopenai:embeddings:text-embedding-ada-002",
-                                    config: {
-                                        apiHost: testProvider
-                                            .replace(HTTPS_REGEX, "")
-                                            .replace(
-                                                /\/openai\/deployments$/i,
-                                                ""
-                                            ),
-                                    },
-                                },
-                            }
-                          : undefined,
-                  },
-              }
-            : undefined,
+        defaultTest: testProvider ? { provider: testProvider } : undefined,
         // Map tests to configuration format
         tests: arrayify(tests).map(
             ({
@@ -162,7 +148,7 @@ export function generatePromptFooConfiguration(
                 asserts = [],
             }) => ({
                 description,
-                vars: cleanUndefined({
+                vars: deleteUndefinedValues({
                     files,
                     vars,
                 }),