diff --git a/LICENSE.txt b/LICENSE.txt index 6cfd873..fa1e65c 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2024, Salesforce.com, Inc. +Copyright (c) 2025, Salesforce.com, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index 2f1a831..5b9d331 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,9 @@ sf plugins - [`sf agent create`](#sf-agent-create) -- [`sf agent generate definition`](#sf-agent-generate-definition) - [`sf agent generate spec`](#sf-agent-generate-spec) -- [`sf agent generate testset`](#sf-agent-generate-testset) +- [`sf agent generate test-cases`](#sf-agent-generate-test-cases) +- [`sf agent generate test-definition`](#sf-agent-generate-test-definition) - [`sf agent preview`](#sf-agent-preview) - [`sf agent test cancel`](#sf-agent-test-cancel) - [`sf agent test results`](#sf-agent-test-results) @@ -111,32 +111,7 @@ EXAMPLES $ sf agent create --name CustomerSupportAgent --spec ./config/agentSpec.json --target-org my-org ``` -_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/create.ts)_ - -## `sf agent generate definition` - -Interactively generate a new AiEvaluationDefinition. - -``` -USAGE - $ sf agent generate definition [--flags-dir ] - -GLOBAL FLAGS - --flags-dir= Import flag values from a directory. - -DESCRIPTION - Interactively generate a new AiEvaluationDefinition. - - This command will prompt you for the necessary information to create a new AiEvaluationDefinition. The definition will - be saved to the `aiEvaluationDefinitions` directory in the project. - - You must have the `Bots` and `AiEvaluationTestSets` metadata types present in your project to use this command. - -EXAMPLES - $ sf agent generate definition -``` - -_See code: [src/commands/agent/generate/definition.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/generate/definition.ts)_ +_See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/create.ts)_ ## `sf agent generate spec` @@ -197,30 +172,55 @@ EXAMPLES $ sf agent generate spec --output-dir specs --target-org my-org ``` -_See code: [src/commands/agent/generate/spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/generate/spec.ts)_ +_See code: [src/commands/agent/generate/spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/generate/spec.ts)_ -## `sf agent generate testset` +## `sf agent generate test-cases` -Interactively generate an AiEvaluationTestSet. +Interactively generate a new Set of AI Evaluation test cases. ``` USAGE - $ sf agent generate testset [--flags-dir ] + $ sf agent generate test-cases [--flags-dir ] GLOBAL FLAGS --flags-dir= Import flag values from a directory. DESCRIPTION - Interactively generate an AiEvaluationTestSet. + Interactively generate a new Set of AI Evaluation test cases. Answer the prompts to generate an AiEvaluationTestSet that will be written to a file. You can then run "sf agent generate definition" to generate the AiEvaluationDefinition that can be used to evaluate the test set. EXAMPLES - $ sf agent generate testset + $ sf agent generate test-cases +``` + +_See code: [src/commands/agent/generate/test-cases.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/generate/test-cases.ts)_ + +## `sf agent generate test-definition` + +Interactively generate a new AI Evaluation Test Definition. + +``` +USAGE + $ sf agent generate test-definition [--flags-dir ] + +GLOBAL FLAGS + --flags-dir= Import flag values from a directory. + +DESCRIPTION + Interactively generate a new AI Evaluation Test Definition. + + This command will prompt you for the necessary information to create a new AiEvaluationDefinition. The definition will + be saved to the `aiEvaluationDefinitions` directory in the project. + + You must have the `Bots` and `AiEvaluationTestSets` metadata types present in your project to use this command. + +EXAMPLES + $ sf agent generate test-definition ``` -_See code: [src/commands/agent/generate/testset.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/generate/testset.ts)_ +_See code: [src/commands/agent/generate/test-definition.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/generate/test-definition.ts)_ ## `sf agent preview` @@ -255,7 +255,7 @@ FLAG DESCRIPTIONS the API name of the agent? (TBD based on agents library) ``` -_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/preview.ts)_ +_See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/preview.ts)_ ## `sf agent test cancel` @@ -292,7 +292,7 @@ EXAMPLES $ sf agent test cancel --job-id 4KBfake0000003F4AQ --target-org my-org ``` -_See code: [src/commands/agent/test/cancel.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/test/cancel.ts)_ +_See code: [src/commands/agent/test/cancel.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/test/cancel.ts)_ ## `sf agent test results` @@ -301,10 +301,10 @@ Get the results of a completed agent test run. ``` USAGE $ sf agent test results -o -i [--json] [--flags-dir ] [--api-version ] [--result-format - json|human|junit|tap] [-f ] + json|human|junit|tap] [-d ] FLAGS - -f, --output-dir= Directory to write the agent test results into. + -d, --output-dir= Directory to write the agent test results into. -i, --job-id= (required) Job ID of the completed agent test run. -o, --target-org= (required) Username or alias of the target org. Not required if the `target-org` configuration variable is already set. @@ -342,13 +342,13 @@ EXAMPLES $ sf agent test results --use-most-recent --output-dir ./test-results --result-format json FLAG DESCRIPTIONS - -f, --output-dir= Directory to write the agent test results into. + -d, --output-dir= Directory to write the agent test results into. If the agent test run completes, write the results to the specified directory. If the test is still running, the test results aren't written. ``` -_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/test/results.ts)_ +_See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/test/results.ts)_ ## `sf agent test resume` @@ -357,10 +357,10 @@ Resume an agent test that you previously started in your org so you can view the ``` USAGE $ sf agent test resume -o [--json] [--flags-dir ] [--api-version ] [-i ] [-r] [-w - ] [--result-format json|human|junit|tap] [-f ] + ] [--result-format json|human|junit|tap] [-d ] FLAGS - -f, --output-dir= Directory to write the agent test results into. + -d, --output-dir= Directory to write the agent test results into. -i, --job-id= Job ID of the original agent test run. -o, --target-org= (required) Username or alias of the target org. Not required if the `target-org` configuration variable is already set. @@ -405,13 +405,13 @@ EXAMPLES $ sf agent test resume --use-most-recent --output-dir ./test-results --result-format json FLAG DESCRIPTIONS - -f, --output-dir= Directory to write the agent test results into. + -d, --output-dir= Directory to write the agent test results into. If the agent test run completes, write the results to the specified directory. If the test is still running, the test results aren't written. ``` -_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/test/resume.ts)_ +_See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/test/resume.ts)_ ## `sf agent test run` @@ -420,10 +420,10 @@ Start an agent test in your org. ``` USAGE $ sf agent test run -o -n [--json] [--flags-dir ] [--api-version ] [-w ] - [--result-format json|human|junit|tap] [-f ] + [--result-format json|human|junit|tap] [-d ] FLAGS - -f, --output-dir= Directory to write the agent test results into. + -d, --output-dir= Directory to write the agent test results into. -n, --name= (required) Name of the agent test to start. -o, --target-org= (required) Username or alias of the target org. Not required if the `target-org` configuration variable is already set. @@ -468,12 +468,12 @@ EXAMPLES $ sf agent test run --name MyAgentTest --wait 10 --output-dir ./test-results --result-format json FLAG DESCRIPTIONS - -f, --output-dir= Directory to write the agent test results into. + -d, --output-dir= Directory to write the agent test results into. If the agent test run completes, write the results to the specified directory. If the test is still running, the test results aren't written. ``` -_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.2/src/commands/agent/test/run.ts)_ +_See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.7.3-dev.5/src/commands/agent/test/run.ts)_ diff --git a/command-snapshot.json b/command-snapshot.json index 753e6b3..382df40 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -7,14 +7,6 @@ "flags": ["api-version", "flags-dir", "json", "name", "spec", "target-org"], "plugin": "@salesforce/plugin-agent" }, - { - "alias": [], - "command": "agent:generate:definition", - "flagAliases": [], - "flagChars": [], - "flags": ["flags-dir"], - "plugin": "@salesforce/plugin-agent" - }, { "alias": [], "command": "agent:generate:spec", @@ -37,7 +29,15 @@ }, { "alias": [], - "command": "agent:generate:testset", + "command": "agent:generate:test-cases", + "flagAliases": [], + "flagChars": [], + "flags": ["flags-dir"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:test-definition", "flagAliases": [], "flagChars": [], "flags": ["flags-dir"], @@ -63,7 +63,7 @@ "alias": [], "command": "agent:test:results", "flagAliases": [], - "flagChars": ["f", "i", "o"], + "flagChars": ["d", "i", "o"], "flags": ["api-version", "flags-dir", "job-id", "json", "output-dir", "result-format", "target-org"], "plugin": "@salesforce/plugin-agent" }, @@ -71,7 +71,7 @@ "alias": [], "command": "agent:test:resume", "flagAliases": [], - "flagChars": ["f", "i", "o", "r", "w"], + "flagChars": ["d", "i", "o", "r", "w"], "flags": [ "api-version", "flags-dir", @@ -89,7 +89,7 @@ "alias": [], "command": "agent:test:run", "flagAliases": [], - "flagChars": ["f", "n", "o", "w"], + "flagChars": ["d", "n", "o", "w"], "flags": ["api-version", "flags-dir", "json", "name", "output-dir", "result-format", "target-org", "wait"], "plugin": "@salesforce/plugin-agent" } diff --git a/messages/agent.generate.testset.md b/messages/agent.generate.test-cases.md similarity index 82% rename from messages/agent.generate.testset.md rename to messages/agent.generate.test-cases.md index 6030bf1..9299e00 100644 --- a/messages/agent.generate.testset.md +++ b/messages/agent.generate.test-cases.md @@ -1,6 +1,6 @@ # summary -Interactively generate an AiEvaluationTestSet. +Interactively generate a new Set of AI Evaluation test cases. # description diff --git a/messages/agent.generate.definition.md b/messages/agent.generate.test-definition.md similarity index 86% rename from messages/agent.generate.definition.md rename to messages/agent.generate.test-definition.md index e11696e..6724286 100644 --- a/messages/agent.generate.definition.md +++ b/messages/agent.generate.test-definition.md @@ -1,6 +1,6 @@ # summary -Interactively generate a new AiEvaluationDefinition. +Interactively generate a new AI Evaluation Test Definition. # description diff --git a/package.json b/package.json index 6316e39..4a9e1fa 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,20 @@ { "name": "@salesforce/plugin-agent", "description": "Commands to interact with Salesforce agents", - "version": "1.7.2", + "version": "1.7.3-dev.5", "author": "Salesforce", "bugs": "https://github.com/forcedotcom/cli/issues", "dependencies": { - "@inquirer/confirm": "^5.1.0", "@inquirer/figures": "^1.0.7", - "@inquirer/input": "^4.0.1", - "@inquirer/select": "^4.0.1", + "@inquirer/prompts": "^7.2.0", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.7.12", - "@salesforce/agents": "^0.5.2", + "@salesforce/agents": "0.5.10-dev.0", "@salesforce/core": "^8.8.0", "@salesforce/kit": "^3.2.1", "@salesforce/sf-plugins-core": "^12.1.0", "ansis": "^3.3.2", + "fast-xml-parser": "^4.5.1", "ink": "^5.0.1", "ink-text-input": "^6.0.0", "react": "^18.3.1" diff --git a/schemas/agent-test-results.json b/schemas/agent-test-results.json index 4b0d329..f1a73b0 100644 --- a/schemas/agent-test-results.json +++ b/schemas/agent-test-results.json @@ -3,9 +3,9 @@ "$ref": "#/definitions/AgentTestResultsResult", "definitions": { "AgentTestResultsResult": { - "$ref": "#/definitions/AgentTestDetailsResponse" + "$ref": "#/definitions/AgentTestResultsResponse" }, - "AgentTestDetailsResponse": { + "AgentTestResultsResponse": { "type": "object", "properties": { "status": { diff --git a/src/commands/agent/generate/spec.ts b/src/commands/agent/generate/spec.ts index df54d53..786db74 100644 --- a/src/commands/agent/generate/spec.ts +++ b/src/commands/agent/generate/spec.ts @@ -10,8 +10,7 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages, SfProject } from '@salesforce/core'; import { Interfaces } from '@oclif/core'; import ansis from 'ansis'; -import select from '@inquirer/select'; -import inquirerInput from '@inquirer/input'; +import { select, input as inquirerInput } from '@inquirer/prompts'; import figures from '@inquirer/figures'; import { Agent, AgentCreateConfig, SfAgent } from '@salesforce/agents'; import { theme } from '../../../inquirer-theme.js'; diff --git a/src/commands/agent/generate/test-cases.ts b/src/commands/agent/generate/test-cases.ts new file mode 100644 index 0000000..1329011 --- /dev/null +++ b/src/commands/agent/generate/test-cases.ts @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { dirname, join } from 'node:path'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { select, input, confirm, checkbox } from '@inquirer/prompts'; +import { XMLParser } from 'fast-xml-parser'; +import { theme } from '../../../inquirer-theme.js'; +import { readDir } from '../../../read-dir.js'; +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.test-cases'); + +// TODO: add these back once we refine the regex +// export const FORTY_CHAR_API_NAME_REGEX = +// /^(?=.{1,57}$)[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,14}(__[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,39})?$/; +// export const EIGHTY_CHAR_API_NAME_REGEX = +// /^(?=.{1,97}$)[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,14}(__[a-zA-Z]([a-zA-Z0-9]|_(?!_)){0,79})?$/; + +export type TestSetInputs = { + utterance: string; + actionSequenceExpectedValue: string[]; + botRatingExpectedValue: string; + topicSequenceExpectedValue: string; +}; + +function castArray(value: T | T[]): T[] { + return Array.isArray(value) ? value : [value]; +} + +async function promptForTestCase(genAiPlugins: Record): Promise { + const utterance = await input({ + message: 'Utterance', + validate: (d: string): boolean | string => d.length > 0 || 'utterance cannot be empty', + theme, + }); + + const customKey = ''; + + const topics = Object.keys(genAiPlugins); + + const askForOtherActions = async (): Promise => + ( + await input({ + message: 'Expected action(s)', + validate: (d: string): boolean | string => { + if (!d.length) { + return 'expected value cannot be empty'; + } + return true; + }, + theme, + }) + ) + .split(',') + .map((a) => a.trim()); + + const askForBotRating = async (): Promise => + input({ + message: 'Expected response', + validate: (d: string): boolean | string => { + if (!d.length) { + return 'expected value cannot be empty'; + } + + return true; + }, + theme, + }); + + const topicSequenceExpectedValue = await select({ + message: 'Expected topic', + choices: [...topics, customKey], + theme, + }); + + if (topicSequenceExpectedValue === customKey) { + return { + utterance, + topicSequenceExpectedValue: await input({ + message: 'Expected topic', + validate: (d: string): boolean | string => { + if (!d.length) { + return 'expected value cannot be empty'; + } + return true; + }, + theme, + }), + // If the user selects OTHER for the topic, then we don't have a genAiPlugin to get actions from so we ask for them for custom input + actionSequenceExpectedValue: await askForOtherActions(), + botRatingExpectedValue: await askForBotRating(), + }; + } + + const genAiPluginXml = await readFile(genAiPlugins[topicSequenceExpectedValue], 'utf-8'); + const parser = new XMLParser(); + const parsed = parser.parse(genAiPluginXml) as { GenAiPlugin: { genAiFunctions: Array<{ functionName: string }> } }; + const actions = castArray(parsed.GenAiPlugin.genAiFunctions).map((f) => f.functionName); + + let actionSequenceExpectedValue = await checkbox({ + message: 'Expected action(s)', + choices: [...actions, customKey], + theme, + required: true, + }); + + if (actionSequenceExpectedValue.includes(customKey)) { + const additional = await askForOtherActions(); + + actionSequenceExpectedValue = [...actionSequenceExpectedValue.filter((a) => a !== customKey), ...additional]; + } + + const botRatingExpectedValue = await askForBotRating(); + + return { + utterance, + actionSequenceExpectedValue, + botRatingExpectedValue, + topicSequenceExpectedValue, + }; +} + +export function constructTestSetXML(testCases: TestSetInputs[]): string { + const tab = ' '; + let xml = `\n\n${tab}AGENT\n`; + testCases.forEach((testCase, i) => { + xml += ` + ${i + 1} + + ${testCase.utterance} + + + topic_sequence_match + ${testCase.topicSequenceExpectedValue} + + + action_sequence_match + ${`[${testCase.actionSequenceExpectedValue.map((v) => `"${v}"`).join(',')}]`} + + + bot_response_rating + ${testCase.botRatingExpectedValue} + + \n`; + }); + xml += ''; + return xml; +} + +export default class AgentGenerateTestCases extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly enableJsonFlag = false; + public static readonly state = 'beta'; + + public async run(): Promise { + const testSetName = await input({ + message: 'What is the name of this set of test cases', + + validate(d: string): boolean | string { + // ensure that it's not empty + if (!d.length) { + return 'Name cannot be empty'; + } + + return true; + + // TODO: add back validation once we refine the regex + // check against FORTY_CHAR_API_NAME_REGEX + // if (!FORTY_CHAR_API_NAME_REGEX.test(d)) { + // return 'The non-namespaced portion an API name must begin with a letter, contain only letters, numbers, and underscores, not contain consecutive underscores, and not end with an underscore.'; + // } + // return true; + }, + }); + + const genAiPluginDir = join('force-app', 'main', 'default', 'genAiPlugins'); + const genAiPlugins = Object.fromEntries( + (await readDir(genAiPluginDir)).map((genAiPlugin) => [ + genAiPlugin.replace('.genAiPlugin-meta.xml', ''), + join(genAiPluginDir, genAiPlugin), + ]) + ); + + const testCases = []; + do { + this.log(); + this.styledHeader(`Adding test case #${testCases.length + 1}`); + // eslint-disable-next-line no-await-in-loop + testCases.push(await promptForTestCase(genAiPlugins)); + } while ( // eslint-disable-next-line no-await-in-loop + await confirm({ + message: 'Would you like to add another test case', + default: true, + }) + ); + + const testSetPath = join( + 'force-app', + 'main', + 'default', + 'aiEvaluationTestSets', + `${testSetName}.aiEvaluationTestSet-meta.xml` + ); + await mkdir(dirname(testSetPath), { recursive: true }); + this.log(); + this.log(`Created ${testSetPath}`); + await writeFile(testSetPath, constructTestSetXML(testCases)); + } +} diff --git a/src/commands/agent/generate/definition.ts b/src/commands/agent/generate/test-definition.ts similarity index 67% rename from src/commands/agent/generate/definition.ts rename to src/commands/agent/generate/test-definition.ts index e9d8ae1..9ec1edf 100644 --- a/src/commands/agent/generate/definition.ts +++ b/src/commands/agent/generate/test-definition.ts @@ -5,17 +5,17 @@ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ import { dirname, join } from 'node:path'; -import { mkdir, readdir, writeFile } from 'node:fs/promises'; +import { mkdir, writeFile } from 'node:fs/promises'; import { SfCommand } from '@salesforce/sf-plugins-core'; import { Messages, SfError } from '@salesforce/core'; -import select from '@inquirer/select'; -import input from '@inquirer/input'; +import { input, select } from '@inquirer/prompts'; import { theme } from '../../../inquirer-theme.js'; +import { readDir } from '../../../read-dir.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); -const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.definition'); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.test-definition'); -export default class AgentGenerateDefinition extends SfCommand { +export default class AgentGenerateTestDefinition extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); @@ -24,7 +24,7 @@ export default class AgentGenerateDefinition extends SfCommand { public async run(): Promise { const testSetDir = join('force-app', 'main', 'default', 'aiEvaluationTestSets'); - const testSets = (await readdir(testSetDir)).map((testSet) => testSet.replace('.xml', '')); + const testSets = (await readDir(testSetDir)).map((testSet) => testSet.replace('.aiEvaluationTestSet-meta.xml', '')); if (testSets.length === 0) { throw new SfError(`No test sets found in ${testSetDir}`, 'NoTestSetsFoundError', [ 'Run the "sf agent generate testset" command to create a test set', @@ -32,57 +32,62 @@ export default class AgentGenerateDefinition extends SfCommand { } const botsDir = join('force-app', 'main', 'default', 'bots'); - const bots = await readdir(botsDir); + const bots = await readDir(botsDir); if (bots.length === 0) { - throw new SfError(`No bots found in ${botsDir}`, 'NoBotsFoundError'); + throw new SfError(`No agents found in ${botsDir}`, 'NoAgentsFoundError'); } - const testSet = await select({ - message: 'Select the AiEvaluationTestSet to use', - choices: testSets, + const subjectType = await select({ + message: 'What are you testing', + choices: ['AGENT'], theme, }); - const bot = await select({ - message: 'Select the Bot to run the tests against', + const agent = await select({ + message: 'Select the Agent to test', choices: bots, theme, }); - const name = await input({ - message: 'Enter a name for the AiEvaluationDefinition', - validate: (i: string): string | boolean => (i.length > 0 ? true : 'Name cannot be empty'), + const testSet = await select({ + message: 'Select the test set to use', + choices: testSets, theme, }); - const description = await input({ - message: 'Enter a description for the AiEvaluationDefinition', + const name = await input({ + message: 'Enter a name for the test definition', + validate: (i: string): string | boolean => (i.length > 0 ? true : 'Name cannot be empty'), theme, }); - const subjectType = await select({ - message: 'Select the type for the AiEvaluationDefinition', - choices: ['AGENT'], + const description = await input({ + message: 'Enter a description for test definition (optional)', theme, }); - this.log(`Generating AiEvaluationDefinition for ${bot} using ${testSet} AiEvaluationTestSet`); - const xml = ` ${description ? `${description}` : ''} ${name} ${subjectType} - ${bot} + ${agent} ${testSet} `; // remove all empty lines const cleanedXml = xml.replace(/^\s*[\r\n]/gm, ''); - const definitionPath = join('force-app', 'main', 'default', 'aiEvaluationDefinitions', `${name}.xml`); + const definitionPath = join( + 'force-app', + 'main', + 'default', + 'aiEvaluationDefinitions', + `${name}.aiEvaluationDefinition-meta.xml` + ); await mkdir(dirname(definitionPath), { recursive: true }); - this.log(`Writing AiEvaluationDefinition to ${definitionPath}`); + this.log(); + this.log(`Created ${definitionPath}`); await writeFile(definitionPath, cleanedXml); } } diff --git a/src/commands/agent/generate/testset.ts b/src/commands/agent/generate/testset.ts deleted file mode 100644 index 7a80a21..0000000 --- a/src/commands/agent/generate/testset.ts +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright (c) 2024, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { dirname, join } from 'node:path'; -import { mkdir, writeFile } from 'node:fs/promises'; -import { SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; -import input from '@inquirer/input'; -import confirm from '@inquirer/confirm'; -import { theme } from '../../../inquirer-theme.js'; - -Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); -const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.testset'); - -export type TestSetInputs = { - utterance: string; - actionSequenceExpectedValue: string; - botRatingExpectedValue: string; - topicSequenceExpectedValue: string; -}; - -async function promptForTestCase(): Promise { - const utterance = await input({ - message: 'What utterance would you like to test?', - validate: (d: string): boolean | string => d.length > 0 || 'utterance cannot be empty', - theme, - }); - - const topicSequenceExpectedValue = await input({ - message: 'What is the expected value for the topic expectation?', - validate: (d: string): boolean | string => { - if (!d.length) { - return 'expected value cannot be empty'; - } - return true; - }, - theme, - }); - - const actionSequenceExpectedValue = await input({ - message: 'What is the expected value for the action expectation?', - validate: (d: string): boolean | string => { - if (!d.length) { - return 'expected value cannot be empty'; - } - return true; - }, - theme, - }); - - const botRatingExpectedValue = await input({ - message: 'What is the expected value for the bot rating expectation?', - validate: (d: string): boolean | string => { - if (!d.length) { - return 'expected value cannot be empty'; - } - - return true; - }, - theme, - }); - - return { - utterance, - actionSequenceExpectedValue, - botRatingExpectedValue, - topicSequenceExpectedValue, - }; -} - -export function constructTestSetXML(testCases: TestSetInputs[]): string { - const tab = ' '; - let xml = `\n\n${tab}AGENT\n`; - testCases.forEach((testCase, i) => { - xml += ` - ${i + 1} - - ${testCase.utterance} - - - - topic_sequence_match - ${testCase.topicSequenceExpectedValue} - - - action_sequence_match - ${`[${testCase.actionSequenceExpectedValue - .split(',') - .map((v) => `"${v}"`) - .join(',')}]`} - - - bot_response_rating - ${testCase.botRatingExpectedValue} - - - \n`; - }); - xml += ''; - return xml; -} - -export default class AgentGenerateTestset extends SfCommand { - public static readonly summary = messages.getMessage('summary'); - public static readonly description = messages.getMessage('description'); - public static readonly examples = messages.getMessages('examples'); - public static readonly enableJsonFlag = false; - public static readonly state = 'beta'; - - public async run(): Promise { - const testSetName = await input({ - message: 'What is the name of the test set?', - }); - const testCases = []; - do { - this.log(); - this.styledHeader(`Adding test case #${testCases.length + 1}`); - // eslint-disable-next-line no-await-in-loop - testCases.push(await promptForTestCase()); - } while ( // eslint-disable-next-line no-await-in-loop - await confirm({ - message: 'Would you like to add another test case?', - default: true, - }) - ); - - const testSetPath = join('force-app', 'main', 'default', 'aiEvaluationTestsets', `${testSetName}.xml`); - await mkdir(dirname(testSetPath), { recursive: true }); - this.log(); - this.log(`Writing new AiEvaluationTestSet to ${testSetPath}`); - await writeFile(testSetPath, constructTestSetXML(testCases)); - } -} diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 7fb2ef5..bcdd000 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -7,14 +7,14 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester, AgentTestDetailsResponse } from '@salesforce/agents'; +import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents'; import { resultFormatFlag, testOutputDirFlag } from '../../../flags.js'; import { handleTestResults } from '../../../handleTestResults.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.results'); -export type AgentTestResultsResult = AgentTestDetailsResponse; +export type AgentTestResultsResult = AgentTestResultsResponse; export default class AgentTestResults extends SfCommand { public static readonly summary = messages.getMessage('summary'); @@ -38,7 +38,7 @@ export default class AgentTestResults extends SfCommand const { flags } = await this.parse(AgentTestResults); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const response = await agentTester.details(flags['job-id']); + const response = await agentTester.results(flags['job-id']); await handleTestResults({ id: flags['job-id'], format: flags['result-format'], diff --git a/src/flags.ts b/src/flags.ts index 1695172..b644958 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -17,7 +17,7 @@ export const resultFormatFlag = Flags.option({ }); export const testOutputDirFlag = Flags.custom({ - char: 'f', + char: 'd', description: messages.getMessage('flags.output-dir.description'), summary: messages.getMessage('flags.output-dir.summary'), }); diff --git a/src/handleTestResults.ts b/src/handleTestResults.ts index 1a6cceb..4cf2e1d 100644 --- a/src/handleTestResults.ts +++ b/src/handleTestResults.ts @@ -6,7 +6,7 @@ */ import { join } from 'node:path'; import { writeFile, mkdir } from 'node:fs/promises'; -import { AgentTestDetailsResponse, jsonFormat, humanFormat, junitFormat, tapFormat } from '@salesforce/agents'; +import { AgentTestResultsResponse, jsonFormat, humanFormat, junitFormat, tapFormat } from '@salesforce/agents'; import { Ux } from '@salesforce/sf-plugins-core/Ux'; async function writeFileToDir(outputDir: string, fileName: string, content: string): Promise { @@ -25,7 +25,7 @@ export async function handleTestResults({ }: { id: string; format: 'human' | 'json' | 'junit' | 'tap'; - results: AgentTestDetailsResponse | undefined; + results: AgentTestResultsResponse | undefined; jsonEnabled: boolean; outputDir?: string; }): Promise { @@ -38,33 +38,45 @@ export async function handleTestResults({ if (format === 'human') { const formatted = await humanFormat(results); - ux.log(formatted); if (outputDir) { - await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted); + const file = `test-result-${id}.txt`; + await writeFileToDir(outputDir, file, formatted); + ux.log(`Created human-readable file at ${join(outputDir, file)}`); + } else { + ux.log(formatted); } } if (format === 'json') { const formatted = await jsonFormat(results); - ux.log(formatted); if (outputDir) { - await writeFileToDir(outputDir, `test-result-${id}.json`, formatted); + const file = `test-result-${id}.json`; + await writeFileToDir(outputDir, file, formatted); + ux.log(`Created JSON file at ${join(outputDir, file)}`); + } else { + ux.log(formatted); } } if (format === 'junit') { const formatted = await junitFormat(results); - ux.log(formatted); if (outputDir) { - await writeFileToDir(outputDir, `test-result-${id}.xml`, formatted); + const file = `test-result-${id}.xml`; + await writeFileToDir(outputDir, file, formatted); + ux.log(`Created JUnit file at ${join(outputDir, file)}`); + } else { + ux.log(formatted); } } if (format === 'tap') { const formatted = await tapFormat(results); - ux.log(formatted); if (outputDir) { - await writeFileToDir(outputDir, `test-result-${id}.txt`, formatted); + const file = `test-result-${id}.txt`; + await writeFileToDir(outputDir, file, formatted); + ux.log(`Created TAP file at ${join(outputDir, file)}`); + } else { + ux.log(formatted); } } } diff --git a/src/read-dir.ts b/src/read-dir.ts new file mode 100644 index 0000000..ab8adab --- /dev/null +++ b/src/read-dir.ts @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { readdir } from 'node:fs/promises'; +export async function readDir(path: string): Promise { + try { + return (await readdir(path)).filter((bot) => !bot.startsWith('.')); + } catch { + return []; + } +} diff --git a/src/testStages.ts b/src/testStages.ts index 98e7d87..85d6a38 100644 --- a/src/testStages.ts +++ b/src/testStages.ts @@ -7,7 +7,7 @@ import { colorize } from '@oclif/core/ux'; import { MultiStageOutput } from '@oclif/multi-stage-output'; -import { AgentTestDetailsResponse, AgentTester } from '@salesforce/agents'; +import { AgentTestResultsResponse, AgentTester } from '@salesforce/agents'; import { Lifecycle } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; import { Ux } from '@salesforce/sf-plugins-core'; @@ -80,7 +80,7 @@ export class TestStages { agentTester: AgentTester, id: string, wait: Duration - ): Promise<{ completed: boolean; response?: AgentTestDetailsResponse }> { + ): Promise<{ completed: boolean; response?: AgentTestResultsResponse }> { this.mso.skipTo('Polling for Test Results'); const lifecycle = Lifecycle.getInstance(); lifecycle.on( diff --git a/test/commands/agent/generate/test-cases.test.ts b/test/commands/agent/generate/test-cases.test.ts new file mode 100644 index 0000000..6e53176 --- /dev/null +++ b/test/commands/agent/generate/test-cases.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { expect } from 'chai'; +import { type TestSetInputs, constructTestSetXML } from '../../../../src/commands/agent/generate/test-cases.js'; + +describe('constructTestSetXML', () => { + it('should return a valid test set XML', () => { + const testCases = [ + { + utterance: 'hello', + actionSequenceExpectedValue: ['foo', 'bar'], + botRatingExpectedValue: 'baz', + topicSequenceExpectedValue: 'qux', + }, + { + utterance: 'goodbye', + actionSequenceExpectedValue: ['foo', 'bar'], + botRatingExpectedValue: 'baz', + topicSequenceExpectedValue: 'qux', + }, + { + utterance: 'how are you', + actionSequenceExpectedValue: ['foo', 'bar'], + botRatingExpectedValue: 'baz', + topicSequenceExpectedValue: 'qux', + }, + ] satisfies TestSetInputs[]; + + const xml = constructTestSetXML(testCases); + + expect(xml).to.equal(` + + AGENT + + 1 + + hello + + + topic_sequence_match + qux + + + action_sequence_match + ["foo","bar"] + + + bot_response_rating + baz + + + + 2 + + goodbye + + + topic_sequence_match + qux + + + action_sequence_match + ["foo","bar"] + + + bot_response_rating + baz + + + + 3 + + how are you + + + topic_sequence_match + qux + + + action_sequence_match + ["foo","bar"] + + + bot_response_rating + baz + + +`); + }); +}); diff --git a/test/commands/agent/generate/testset.test.ts b/test/commands/agent/generate/testset.test.ts deleted file mode 100644 index 10d8eb4..0000000 --- a/test/commands/agent/generate/testset.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { expect } from 'chai'; -import { type TestSetInputs, constructTestSetXML } from '../../../../src/commands/agent/generate/testset.js'; - -describe('constructTestSetXML', () => { - it('should return a valid test set XML', () => { - const testCases = [ - { - utterance: 'hello', - actionSequenceExpectedValue: 'foo,bar', - botRatingExpectedValue: 'baz', - topicSequenceExpectedValue: 'qux', - }, - { - utterance: 'goodbye', - actionSequenceExpectedValue: 'foo,bar', - botRatingExpectedValue: 'baz', - topicSequenceExpectedValue: 'qux', - }, - { - utterance: 'how are you', - actionSequenceExpectedValue: 'foo,bar', - botRatingExpectedValue: 'baz', - topicSequenceExpectedValue: 'qux', - }, - ] satisfies TestSetInputs[]; - - const xml = constructTestSetXML(testCases); - - expect(xml).to.equal(` - - AGENT - - 1 - - hello - - - - topic_sequence_match - qux - - - action_sequence_match - ["foo","bar"] - - - bot_response_rating - baz - - - - - 2 - - goodbye - - - - topic_sequence_match - qux - - - action_sequence_match - ["foo","bar"] - - - bot_response_rating - baz - - - - - 3 - - how are you - - - - topic_sequence_match - qux - - - action_sequence_match - ["foo","bar"] - - - bot_response_rating - baz - - - -`); - }); -}); diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_results.json similarity index 100% rename from test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json rename to test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_results.json diff --git a/yarn.lock b/yarn.lock index 6e11caa..696183d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1053,6 +1053,17 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz#d9fae00a2d5cb40f92cfe64b47ad749fbc38f917" integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw== +"@inquirer/checkbox@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/checkbox/-/checkbox-4.0.3.tgz#cbd9694e925964f5b0432cc84ab107a8d7a8202d" + integrity sha512-CEt9B4e8zFOGtc/LYeQx5m8nfqQeG/4oNNv0PUvXGG0mys+wR/WbJ3B4KfSQ4Fcr3AQfpiuFOi3fVvmPfvNbxw== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/figures" "^1.0.8" + "@inquirer/type" "^3.0.1" + ansi-escapes "^4.3.2" + yoctocolors-cjs "^2.1.2" + "@inquirer/confirm@^3.1.22", "@inquirer/confirm@^3.2.0": version "3.2.0" resolved "https://registry.yarnpkg.com/@inquirer/confirm/-/confirm-3.2.0.tgz#6af1284670ea7c7d95e3f1253684cfbd7228ad6a" @@ -1069,21 +1080,6 @@ "@inquirer/core" "^10.1.1" "@inquirer/type" "^3.0.1" -"@inquirer/core@^10.0.1": - version "10.0.1" - resolved "https://registry.yarnpkg.com/@inquirer/core/-/core-10.0.1.tgz#22068da87d8f6317452172dfd521e811ccbcb90e" - integrity sha512-KKTgjViBQUi3AAssqjUFMnMO3CM3qwCHvePV9EW+zTKGKafFGFF01sc1yOIYjLJ7QU52G/FbzKc+c01WLzXmVQ== - dependencies: - "@inquirer/figures" "^1.0.7" - "@inquirer/type" "^3.0.0" - ansi-escapes "^4.3.2" - cli-width "^4.1.0" - mute-stream "^2.0.0" - signal-exit "^4.1.0" - strip-ansi "^6.0.1" - wrap-ansi "^6.2.0" - yoctocolors-cjs "^2.1.2" - "@inquirer/core@^10.1.1": version "10.1.1" resolved "https://registry.yarnpkg.com/@inquirer/core/-/core-10.1.1.tgz#801e82649fb64bcb2b5e4667397ff8c25bccebab" @@ -1118,6 +1114,24 @@ wrap-ansi "^6.2.0" yoctocolors-cjs "^2.1.2" +"@inquirer/editor@^4.2.0": + version "4.2.0" + resolved "https://registry.yarnpkg.com/@inquirer/editor/-/editor-4.2.0.tgz#469a00e876afebcfc574bf8114e40c40795688c1" + integrity sha512-Z3LeGsD3WlItDqLxTPciZDbGtm0wrz7iJGS/uUxSiQxef33ZrBq7LhsXg30P7xrWz1kZX4iGzxxj5SKZmJ8W+w== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/type" "^3.0.1" + external-editor "^3.1.0" + +"@inquirer/expand@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/expand/-/expand-4.0.3.tgz#7593b841d9355c4e7a047071b33e5a58f202ac96" + integrity sha512-MDszqW4HYBpVMmAoy/FA9laLrgo899UAga0itEjsYrBthKieDZNc0e16gdn7N3cQ0DSf/6zsTBZMuDYDQU4ktg== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/type" "^3.0.1" + yoctocolors-cjs "^2.1.2" + "@inquirer/figures@^1.0.5", "@inquirer/figures@^1.0.7": version "1.0.7" resolved "https://registry.yarnpkg.com/@inquirer/figures/-/figures-1.0.7.tgz#d050ccc0eabfacc0248c4ff647a9dfba1b01594b" @@ -1136,13 +1150,21 @@ "@inquirer/core" "^9.0.8" "@inquirer/type" "^1.5.1" -"@inquirer/input@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@inquirer/input/-/input-4.0.1.tgz#7b676aad726e8a3baf3793cf1e9cec665a815a2b" - integrity sha512-m+SliZ2m43cDRIpAdQxfv5QOeAQCuhS8TGLvtzEP1An4IH1kBES4RLMRgE/fC+z29aN8qYG8Tq/eXQQKTYwqAg== +"@inquirer/input@^4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@inquirer/input/-/input-4.1.0.tgz#54b484550c3ecb2e7bf62149a14e9784f08efe6b" + integrity sha512-16B8A9hY741yGXzd8UJ9R8su/fuuyO2e+idd7oVLYjP23wKJ6ILRIIHcnXe8/6AoYgwRS2zp4PNsW/u/iZ24yg== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/type" "^3.0.1" + +"@inquirer/number@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/number/-/number-3.0.3.tgz#e3dd7520e21e9708fea9465b73d20ac851f5f60d" + integrity sha512-HA/W4YV+5deKCehIutfGBzNxWH1nhvUC67O4fC9ufSijn72yrYnRmzvC61dwFvlXIG1fQaYWi+cqNE9PaB9n6Q== dependencies: - "@inquirer/core" "^10.0.1" - "@inquirer/type" "^3.0.0" + "@inquirer/core" "^10.1.1" + "@inquirer/type" "^3.0.1" "@inquirer/password@^2.2.0": version "2.2.0" @@ -1153,6 +1175,50 @@ "@inquirer/type" "^1.5.3" ansi-escapes "^4.3.2" +"@inquirer/password@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/password/-/password-4.0.3.tgz#17af6d8983e2e5c0f231b382ef5c78a8b4b63e95" + integrity sha512-3qWjk6hS0iabG9xx0U1plwQLDBc/HA/hWzLFFatADpR6XfE62LqPr9GpFXBkLU0KQUaIXZ996bNG+2yUvocH8w== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/type" "^3.0.1" + ansi-escapes "^4.3.2" + +"@inquirer/prompts@^7.2.0": + version "7.2.0" + resolved "https://registry.yarnpkg.com/@inquirer/prompts/-/prompts-7.2.0.tgz#15010df2257a243866480513d36f3e19c98d7fb1" + integrity sha512-ZXYZ5oGVrb+hCzcglPeVerJ5SFwennmDOPfXq1WyeZIrPGySLbl4W6GaSsBFvu3WII36AOK5yB8RMIEEkBjf8w== + dependencies: + "@inquirer/checkbox" "^4.0.3" + "@inquirer/confirm" "^5.1.0" + "@inquirer/editor" "^4.2.0" + "@inquirer/expand" "^4.0.3" + "@inquirer/input" "^4.1.0" + "@inquirer/number" "^3.0.3" + "@inquirer/password" "^4.0.3" + "@inquirer/rawlist" "^4.0.3" + "@inquirer/search" "^3.0.3" + "@inquirer/select" "^4.0.3" + +"@inquirer/rawlist@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/rawlist/-/rawlist-4.0.3.tgz#9964521d3470e153e7e11f228a53cf0afefb217c" + integrity sha512-5MhinSzfmOiZlRoPezfbJdfVCZikZs38ja3IOoWe7H1dxL0l3Z2jAUgbBldeyhhOkELdGvPlBfQaNbeLslib1w== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/type" "^3.0.1" + yoctocolors-cjs "^2.1.2" + +"@inquirer/search@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/search/-/search-3.0.3.tgz#791f19a4ee87d65816fd3bb17bd8d76bc11bff07" + integrity sha512-mQTCbdNolTGvGGVCJSI6afDwiSGTV+fMLPEIMDJgIV6L/s3+RYRpxt6t0DYnqMQmemnZ/Zq0vTIRwoHT1RgcTg== + dependencies: + "@inquirer/core" "^10.1.1" + "@inquirer/figures" "^1.0.8" + "@inquirer/type" "^3.0.1" + yoctocolors-cjs "^2.1.2" + "@inquirer/select@^2.5.0": version "2.5.0" resolved "https://registry.yarnpkg.com/@inquirer/select/-/select-2.5.0.tgz#345c6908ecfaeef3d84ddd2f9feb2f487c558efb" @@ -1164,14 +1230,14 @@ ansi-escapes "^4.3.2" yoctocolors-cjs "^2.1.2" -"@inquirer/select@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@inquirer/select/-/select-4.0.1.tgz#fb651f0e0fb7da1256cc75a399dc2ac72a7f7df4" - integrity sha512-tVRatFRGU49bxFCKi/3P+C0E13KZduNFbWuHWRx0L2+jbiyKRpXgHp9qiRHWRk/KarhYBXzH/di6w3VQ5aJd5w== +"@inquirer/select@^4.0.3": + version "4.0.3" + resolved "https://registry.yarnpkg.com/@inquirer/select/-/select-4.0.3.tgz#24a9d744685608ff26262fccb41fa93b4dac615f" + integrity sha512-OZfKDtDE8+J54JYAFTUGZwvKNfC7W/gFCjDkcsO7HnTH/wljsZo9y/FJquOxMy++DY0+9l9o/MOZ8s5s1j5wmw== dependencies: - "@inquirer/core" "^10.0.1" - "@inquirer/figures" "^1.0.7" - "@inquirer/type" "^3.0.0" + "@inquirer/core" "^10.1.1" + "@inquirer/figures" "^1.0.8" + "@inquirer/type" "^3.0.1" ansi-escapes "^4.3.2" yoctocolors-cjs "^2.1.2" @@ -1182,11 +1248,6 @@ dependencies: mute-stream "^1.0.0" -"@inquirer/type@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@inquirer/type/-/type-3.0.0.tgz#1762ebe667ec1d838012b20bf0cf90b841ba68bc" - integrity sha512-YYykfbw/lefC7yKj7nanzQXILM7r3suIvyFlCcMskc99axmsSewXWkAfXKwMbgxL76iAFVmRwmYdwNZNc8gjog== - "@inquirer/type@^3.0.1": version "3.0.1" resolved "https://registry.yarnpkg.com/@inquirer/type/-/type-3.0.1.tgz#619ce9f65c3e114d8e39c41822bed3440d20b478" @@ -1321,6 +1382,30 @@ wordwrap "^1.0.0" wrap-ansi "^7.0.0" +"@oclif/core@^4.0.34": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.2.1.tgz#3ee038e89686d2ffe75bde63ae0091ca02cef353" + integrity sha512-oznLlFCXuOpRSfsBUv2YxNXwD0hZc4l37Vmue+1v6T1IgXD10DXNlGNxm20C24t7MgrXdvrbmnajTdzer3Xrbw== + dependencies: + ansi-escapes "^4.3.2" + ansis "^3.5.2" + clean-stack "^3.0.1" + cli-spinners "^2.9.2" + debug "^4.4.0" + ejs "^3.1.10" + get-package-type "^0.1.0" + globby "^11.1.0" + indent-string "^4.0.0" + is-wsl "^2.2.0" + lilconfig "^3.1.3" + minimatch "^9.0.5" + semver "^7.6.3" + string-width "^4.2.3" + supports-color "^8" + widest-line "^3.1.0" + wordwrap "^1.0.0" + wrap-ansi "^7.0.0" + "@oclif/multi-stage-output@^0.7.12": version "0.7.12" resolved "https://registry.yarnpkg.com/@oclif/multi-stage-output/-/multi-stage-output-0.7.12.tgz#04df5efb6dce527920cf475c9ad9f20236803ccd" @@ -1393,10 +1478,10 @@ strip-ansi "^7.1.0" wrap-ansi "^9.0.0" -"@oclif/table@^0.3.5": - version "0.3.7" - resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.3.7.tgz#b13fb4525413c570cf1c34c8f6da9e13ea7ced0e" - integrity sha512-ixk/2swooqgg07N3mICe3gvkq/G4Au3jNMaRNCWIfWtcoyT+tTrOQxD+Ead0wMBuZl/J+CyEwImIjJqC21P1kA== +"@oclif/table@^0.4.0": + version "0.4.0" + resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.4.0.tgz#3bf97121a4fc3524801883f550eeb0fd822852a9" + integrity sha512-cXmZDb0VcxIk8o5cvd8J4CUtMGJeTuEDC8wQmlZC55Dl9FHcWE9TdZxksWLLp4w0qKP0YfJCFSvfGPMeMPyO5Q== dependencies: "@oclif/core" "^4" "@types/react" "^18.3.12" @@ -1422,17 +1507,16 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@salesforce/agents@^0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.5.2.tgz#b60e7227a78e3c4431565ba6e3862aa97c65306f" - integrity sha512-Khr24nZlV875PPwrmFjfDBSMi7Hqfrb6+Y+d8I4OHMU7iP+U9lnvRAwczD9o+WB5gq6WSCkhgQY89QmAA55dKw== +"@salesforce/agents@0.5.10-dev.0": + version "0.5.10-dev.0" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.5.10-dev.0.tgz#733a3d66d9a56acef3a731f3b879a05180fa17ce" + integrity sha512-7u6x7Ufb/P406wswpJVBzpZBMWK6DE4Ib0WAwlIfL86QpQg555jJatyov1ekKQLizhr/+hKszF7lqv/cXdk4qw== dependencies: - "@oclif/table" "^0.3.5" "@salesforce/core" "^8.8.0" "@salesforce/kit" "^3.2.3" - "@salesforce/sf-plugins-core" "^12.1.0" - "@salesforce/source-deploy-retrieve" "^12.10.3" - ansis "^3.4.0" + "@salesforce/sf-plugins-core" "^12.1.2" + "@salesforce/source-deploy-retrieve" "^12.10.4" + ansis "^3.5.2" fast-xml-parser "^4" nock "^13.5.6" @@ -1578,16 +1662,32 @@ cli-progress "^3.12.0" terminal-link "^3.0.0" -"@salesforce/source-deploy-retrieve@^12.10.3": - version "12.10.3" - resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.10.3.tgz#fa16910781188877ffdfa5fde3a0318c0dfe3d07" - integrity sha512-bKIcN6VJajre2chF1xhPCjtR9gZpp8PrFFZ55UcWUMkoFAXscBPRJ7poAeorted3qMzS6wx+AuB27qYUCO+4iQ== +"@salesforce/sf-plugins-core@^12.1.2": + version "12.1.2" + resolved "https://registry.yarnpkg.com/@salesforce/sf-plugins-core/-/sf-plugins-core-12.1.2.tgz#0c4fba3787ee670014c8bca447c2fc59fb9c3ec3" + integrity sha512-qrmGZr2mG7uOpqhOpeOJjBEVztM7Mt8B10lD3rG+aOBZQelTm3YQ738yZYcwHjksQW3AOyf0QtqZICCfe0nD8A== + dependencies: + "@inquirer/confirm" "^3.1.22" + "@inquirer/password" "^2.2.0" + "@oclif/core" "^4.0.34" + "@oclif/table" "^0.4.0" + "@salesforce/core" "^8.5.1" + "@salesforce/kit" "^3.2.3" + "@salesforce/ts-types" "^2.0.12" + ansis "^3.3.2" + cli-progress "^3.12.0" + terminal-link "^3.0.0" + +"@salesforce/source-deploy-retrieve@^12.10.4": + version "12.11.0" + resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.11.0.tgz#1cc4445c98272128177161b2a31339e720818b12" + integrity sha512-9j4iXVBUrHZ7eXgmKYyPTUOHUVwTENJntf3FuIdBMyTN+QFKVeyUJ11vHHPvn0PNI8BZ+b51CCp9kdqIG1K36Q== dependencies: "@salesforce/core" "^8.8.0" "@salesforce/kit" "^3.2.2" "@salesforce/ts-types" "^2.0.12" fast-levenshtein "^3.0.0" - fast-xml-parser "^4.5.0" + fast-xml-parser "^4.5.1" got "^11.8.6" graceful-fs "^4.2.11" ignore "^5.3.2" @@ -2632,10 +2732,10 @@ ansis@^3.3.1, ansis@^3.3.2: resolved "https://registry.yarnpkg.com/ansis/-/ansis-3.3.2.tgz#15adc36fea112da95c74d309706e593618accac3" integrity sha512-cFthbBlt+Oi0i9Pv/j6YdVWJh54CtjGACaMPCIrEV4Ha7HWsIjXDwseYV79TIL0B4+KfSwD5S70PeQDkPUd1rA== -ansis@^3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/ansis/-/ansis-3.4.0.tgz#d49ebb72f228aa122733a585c600dcba4f5f0838" - integrity sha512-zVESKSQhWaPhGaWiKj1k+UqvpC7vPBBgG3hjQEeIx2YGzylWt8qA3ziAzRuUtm0OnaGsZKjIvfl8D/sJTt/I0w== +ansis@^3.5.2: + version "3.6.0" + resolved "https://registry.yarnpkg.com/ansis/-/ansis-3.6.0.tgz#f4d8437fb27659bf5a6adca90135801919dee764" + integrity sha512-8KluYVZM+vx19f5rInhdEBdIAjvBp7ASzyF/DoStcDpMJ3JOM55ybvUcs9nMRVP8XN2K3ABBdO7zCXezvrT0pg== anymatch@~3.1.2: version "3.1.2" @@ -3142,6 +3242,11 @@ change-case@^5.4.4: resolved "https://registry.yarnpkg.com/change-case/-/change-case-5.4.4.tgz#0d52b507d8fb8f204343432381d1a6d7bff97a02" integrity sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w== +chardet@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" + integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== + check-error@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.3.tgz#a6502e4312a7ee969f646e83bb3ddd56281bd694" @@ -3503,6 +3608,13 @@ debug@^3.2.7: dependencies: ms "^2.1.1" +debug@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.0.tgz#2b3f2aea2ffeb776477460267377dc8710faba8a" + integrity sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA== + dependencies: + ms "^2.1.3" + decamelize-keys@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9" @@ -4275,6 +4387,15 @@ extend@^3.0.2: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== +external-editor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/external-editor/-/external-editor-3.1.0.tgz#cb03f740befae03ea4d283caed2741a83f335495" + integrity sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew== + dependencies: + chardet "^0.7.0" + iconv-lite "^0.4.24" + tmp "^0.0.33" + fast-copy@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/fast-copy/-/fast-copy-3.0.2.tgz#59c68f59ccbcac82050ba992e0d5c389097c9d35" @@ -4335,13 +4456,20 @@ fast-xml-parser@4.4.1: dependencies: strnum "^1.0.5" -fast-xml-parser@^4, fast-xml-parser@^4.5.0: +fast-xml-parser@^4: version "4.5.0" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz#2882b7d01a6825dfdf909638f2de0256351def37" integrity sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg== dependencies: strnum "^1.0.5" +fast-xml-parser@^4.5.1: + version "4.5.1" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.5.1.tgz#a7e665ff79b7919100a5202f23984b6150f9b31e" + integrity sha512-y655CeyUQ+jj7KBbYMc4FG01V8ZQqjN+gDYGJ50RtfsUB8iG9AmwmwoAgeKLJdmueKKMrH1RJ7yXHTSoczdv5w== + dependencies: + strnum "^1.0.5" + fastest-levenshtein@^1.0.7: version "1.0.16" resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5" @@ -5072,6 +5200,13 @@ husky@^7.0.4: resolved "https://registry.yarnpkg.com/husky/-/husky-7.0.4.tgz#242048245dc49c8fb1bf0cc7cfb98dd722531535" integrity sha512-vbaCKN2QLtP/vD4yvs6iz6hBEo6wkSzs8HpRah1Z6aGmF2KW5PdYuAd7uX5a+OyBZHBhd+TFLqgjUgytQr4RvQ== +iconv-lite@^0.4.24: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -5891,6 +6026,11 @@ lilconfig@^3.1.2: resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-3.1.2.tgz#e4a7c3cb549e3a606c8dcc32e5ae1005e62c05cb" integrity sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow== +lilconfig@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-3.1.3.tgz#a1bcfd6257f9585bf5ae14ceeebb7b559025e4c4" + integrity sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw== + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -6663,6 +6803,11 @@ optionator@^0.9.3: prelude-ls "^1.2.1" type-check "^0.4.0" +os-tmpdir@~1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" + integrity sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g== + p-cancelable@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" @@ -7402,6 +7547,11 @@ safe-stable-stringify@^2.3.1, safe-stable-stringify@^2.4.3: resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886" integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g== +"safer-buffer@>= 2.1.2 < 3": + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -8033,6 +8183,13 @@ tiny-jsonc@^1.0.1: resolved "https://registry.yarnpkg.com/tiny-jsonc/-/tiny-jsonc-1.0.1.tgz#71de47c9d812b411e87a9f3ab4a5fe42cd8d8f9c" integrity sha512-ik6BCxzva9DoiEfDX/li0L2cWKPPENYvixUprFdl3YPi4bZZUhDnNI9YUkacrv+uIG90dnxR5mNqaoD6UhD6Bw== +tmp@^0.0.33: + version "0.0.33" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" + integrity sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw== + dependencies: + os-tmpdir "~1.0.2" + to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"