From 81ec3a3d3fb253e8d721881dfba7fb0036323e84 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Thu, 14 Nov 2024 13:31:41 -0700 Subject: [PATCH 01/12] feat: implement agent test run --- command-snapshot.json | 4 +- messages/agent.test.run.md | 4 ++ package.json | 3 +- src/commands/agent/test/run.ts | 77 +++++++++++++++++++++++++++----- test/unit/agent-test-run.test.ts | 17 ++----- yarn.lock | 10 ++++- 6 files changed, 87 insertions(+), 28 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index 176d7c7..0dba124 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -39,8 +39,8 @@ "alias": [], "command": "agent:test:run", "flagAliases": [], - "flagChars": ["d", "i", "o", "w"], - "flags": ["flags-dir", "id", "json", "output-dir", "target-org", "wait"], + "flagChars": ["d", "i", "o", "r", "w"], + "flags": ["api-version", "flags-dir", "id", "json", "output-dir", "result-format", "target-org", "wait"], "plugin": "@salesforce/plugin-agent" } ] diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 4b73b25..0d5a90a 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -26,6 +26,10 @@ If the command continues to run after the wait period, the CLI returns control o Directory in which to store test run files. +# flags.result-format.summary + +Format of the test run results. + # examples - Start a test for an Agent: diff --git a/package.json b/package.json index 6e40d2f..2824bb6 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,8 @@ "@inquirer/input": "^4.0.1", "@inquirer/select": "^4.0.1", "@oclif/core": "^4", - "@salesforce/agents": "^0.1.4", "@oclif/multi-stage-output": "^0.7.12", + "@salesforce/agents": "^0.1.4", "@salesforce/core": "^8.5.2", "@salesforce/kit": "^3.2.1", "@salesforce/sf-plugins-core": "^12", @@ -18,6 +18,7 @@ }, "devDependencies": { "@oclif/plugin-command-snapshot": "^5.2.19", + "@oclif/test": "^4.1.0", "@salesforce/cli-plugins-testkit": "^5.3.35", "@salesforce/dev-scripts": "^10.2.10", "@salesforce/plugin-command-reference": "^3.1.29", diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 2a55c7a..45b4269 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -5,12 +5,18 @@ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { MultiStageOutput } from '@oclif/multi-stage-output'; import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { Lifecycle, Messages } from '@salesforce/core'; +import { AgentTester } from '@salesforce/agents'; +import { colorize } from '@oclif/core/ux'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); +const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => + (e as { name: string })?.name === 'PollingClientTimeout'; + export type AgentTestRunResult = { jobId: string; // AiEvaluation.Id success: boolean; @@ -25,6 +31,7 @@ export default class AgentTestRun extends SfCommand { public static readonly flags = { 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), // AiEvalDefinitionVersion.Id -- This should really be "test-name" id: Flags.string({ char: 'i', @@ -32,6 +39,8 @@ export default class AgentTestRun extends SfCommand { summary: messages.getMessage('flags.id.summary'), description: messages.getMessage('flags.id.description'), }), + // we want to pass `undefined` to the API + // eslint-disable-next-line sf-plugin/flag-min-max-default wait: Flags.duration({ char: 'w', unit: 'minutes', @@ -43,27 +52,73 @@ export default class AgentTestRun extends SfCommand { char: 'd', summary: messages.getMessage('flags.output-dir.summary'), }), + 'result-format': Flags.option({ + options: ['json', 'human', 'tap', 'junit'], + default: 'human', + char: 'r', + summary: messages.getMessage('flags.result-format.summary'), + })(), // // Future flags: - // result-format [csv, json, table, junit, TAP] // suites [array of suite names] // verbose [boolean] - // ??? api-version or build-version ??? }; public async run(): Promise { const { flags } = await this.parse(AgentTestRun); + const mso = new MultiStageOutput<{ id: string; status: string }>({ + jsonEnabled: this.jsonEnabled(), + title: `Agent Test Run: ${flags.id}`, + stages: ['Starting Tests', 'Polling for Test Results'], + stageSpecificBlock: [ + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Status', + get: (data) => data?.status, + }, + ], + postStagesBlock: [ + { + type: 'dynamic-key-value', + label: 'Job ID', + get: (data) => data?.id, + }, + ], + }); + mso.skipTo('Starting Tests'); + const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + const response = await agentTester.start(flags.id); + mso.updateData({ id: response.id }); + if (flags.wait?.minutes) { + mso.skipTo('Polling for Test Results'); + const lifecycle = Lifecycle.getInstance(); + lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => + Promise.resolve(mso.updateData({ status: event?.status })) + ); + try { + const { formatted } = await agentTester.poll(response.id, { timeout: flags.wait }); + mso.stop(); + this.log(formatted); + } catch (e) { + if (isTimeoutError(e)) { + mso.stop('async'); + this.log(`Client timed out after ${flags.wait.minutes} minutes.`); + this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`); + } else { + mso.error(); + throw e; + } + } + } else { + mso.stop(); + this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`); + } - this.log(`Starting tests for AiEvalDefinitionVersion: ${flags.id}`); - - // Call SF Eval Connect API passing AiEvalDefinitionVersion.Id - // POST to /einstein/ai-evaluations/{aiEvalDefinitionVersionId}/start - - // Returns: AiEvaluation.Id - + mso.stop(); return { success: true, - jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id; needed for getting status and stopping + jobId: response.id, // AiEvaluation.Id; needed for getting status and stopping }; } } diff --git a/test/unit/agent-test-run.test.ts b/test/unit/agent-test-run.test.ts index 0bb1cd1..0b13124 100644 --- a/test/unit/agent-test-run.test.ts +++ b/test/unit/agent-test-run.test.ts @@ -4,30 +4,21 @@ * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ + +import { runCommand } from '@oclif/test'; import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup'; import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; -import AgentTestRun from '../../src/commands/agent/test/run.js'; describe('agent run test', () => { const $$ = new TestContext(); const testOrg = new MockTestOrgData(); - let sfCommandStubs: ReturnType; - - beforeEach(() => { - sfCommandStubs = stubSfCommandUx($$.SANDBOX); - }); afterEach(() => { $$.restore(); }); it('runs agent run test', async () => { - await AgentTestRun.run(['-i', 'the-id', '-o', testOrg.username]); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('Starting tests for AiEvalDefinitionVersion:'); + const { stdout } = await runCommand(`agent:test:run -i the-id -o ${testOrg.username}`); + expect(stdout).to.include('Agent Test Run: the-id'); }); }); diff --git a/yarn.lock b/yarn.lock index 923425d..0588b29 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1365,6 +1365,14 @@ strip-ansi "^7.1.0" wrap-ansi "^9.0.0" +"@oclif/test@^4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@oclif/test/-/test-4.1.0.tgz#7935e3707cf07480790139e02973196d18d16822" + integrity sha512-2ugir6NhRsWJqHM9d2lMEWNiOTD678Jlx5chF/fg6TCAlc7E6E/6+zt+polrCTnTIpih5P/HxOtDekgtjgARwQ== + dependencies: + ansis "^3.3.2" + debug "^4.3.6" + "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" @@ -3217,7 +3225,7 @@ dateformat@^4.6.3: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.7: +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.3.5, debug@^4.3.6, debug@^4.3.7: version "4.3.7" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== From 3f942b22aed7d532f3243654292809af07dfb592 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Tue, 26 Nov 2024 09:58:48 -0700 Subject: [PATCH 02/12] feat: add command stubs --- messages/agent.generate.test.md | 19 +++++++ messages/agent.test.cancel.md | 2 +- messages/agent.test.results.md | 19 +++++++ messages/agent.test.resume.md | 19 +++++++ package.json | 15 ++++-- schemas/agent-create.json | 6 +-- schemas/agent-generate-spec.json | 6 +-- schemas/agent-test-cancel.json | 7 +-- schemas/agent-test-run.json | 7 +-- src/commands/agent/generate/test.ts | 41 ++++++++++++++++ src/commands/agent/test/cancel.ts | 2 +- src/commands/agent/test/results.ts | 41 ++++++++++++++++ src/commands/agent/test/resume.ts | 41 ++++++++++++++++ src/commands/agent/test/run.ts | 11 +++-- test/commands/agent/generate/test.nut.ts | 27 ++++++++++ test/commands/agent/generate/test.test.ts | 46 +++++++++++++++++ test/commands/agent/test/results.nut.ts | 27 ++++++++++ test/commands/agent/test/results.test.ts | 46 +++++++++++++++++ test/commands/agent/test/resume.nut.ts | 27 ++++++++++ test/commands/agent/test/resume.test.ts | 46 +++++++++++++++++ yarn.lock | 60 +++++++++-------------- 21 files changed, 453 insertions(+), 62 deletions(-) create mode 100644 messages/agent.generate.test.md create mode 100644 messages/agent.test.results.md create mode 100644 messages/agent.test.resume.md create mode 100644 src/commands/agent/generate/test.ts create mode 100644 src/commands/agent/test/results.ts create mode 100644 src/commands/agent/test/resume.ts create mode 100644 test/commands/agent/generate/test.nut.ts create mode 100644 test/commands/agent/generate/test.test.ts create mode 100644 test/commands/agent/test/results.nut.ts create mode 100644 test/commands/agent/test/results.test.ts create mode 100644 test/commands/agent/test/resume.nut.ts create mode 100644 test/commands/agent/test/resume.test.ts diff --git a/messages/agent.generate.test.md b/messages/agent.generate.test.md new file mode 100644 index 0000000..fb78f84 --- /dev/null +++ b/messages/agent.generate.test.md @@ -0,0 +1,19 @@ +# summary + +Summary of a command. + +# description + +More information about a command. Don't repeat the summary. + +# flags.name.summary + +Description of a flag. + +# flags.name.description + +More information about a flag. Don't repeat the summary. + +# examples + +- <%= config.bin %> <%= command.id %> diff --git a/messages/agent.test.cancel.md b/messages/agent.test.cancel.md index 5d6f229..d67f6ee 100644 --- a/messages/agent.test.cancel.md +++ b/messages/agent.test.cancel.md @@ -6,7 +6,7 @@ Cancel a running test for an Agent. Cancel a running test for an Agent, providing the AiEvaluation ID. -# flags.id.summary +# flags.job-id.summary The AiEvaluation ID. diff --git a/messages/agent.test.results.md b/messages/agent.test.results.md new file mode 100644 index 0000000..fb78f84 --- /dev/null +++ b/messages/agent.test.results.md @@ -0,0 +1,19 @@ +# summary + +Summary of a command. + +# description + +More information about a command. Don't repeat the summary. + +# flags.name.summary + +Description of a flag. + +# flags.name.description + +More information about a flag. Don't repeat the summary. + +# examples + +- <%= config.bin %> <%= command.id %> diff --git a/messages/agent.test.resume.md b/messages/agent.test.resume.md new file mode 100644 index 0000000..fb78f84 --- /dev/null +++ b/messages/agent.test.resume.md @@ -0,0 +1,19 @@ +# summary + +Summary of a command. + +# description + +More information about a command. Don't repeat the summary. + +# flags.name.summary + +Description of a flag. + +# flags.name.description + +More information about a flag. Don't repeat the summary. + +# examples + +- <%= config.bin %> <%= command.id %> diff --git a/package.json b/package.json index 2824bb6..bc98ff7 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,9 @@ "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.7.12", "@salesforce/agents": "^0.1.4", - "@salesforce/core": "^8.5.2", + "@salesforce/core": "^8.8.0", "@salesforce/kit": "^3.2.1", - "@salesforce/sf-plugins-core": "^12", + "@salesforce/sf-plugins-core": "^12.1.0", "ansis": "^3.3.2" }, "devDependencies": { @@ -60,7 +60,16 @@ ], "topics": { "agent": { - "description": "Commands to work with agents." + "description": "Commands to work with agents.", + "external": true, + "subtopics": { + "test": { + "external": true + }, + "generate": { + "external": true + } + } } }, "flexibleTaxonomy": true diff --git a/schemas/agent-create.json b/schemas/agent-create.json index 9196a7f..eb28296 100644 --- a/schemas/agent-create.json +++ b/schemas/agent-create.json @@ -12,10 +12,8 @@ "type": "string" } }, - "required": [ - "isSuccess" - ], + "required": ["isSuccess"], "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/schemas/agent-generate-spec.json b/schemas/agent-generate-spec.json index 38d003f..51bb4c2 100644 --- a/schemas/agent-generate-spec.json +++ b/schemas/agent-generate-spec.json @@ -15,10 +15,8 @@ "type": "string" } }, - "required": [ - "isSuccess" - ], + "required": ["isSuccess"], "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/schemas/agent-test-cancel.json b/schemas/agent-test-cancel.json index 283df86..81f5cbf 100644 --- a/schemas/agent-test-cancel.json +++ b/schemas/agent-test-cancel.json @@ -18,11 +18,8 @@ "type": "string" } }, - "required": [ - "jobId", - "success" - ], + "required": ["jobId", "success"], "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/schemas/agent-test-run.json b/schemas/agent-test-run.json index 1fc7379..5ac6a22 100644 --- a/schemas/agent-test-run.json +++ b/schemas/agent-test-run.json @@ -18,11 +18,8 @@ "type": "string" } }, - "required": [ - "jobId", - "success" - ], + "required": ["jobId", "success"], "additionalProperties": false } } -} \ No newline at end of file +} diff --git a/src/commands/agent/generate/test.ts b/src/commands/agent/generate/test.ts new file mode 100644 index 0000000..956f4db --- /dev/null +++ b/src/commands/agent/generate/test.ts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.test'); + +export type AgentGenerateTestResult = { + path: string; +}; + +export default class AgentGenerateTest extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly flags = { + name: Flags.string({ + summary: messages.getMessage('flags.name.summary'), + description: messages.getMessage('flags.name.description'), + char: 'n', + required: false, + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentGenerateTest); + + const name = flags.name ?? 'world'; + this.log(`hello ${name} from src/commands/agent/generate/test.ts`); + return { + path: 'src/commands/agent/generate/test.ts', + }; + } +} diff --git a/src/commands/agent/test/cancel.ts b/src/commands/agent/test/cancel.ts index a4e1f2c..6f70fee 100644 --- a/src/commands/agent/test/cancel.ts +++ b/src/commands/agent/test/cancel.ts @@ -28,7 +28,7 @@ export default class AgentTestCancel extends SfCommand { 'job-id': Flags.string({ char: 'i', required: true, - summary: messages.getMessage('flags.id.summary'), + summary: messages.getMessage('flags.job-id.summary'), }), 'use-most-recent': Flags.boolean({ char: 'r', diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts new file mode 100644 index 0000000..5a94836 --- /dev/null +++ b/src/commands/agent/test/results.ts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.results'); + +export type AgentTestResultsResult = { + path: string; +}; + +export default class AgentTestResults extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly flags = { + name: Flags.string({ + summary: messages.getMessage('flags.name.summary'), + description: messages.getMessage('flags.name.description'), + char: 'n', + required: false, + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTestResults); + + const name = flags.name ?? 'world'; + this.log(`hello ${name} from src/commands/agent/test/results.ts`); + return { + path: 'src/commands/agent/test/results.ts', + }; + } +} diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts new file mode 100644 index 0000000..481ab13 --- /dev/null +++ b/src/commands/agent/test/resume.ts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); + +export type AgentTestResumeResult = { + path: string; +}; + +export default class AgentTestResume extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly flags = { + name: Flags.string({ + summary: messages.getMessage('flags.name.summary'), + description: messages.getMessage('flags.name.description'), + char: 'n', + required: false, + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTestResume); + + const name = flags.name ?? 'world'; + this.log(`hello ${name} from src/commands/agent/test/resume.ts`); + return { + path: 'src/commands/agent/test/resume.ts', + }; + } +} diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 45b4269..9a7f1af 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -32,7 +32,8 @@ export default class AgentTestRun extends SfCommand { public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), - // AiEvalDefinitionVersion.Id -- This should really be "test-name" + // This should probably be "test-name" + // Is it `AiEvaluationDefinition_My_first_test_v1`? or `"My first test v1"`? or `My_first_test_v1`? id: Flags.string({ char: 'i', required: true, @@ -62,6 +63,7 @@ export default class AgentTestRun extends SfCommand { // Future flags: // suites [array of suite names] // verbose [boolean] + // fail-fast [boolean] }; public async run(): Promise { @@ -104,15 +106,18 @@ export default class AgentTestRun extends SfCommand { if (isTimeoutError(e)) { mso.stop('async'); this.log(`Client timed out after ${flags.wait.minutes} minutes.`); - this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`); + this.log( + `Run ${colorize('dim', `sf agent test resume --id ${response.id}`)} to resuming watching this test.` + ); } else { mso.error(); throw e; } } } else { + // TODO: cache jobId in TTL cache so we can use it for resume mso.stop(); - this.log(`Run ${colorize('dim', `sf agent test result --id ${response.id}`)} to check status and results.`); + this.log(`Run ${colorize('dim', `sf agent test resume --id ${response.id}`)} to resuming watching this test.`); } mso.stop(); diff --git a/test/commands/agent/generate/test.nut.ts b/test/commands/agent/generate/test.nut.ts new file mode 100644 index 0000000..98e4735 --- /dev/null +++ b/test/commands/agent/generate/test.nut.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; + +describe('agent generate test NUTs', () => { + let session: TestSession; + + before(async () => { + session = await TestSession.create({ devhubAuthStrategy: 'NONE' }); + }); + + after(async () => { + await session?.clean(); + }); + + it('should display provided name', () => { + const name = 'World'; + const command = `agent generate test --name ${name}`; + const output = execCmd(command, { ensureExitCode: 0 }).shellOutput.stdout; + expect(output).to.contain(name); + }); +}); diff --git a/test/commands/agent/generate/test.test.ts b/test/commands/agent/generate/test.test.ts new file mode 100644 index 0000000..934f8c9 --- /dev/null +++ b/test/commands/agent/generate/test.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { TestContext } from '@salesforce/core/testSetup'; +import { expect } from 'chai'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import AgentGenerateTest from '../../../../src/commands/agent/generate/test.js'; + +describe('agent generate test', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + + beforeEach(() => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + }); + + afterEach(() => { + $$.restore(); + }); + + it('runs hello', async () => { + await AgentGenerateTest.run([]); + const output = sfCommandStubs.log + .getCalls() + .flatMap((c) => c.args) + .join('\n'); + expect(output).to.include('hello world'); + }); + + it('runs hello with --json and no provided name', async () => { + const result = await AgentGenerateTest.run([]); + expect(result.path).to.equal('src/commands/agent/generate/test.ts'); + }); + + it('runs hello world --name Astro', async () => { + await AgentGenerateTest.run(['--name', 'Astro']); + const output = sfCommandStubs.log + .getCalls() + .flatMap((c) => c.args) + .join('\n'); + expect(output).to.include('hello Astro'); + }); +}); diff --git a/test/commands/agent/test/results.nut.ts b/test/commands/agent/test/results.nut.ts new file mode 100644 index 0000000..c2d0a8e --- /dev/null +++ b/test/commands/agent/test/results.nut.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; + +describe('agent test results NUTs', () => { + let session: TestSession; + + before(async () => { + session = await TestSession.create({ devhubAuthStrategy: 'NONE' }); + }); + + after(async () => { + await session?.clean(); + }); + + it('should display provided name', () => { + const name = 'World'; + const command = `agent test results --name ${name}`; + const output = execCmd(command, { ensureExitCode: 0 }).shellOutput.stdout; + expect(output).to.contain(name); + }); +}); diff --git a/test/commands/agent/test/results.test.ts b/test/commands/agent/test/results.test.ts new file mode 100644 index 0000000..eb80975 --- /dev/null +++ b/test/commands/agent/test/results.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { TestContext } from '@salesforce/core/testSetup'; +import { expect } from 'chai'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import AgentTestResults from '../../../../src/commands/agent/test/results.js'; + +describe('agent test results', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + + beforeEach(() => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + }); + + afterEach(() => { + $$.restore(); + }); + + it('runs hello', async () => { + await AgentTestResults.run([]); + const output = sfCommandStubs.log + .getCalls() + .flatMap((c) => c.args) + .join('\n'); + expect(output).to.include('hello world'); + }); + + it('runs hello with --json and no provided name', async () => { + const result = await AgentTestResults.run([]); + expect(result.path).to.equal('src/commands/agent/test/results.ts'); + }); + + it('runs hello world --name Astro', async () => { + await AgentTestResults.run(['--name', 'Astro']); + const output = sfCommandStubs.log + .getCalls() + .flatMap((c) => c.args) + .join('\n'); + expect(output).to.include('hello Astro'); + }); +}); diff --git a/test/commands/agent/test/resume.nut.ts b/test/commands/agent/test/resume.nut.ts new file mode 100644 index 0000000..ba8a9bb --- /dev/null +++ b/test/commands/agent/test/resume.nut.ts @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; + +describe('agent test resume NUTs', () => { + let session: TestSession; + + before(async () => { + session = await TestSession.create({ devhubAuthStrategy: 'NONE' }); + }); + + after(async () => { + await session?.clean(); + }); + + it('should display provided name', () => { + const name = 'World'; + const command = `agent test resume --name ${name}`; + const output = execCmd(command, { ensureExitCode: 0 }).shellOutput.stdout; + expect(output).to.contain(name); + }); +}); diff --git a/test/commands/agent/test/resume.test.ts b/test/commands/agent/test/resume.test.ts new file mode 100644 index 0000000..3c2e9b0 --- /dev/null +++ b/test/commands/agent/test/resume.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { TestContext } from '@salesforce/core/testSetup'; +import { expect } from 'chai'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import AgentTestResume from '../../../../src/commands/agent/test/resume.js'; + +describe('agent test resume', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + + beforeEach(() => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + }); + + afterEach(() => { + $$.restore(); + }); + + it('runs hello', async () => { + await AgentTestResume.run([]); + const output = sfCommandStubs.log + .getCalls() + .flatMap((c) => c.args) + .join('\n'); + expect(output).to.include('hello world'); + }); + + it('runs hello with --json and no provided name', async () => { + const result = await AgentTestResume.run([]); + expect(result.path).to.equal('src/commands/agent/test/resume.ts'); + }); + + it('runs hello world --name Astro', async () => { + await AgentTestResume.run(['--name', 'Astro']); + const output = sfCommandStubs.log + .getCalls() + .flatMap((c) => c.args) + .join('\n'); + expect(output).to.include('hello Astro'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 0588b29..43fd884 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1095,12 +1095,7 @@ wrap-ansi "^6.2.0" yoctocolors-cjs "^2.1.2" -"@inquirer/figures@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@inquirer/figures/-/figures-1.0.5.tgz#57f9a996d64d3e3345d2a3ca04d36912e94f8790" - integrity sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA== - -"@inquirer/figures@^1.0.7": +"@inquirer/figures@^1.0.5", "@inquirer/figures@^1.0.7": version "1.0.7" resolved "https://registry.yarnpkg.com/@inquirer/figures/-/figures-1.0.7.tgz#d050ccc0eabfacc0248c4ff647a9dfba1b01594b" integrity sha512-m+Trk77mp54Zma6xLkLuY+mvanPxlE4A7yNKs2HBiyZ4UkVs28Mv5c/pgWrHeInx+USHeX/WEPzjrWrcJiQgjw== @@ -1269,10 +1264,10 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@oclif/core@^4", "@oclif/core@^4.0.27", "@oclif/core@^4.0.29": - version "4.0.30" - resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.0.30.tgz#65672282756bf19fde3830cab6d8031bf6236064" - integrity sha512-Ak3OUdOcoovIRWZOT6oC5JhZgyJD90uWX/7HjSofn+C4LEmHxxfiyu04a73dwnezfzqDu9jEXfd2mQOOC54KZw== +"@oclif/core@^4", "@oclif/core@^4.0.27", "@oclif/core@^4.0.29", "@oclif/core@^4.0.32": + version "4.0.33" + resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.0.33.tgz#fcaf3dd2850c5999de20459a1445d31a230cd24b" + integrity sha512-NoTDwJ2L/ywpsSjcN7jAAHf3m70Px4Yim2SJrm16r70XpnfbNOdlj1x0HEJ0t95gfD+p/y5uy+qPT/VXTh/1gw== dependencies: ansi-escapes "^4.3.2" ansis "^3.3.2" @@ -1283,7 +1278,7 @@ get-package-type "^0.1.0" globby "^11.1.0" indent-string "^4.0.0" - is-wsl "^3" + is-wsl "^2.2.0" lilconfig "^3.1.2" minimatch "^9.0.5" semver "^7.6.3" @@ -1402,10 +1397,10 @@ strip-ansi "6.0.1" ts-retry-promise "^0.8.1" -"@salesforce/core@^8.5.1", "@salesforce/core@^8.5.2", "@salesforce/core@^8.5.7", "@salesforce/core@^8.6.2", "@salesforce/core@^8.6.3": - version "8.6.3" - resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-8.6.3.tgz#1a4d50eaa2b731c1e480986cef96b414ccafd347" - integrity sha512-fxY3J9AttGztTY45AYH4QP1cKB3OD1fJMDd1j/ALGCI6EMb2iMPp52awKVKHxrd/eTbZhn1OV5Jr0r6nJx5Hhw== +"@salesforce/core@^8.5.1", "@salesforce/core@^8.5.2", "@salesforce/core@^8.5.7", "@salesforce/core@^8.6.2", "@salesforce/core@^8.6.3", "@salesforce/core@^8.8.0": + version "8.8.0" + resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-8.8.0.tgz#849c07ea3a2548ca201fc0fe8baef9b36a462194" + integrity sha512-HWGdRiy/MPCJ2KHz+W+cnqx0O9xhx9+QYvwP8bn9PE27wj0A/NjTi4xrqIWk1M+fE4dXHycE+8qPf4b540euvg== dependencies: "@jsforce/jsforce-node" "^3.6.1" "@salesforce/kit" "^3.2.2" @@ -1512,14 +1507,14 @@ string-width "^7.2.0" terminal-link "^3.0.0" -"@salesforce/sf-plugins-core@^12": - version "12.0.11" - resolved "https://registry.yarnpkg.com/@salesforce/sf-plugins-core/-/sf-plugins-core-12.0.11.tgz#5837bc385cb8f057c4bc86b71ead71464ba5063b" - integrity sha512-DYb54IeszQxcyl0N3e5qxSx3Vc571f36alZNE54qPqBTi9RAGEHQN4XR03dKLic0aNS/j4Z09RGH6YoH2zSL6A== +"@salesforce/sf-plugins-core@^12.1.0": + version "12.1.0" + resolved "https://registry.yarnpkg.com/@salesforce/sf-plugins-core/-/sf-plugins-core-12.1.0.tgz#874531acb39755a634ceda5de6462c3b6256baf6" + integrity sha512-xJXF0WE+4lq2kb/w24wcZc+76EUCIKv7dj1oATugk9JFzYKySdC1smzCY/BhPGzMQGvXcbkWo5PG5iXDBrtwYQ== dependencies: "@inquirer/confirm" "^3.1.22" "@inquirer/password" "^2.2.0" - "@oclif/core" "^4.0.27" + "@oclif/core" "^4.0.32" "@oclif/table" "^0.3.2" "@salesforce/core" "^8.5.1" "@salesforce/kit" "^3.2.3" @@ -4665,10 +4660,10 @@ is-date-object@^1.0.1: dependencies: has-tostringtag "^1.0.0" -is-docker@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-3.0.0.tgz#90093aa3106277d8a77a5910dbae71747e15a200" - integrity sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ== +is-docker@^2.0.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" + integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== is-extglob@^2.1.1: version "2.1.1" @@ -4704,13 +4699,6 @@ is-in-ci@^0.1.0: resolved "https://registry.yarnpkg.com/is-in-ci/-/is-in-ci-0.1.0.tgz#5e07d6a02ec3a8292d3f590973357efa3fceb0d3" integrity sha512-d9PXLEY0v1iJ64xLiQMJ51J128EYHAaOR4yZqQi8aHGfw6KgifM3/Viw1oZZ1GCVmb3gBuyhLyHj0HgR2DhSXQ== -is-inside-container@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-inside-container/-/is-inside-container-1.0.0.tgz#e81fba699662eb31dbdaf26766a61d4814717ea4" - integrity sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA== - dependencies: - is-docker "^3.0.0" - is-negative-zero@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150" @@ -4833,12 +4821,12 @@ is-windows@^1.0.2: resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== -is-wsl@^3: - version "3.1.0" - resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-3.1.0.tgz#e1c657e39c10090afcbedec61720f6b924c3cbd2" - integrity sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw== +is-wsl@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271" + integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== dependencies: - is-inside-container "^1.0.0" + is-docker "^2.0.0" isarray@0.0.1: version "0.0.1" From d3778a36a2f497e3dcf6dc6dd6ee7fbf9d020c95 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Tue, 26 Nov 2024 10:43:27 -0700 Subject: [PATCH 03/12] feature: steel thread basically works --- messages/agent.test.results.md | 4 +- messages/agent.test.resume.md | 28 ++++++--- messages/agent.test.run.md | 4 -- messages/shared.md | 3 + src/agentTestCache.ts | 63 ++++++++++++++++++++ src/commands/agent/test/cancel.ts | 25 ++++---- src/commands/agent/test/results.ts | 30 +++++----- src/commands/agent/test/resume.ts | 93 ++++++++++++++++++++++++++---- src/commands/agent/test/run.ts | 32 +++++----- src/flags.ts | 23 ++++++++ 10 files changed, 240 insertions(+), 65 deletions(-) create mode 100644 messages/shared.md create mode 100644 src/agentTestCache.ts create mode 100644 src/flags.ts diff --git a/messages/agent.test.results.md b/messages/agent.test.results.md index fb78f84..d93bb90 100644 --- a/messages/agent.test.results.md +++ b/messages/agent.test.results.md @@ -6,11 +6,11 @@ Summary of a command. More information about a command. Don't repeat the summary. -# flags.name.summary +# flags.job-id.summary Description of a flag. -# flags.name.description +# flags.job-id.description More information about a flag. Don't repeat the summary. diff --git a/messages/agent.test.resume.md b/messages/agent.test.resume.md index fb78f84..2a2c78a 100644 --- a/messages/agent.test.resume.md +++ b/messages/agent.test.resume.md @@ -1,19 +1,33 @@ # summary -Summary of a command. +Resume a running test for an Agent. # description -More information about a command. Don't repeat the summary. +Resume a running test for an Agent, providing the AiEvaluation ID. -# flags.name.summary +# flags.job-id.summary -Description of a flag. +The AiEvaluation ID. -# flags.name.description +# flags.use-most-recent.summary -More information about a flag. Don't repeat the summary. +Use the job ID of the most recent test evaluation. + +# flags.wait.summary + +Number of minutes to wait for the command to complete and display results to the terminal window. + +# flags.wait.description + +If the command continues to run after the wait period, the CLI returns control of the terminal window to you. + +# flags.output-dir.summary + +Directory in which to store test run files. # examples -- <%= config.bin %> <%= command.id %> +- Resume a test for an Agent: + + <%= config.bin %> <%= command.id %> --id AiEvalId diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 0d5a90a..4b73b25 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -26,10 +26,6 @@ If the command continues to run after the wait period, the CLI returns control o Directory in which to store test run files. -# flags.result-format.summary - -Format of the test run results. - # examples - Start a test for an Agent: diff --git a/messages/shared.md b/messages/shared.md new file mode 100644 index 0000000..af9268c --- /dev/null +++ b/messages/shared.md @@ -0,0 +1,3 @@ +# flags.result-format.summary + +Format of the test run results. diff --git a/src/agentTestCache.ts b/src/agentTestCache.ts new file mode 100644 index 0000000..d0fccc4 --- /dev/null +++ b/src/agentTestCache.ts @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { Global, SfError, TTLConfig } from '@salesforce/core'; +import { Duration } from '@salesforce/kit'; + +export class AgentTestCache extends TTLConfig { + public static getFileName(): string { + return 'agent-test-cache.json'; + } + + public static getDefaultOptions(): TTLConfig.Options { + return { + isGlobal: true, + isState: true, + filename: AgentTestCache.getFileName(), + stateFolder: Global.SF_STATE_FOLDER, + ttl: Duration.days(7), + }; + } + + public async createCacheEntry(jobId: string): Promise { + if (!jobId) throw new SfError('Job ID is required to create a cache entry'); + + this.set(jobId, { jobId }); + await this.write(); + } + + public async removeCacheEntry(jobId: string): Promise { + if (!jobId) throw new SfError('Job ID is required to remove a cache entry'); + + this.unset(jobId); + await this.write(); + } + + public resolveFromCache(): { jobId: string } { + const key = this.getLatestKey(); + if (!key) throw new SfError('Could not find a job ID to resume'); + + const { jobId } = this.get(key); + return { jobId }; + } + + public useIdOrMostRecent(id: string | undefined, useMostRecent: boolean): string { + if (id && useMostRecent) { + throw new SfError('Cannot specify both a job ID and use most recent flag'); + } + + if (!id && !useMostRecent) { + throw new SfError('Must specify either a job ID or use most recent flag'); + } + + if (id) { + return id; + } + + return this.resolveFromCache().jobId; + } +} diff --git a/src/commands/agent/test/cancel.ts b/src/commands/agent/test/cancel.ts index 6f70fee..360d465 100644 --- a/src/commands/agent/test/cancel.ts +++ b/src/commands/agent/test/cancel.ts @@ -7,6 +7,8 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; +import { AgentTester } from '@salesforce/agents'; +import { AgentTestCache } from '../../../agentTestCache.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.cancel'); @@ -25,34 +27,37 @@ export default class AgentTestCancel extends SfCommand { public static readonly flags = { 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), 'job-id': Flags.string({ char: 'i', - required: true, summary: messages.getMessage('flags.job-id.summary'), + exactlyOne: ['use-most-recent', 'job-id'], }), 'use-most-recent': Flags.boolean({ char: 'r', summary: messages.getMessage('flags.use-most-recent.summary'), exactlyOne: ['use-most-recent', 'job-id'], }), - // - // Future flags: - // ??? api-version ??? }; public async run(): Promise { const { flags } = await this.parse(AgentTestCancel); - this.log(`Canceling tests for AiEvaluation Job: ${flags['job-id']}`); + const agentTestCache = await AgentTestCache.create(); + const id = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + + this.log(`Canceling tests for AiEvaluation Job: ${id}`); - // Call SF Eval Connect API passing AiEvaluation.Id - // POST to /einstein/ai-evaluations/{aiEvaluationId}/stop + const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + const result = await agentTester.cancel(id); - // Returns: AiEvaluation.Id + if (result.success) { + await agentTestCache.removeCacheEntry(id); + } return { - success: true, - jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id + success: result.success, + jobId: id, }; } } diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 5a94836..f21babb 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, salesforce.com, inc. + * Copyright (c) 2024, salesforce.com, inc. * All rights reserved. * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause @@ -7,13 +7,13 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; +import { AgentTester } from '@salesforce/agents'; +import { resultFormatFlag } from '../../../flags.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.results'); -export type AgentTestResultsResult = { - path: string; -}; +export type AgentTestResultsResult = Awaited>['response']; export default class AgentTestResults extends SfCommand { public static readonly summary = messages.getMessage('summary'); @@ -21,21 +21,23 @@ export default class AgentTestResults extends SfCommand public static readonly examples = messages.getMessages('examples'); public static readonly flags = { - name: Flags.string({ - summary: messages.getMessage('flags.name.summary'), - description: messages.getMessage('flags.name.description'), - char: 'n', - required: false, + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + 'job-id': Flags.string({ + summary: messages.getMessage('flags.job-id.summary'), + description: messages.getMessage('flags.job-id.description'), + char: 'i', + required: true, }), + 'result-format': resultFormatFlag(), }; public async run(): Promise { const { flags } = await this.parse(AgentTestResults); - const name = flags.name ?? 'world'; - this.log(`hello ${name} from src/commands/agent/test/results.ts`); - return { - path: 'src/commands/agent/test/results.ts', - }; + const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + const { response, formatted } = await agentTester.details(flags['job-id'], flags['result-format']); + this.log(formatted); + return response; } } diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 481ab13..cb61611 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -6,36 +6,109 @@ */ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { Lifecycle, Messages } from '@salesforce/core'; +import { MultiStageOutput } from '@oclif/multi-stage-output'; +import { colorize } from '@oclif/core/ux'; +import { AgentTester } from '@salesforce/agents'; +import { AgentTestCache } from '../../../agentTestCache.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); export type AgentTestResumeResult = { - path: string; + jobId: string; }; +const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => + (e as { name: string })?.name === 'PollingClientTimeout'; + export default class AgentTestResume extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); public static readonly flags = { - name: Flags.string({ - summary: messages.getMessage('flags.name.summary'), - description: messages.getMessage('flags.name.description'), - char: 'n', - required: false, + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + 'job-id': Flags.string({ + char: 'i', + summary: messages.getMessage('flags.job-id.summary'), + exactlyOne: ['use-most-recent', 'job-id'], + }), + 'use-most-recent': Flags.boolean({ + char: 'r', + summary: messages.getMessage('flags.use-most-recent.summary'), + exactlyOne: ['use-most-recent', 'job-id'], + }), + // we want to pass `undefined` to the API + // eslint-disable-next-line sf-plugin/flag-min-max-default + wait: Flags.duration({ + char: 'w', + unit: 'minutes', + min: 1, + summary: messages.getMessage('flags.wait.summary'), + description: messages.getMessage('flags.wait.description'), }), }; public async run(): Promise { const { flags } = await this.parse(AgentTestResume); - const name = flags.name ?? 'world'; - this.log(`hello ${name} from src/commands/agent/test/resume.ts`); + const agentTestCache = await AgentTestCache.create(); + const id = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + + const mso = new MultiStageOutput<{ id: string; status: string }>({ + jsonEnabled: this.jsonEnabled(), + title: `Agent Test Run: ${id}`, + stages: ['Starting Tests', 'Polling for Test Results'], + stageSpecificBlock: [ + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Status', + get: (data) => data?.status, + }, + ], + postStagesBlock: [ + { + type: 'dynamic-key-value', + label: 'Job ID', + get: (data) => data?.id, + }, + ], + }); + const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); + mso.skipTo('Starting Tests', { id }); + + if (flags.wait?.minutes) { + mso.skipTo('Polling for Test Results'); + const lifecycle = Lifecycle.getInstance(); + lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => + Promise.resolve(mso.updateData({ status: event?.status })) + ); + try { + const { formatted } = await agentTester.poll(id, { timeout: flags.wait }); + mso.stop(); + this.log(formatted); + await agentTestCache.removeCacheEntry(id); + } catch (e) { + if (isTimeoutError(e)) { + mso.stop('async'); + this.log(`Client timed out after ${flags.wait.minutes} minutes.`); + this.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); + } else { + mso.error(); + throw e; + } + } + } else { + mso.stop(); + this.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); + } + + mso.stop(); return { - path: 'src/commands/agent/test/resume.ts', + jobId: id, // AiEvaluation.Id; needed for getting status and stopping }; } } diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 9a7f1af..5b7e86b 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -10,6 +10,8 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Lifecycle, Messages } from '@salesforce/core'; import { AgentTester } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; +import { resultFormatFlag } from '../../../flags.js'; +import { AgentTestCache } from '../../../agentTestCache.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); @@ -17,11 +19,9 @@ const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.r const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => (e as { name: string })?.name === 'PollingClientTimeout'; +// TODO: this should include details and status export type AgentTestRunResult = { jobId: string; // AiEvaluation.Id - success: boolean; - errorCode?: string; - message?: string; }; export default class AgentTestRun extends SfCommand { @@ -53,17 +53,7 @@ export default class AgentTestRun extends SfCommand { char: 'd', summary: messages.getMessage('flags.output-dir.summary'), }), - 'result-format': Flags.option({ - options: ['json', 'human', 'tap', 'junit'], - default: 'human', - char: 'r', - summary: messages.getMessage('flags.result-format.summary'), - })(), - // - // Future flags: - // suites [array of suite names] - // verbose [boolean] - // fail-fast [boolean] + 'result-format': resultFormatFlag(), }; public async run(): Promise { @@ -91,7 +81,12 @@ export default class AgentTestRun extends SfCommand { mso.skipTo('Starting Tests'); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); const response = await agentTester.start(flags.id); + mso.updateData({ id: response.id }); + + const ttlConfig = await AgentTestCache.create(); + await ttlConfig.createCacheEntry(response.id); + if (flags.wait?.minutes) { mso.skipTo('Polling for Test Results'); const lifecycle = Lifecycle.getInstance(); @@ -102,12 +97,13 @@ export default class AgentTestRun extends SfCommand { const { formatted } = await agentTester.poll(response.id, { timeout: flags.wait }); mso.stop(); this.log(formatted); + await ttlConfig.removeCacheEntry(response.id); } catch (e) { if (isTimeoutError(e)) { mso.stop('async'); this.log(`Client timed out after ${flags.wait.minutes} minutes.`); this.log( - `Run ${colorize('dim', `sf agent test resume --id ${response.id}`)} to resuming watching this test.` + `Run ${colorize('dim', `sf agent test resume --job-id ${response.id}`)} to resuming watching this test.` ); } else { mso.error(); @@ -115,14 +111,14 @@ export default class AgentTestRun extends SfCommand { } } } else { - // TODO: cache jobId in TTL cache so we can use it for resume mso.stop(); - this.log(`Run ${colorize('dim', `sf agent test resume --id ${response.id}`)} to resuming watching this test.`); + this.log( + `Run ${colorize('dim', `sf agent test resume --job-id ${response.id}`)} to resuming watching this test.` + ); } mso.stop(); return { - success: true, jobId: response.id, // AiEvaluation.Id; needed for getting status and stopping }; } diff --git a/src/flags.ts b/src/flags.ts new file mode 100644 index 0000000..a2c0c35 --- /dev/null +++ b/src/flags.ts @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'shared'); + +export const resultFormatFlag = Flags.option({ + options: [ + 'json', + 'human', + // 'tap', + // 'junit' + ] as const, + default: 'human', + char: 'r', + summary: messages.getMessage('flags.result-format.summary'), +}); From 1979c664d98a3a076578f2ce556d1fc549eecb0d Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Wed, 27 Nov 2024 13:44:10 -0700 Subject: [PATCH 04/12] chore: dry up mso code --- src/agentTestCache.ts | 31 +++++++---- src/commands/agent/test/cancel.ts | 10 ++-- src/commands/agent/test/resume.ts | 64 ++++----------------- src/commands/agent/test/run.ts | 62 ++++----------------- src/testStages.ts | 92 +++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 122 deletions(-) create mode 100644 src/testStages.ts diff --git a/src/agentTestCache.ts b/src/agentTestCache.ts index d0fccc4..a63f2db 100644 --- a/src/agentTestCache.ts +++ b/src/agentTestCache.ts @@ -8,7 +8,12 @@ import { Global, SfError, TTLConfig } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; -export class AgentTestCache extends TTLConfig { +type CacheContents = { + aiEvaluationId: string; + jobId: string; +}; + +export class AgentTestCache extends TTLConfig { public static getFileName(): string { return 'agent-test-cache.json'; } @@ -23,10 +28,10 @@ export class AgentTestCache extends TTLConfig { + public async createCacheEntry(jobId: string, aiEvaluationId: string): Promise { if (!jobId) throw new SfError('Job ID is required to create a cache entry'); - this.set(jobId, { jobId }); + this.set(jobId, { aiEvaluationId, jobId }); await this.write(); } @@ -37,27 +42,29 @@ export class AgentTestCache extends TTLConfig { const { flags } = await this.parse(AgentTestCancel); const agentTestCache = await AgentTestCache.create(); - const id = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + const { jobId } = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); - this.log(`Canceling tests for AiEvaluation Job: ${id}`); + this.log(`Canceling tests for AiEvaluation Job: ${jobId}`); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const result = await agentTester.cancel(id); + const result = await agentTester.cancel(jobId); if (result.success) { - await agentTestCache.removeCacheEntry(id); + await agentTestCache.removeCacheEntry(jobId); } return { success: result.success, - jobId: id, + jobId, }; } } diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index cb61611..2f12820 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -6,11 +6,10 @@ */ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Lifecycle, Messages } from '@salesforce/core'; -import { MultiStageOutput } from '@oclif/multi-stage-output'; -import { colorize } from '@oclif/core/ux'; +import { Messages } from '@salesforce/core'; import { AgentTester } from '@salesforce/agents'; import { AgentTestCache } from '../../../agentTestCache.js'; +import { TestStages } from '../../../testStages.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); @@ -19,9 +18,6 @@ export type AgentTestResumeResult = { jobId: string; }; -const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => - (e as { name: string })?.name === 'PollingClientTimeout'; - export default class AgentTestResume extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); @@ -40,12 +36,11 @@ export default class AgentTestResume extends SfCommand { summary: messages.getMessage('flags.use-most-recent.summary'), exactlyOne: ['use-most-recent', 'job-id'], }), - // we want to pass `undefined` to the API - // eslint-disable-next-line sf-plugin/flag-min-max-default wait: Flags.duration({ char: 'w', unit: 'minutes', min: 1, + defaultValue: 5, summary: messages.getMessage('flags.wait.summary'), description: messages.getMessage('flags.wait.description'), }), @@ -55,60 +50,21 @@ export default class AgentTestResume extends SfCommand { const { flags } = await this.parse(AgentTestResume); const agentTestCache = await AgentTestCache.create(); - const id = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + const { jobId, aiEvaluationId } = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); - const mso = new MultiStageOutput<{ id: string; status: string }>({ + const mso = new TestStages({ + title: `Agent Test Run: ${aiEvaluationId ?? jobId}`, jsonEnabled: this.jsonEnabled(), - title: `Agent Test Run: ${id}`, - stages: ['Starting Tests', 'Polling for Test Results'], - stageSpecificBlock: [ - { - stage: 'Polling for Test Results', - type: 'dynamic-key-value', - label: 'Status', - get: (data) => data?.status, - }, - ], - postStagesBlock: [ - { - type: 'dynamic-key-value', - label: 'Job ID', - get: (data) => data?.id, - }, - ], }); + mso.start({ id: jobId }); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - mso.skipTo('Starting Tests', { id }); - if (flags.wait?.minutes) { - mso.skipTo('Polling for Test Results'); - const lifecycle = Lifecycle.getInstance(); - lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => - Promise.resolve(mso.updateData({ status: event?.status })) - ); - try { - const { formatted } = await agentTester.poll(id, { timeout: flags.wait }); - mso.stop(); - this.log(formatted); - await agentTestCache.removeCacheEntry(id); - } catch (e) { - if (isTimeoutError(e)) { - mso.stop('async'); - this.log(`Client timed out after ${flags.wait.minutes} minutes.`); - this.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); - } else { - mso.error(); - throw e; - } - } - } else { - mso.stop(); - this.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); - } + const completed = await mso.poll(agentTester, jobId, flags.wait); + if (completed) await agentTestCache.removeCacheEntry(jobId); mso.stop(); return { - jobId: id, // AiEvaluation.Id; needed for getting status and stopping + jobId, }; } } diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 5b7e86b..ce63ee5 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -5,20 +5,17 @@ * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { MultiStageOutput } from '@oclif/multi-stage-output'; import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Lifecycle, Messages } from '@salesforce/core'; +import { Messages } from '@salesforce/core'; import { AgentTester } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; import { resultFormatFlag } from '../../../flags.js'; import { AgentTestCache } from '../../../agentTestCache.js'; +import { TestStages } from '../../../testStages.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); -const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => - (e as { name: string })?.name === 'PollingClientTimeout'; - // TODO: this should include details and status export type AgentTestRunResult = { jobId: string; // AiEvaluation.Id @@ -58,58 +55,21 @@ export default class AgentTestRun extends SfCommand { public async run(): Promise { const { flags } = await this.parse(AgentTestRun); - const mso = new MultiStageOutput<{ id: string; status: string }>({ - jsonEnabled: this.jsonEnabled(), - title: `Agent Test Run: ${flags.id}`, - stages: ['Starting Tests', 'Polling for Test Results'], - stageSpecificBlock: [ - { - stage: 'Polling for Test Results', - type: 'dynamic-key-value', - label: 'Status', - get: (data) => data?.status, - }, - ], - postStagesBlock: [ - { - type: 'dynamic-key-value', - label: 'Job ID', - get: (data) => data?.id, - }, - ], - }); - mso.skipTo('Starting Tests'); + + const mso = new TestStages({ title: `Agent Test Run: ${flags.id}`, jsonEnabled: this.jsonEnabled() }); + mso.start(); + const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); const response = await agentTester.start(flags.id); - mso.updateData({ id: response.id }); + mso.update({ id: response.id }); - const ttlConfig = await AgentTestCache.create(); - await ttlConfig.createCacheEntry(response.id); + const agentTestCache = await AgentTestCache.create(); + await agentTestCache.createCacheEntry(response.id, flags.id); if (flags.wait?.minutes) { - mso.skipTo('Polling for Test Results'); - const lifecycle = Lifecycle.getInstance(); - lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => - Promise.resolve(mso.updateData({ status: event?.status })) - ); - try { - const { formatted } = await agentTester.poll(response.id, { timeout: flags.wait }); - mso.stop(); - this.log(formatted); - await ttlConfig.removeCacheEntry(response.id); - } catch (e) { - if (isTimeoutError(e)) { - mso.stop('async'); - this.log(`Client timed out after ${flags.wait.minutes} minutes.`); - this.log( - `Run ${colorize('dim', `sf agent test resume --job-id ${response.id}`)} to resuming watching this test.` - ); - } else { - mso.error(); - throw e; - } - } + const completed = await mso.poll(agentTester, response.id, flags.wait); + if (completed) await agentTestCache.removeCacheEntry(response.id); } else { mso.stop(); this.log( diff --git a/src/testStages.ts b/src/testStages.ts new file mode 100644 index 0000000..abbb513 --- /dev/null +++ b/src/testStages.ts @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { colorize } from '@oclif/core/ux'; +import { MultiStageOutput } from '@oclif/multi-stage-output'; +import { AgentTester } from '@salesforce/agents'; +import { Lifecycle } from '@salesforce/core'; +import { Duration } from '@salesforce/kit'; +import { Ux } from '@salesforce/sf-plugins-core'; + +type Data = { id: string; status: string }; + +const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => + (e as { name: string })?.name === 'PollingClientTimeout'; + +export class TestStages { + private mso: MultiStageOutput; + private ux: Ux; + + public constructor({ title, jsonEnabled }: { title: string; jsonEnabled: boolean }) { + this.ux = new Ux({ jsonEnabled }); + this.mso = new MultiStageOutput({ + title, + jsonEnabled, + stages: ['Starting Tests', 'Polling for Test Results'], + stageSpecificBlock: [ + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Status', + get: (data): string | undefined => data?.status, + }, + ], + postStagesBlock: [ + { + type: 'dynamic-key-value', + label: 'Job ID', + get: (data): string | undefined => data?.id, + }, + ], + }); + } + + public start(data?: Partial): void { + this.mso.skipTo('Starting Tests', data); + } + + public async poll(agentTester: AgentTester, id: string, wait: Duration): Promise { + this.mso.skipTo('Polling for Test Results'); + const lifecycle = Lifecycle.getInstance(); + lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => + Promise.resolve(this.update({ status: event?.status })) + ); + + try { + const { formatted } = await agentTester.poll(id, { timeout: wait }); + this.stop(); + this.ux.log(formatted); + return true; + } catch (e) { + if (isTimeoutError(e)) { + this.stop('async'); + this.ux.log(`Client timed out after ${wait.minutes} minutes.`); + this.ux.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); + return true; + } else { + this.error(); + throw e; + } + } + } + + public update(data: Partial): void { + this.mso.updateData(data); + } + + public stop(finalStatus?: 'async'): void { + this.mso.stop(finalStatus); + } + + public error(): void { + this.mso.error(); + } + + public done(data?: Partial): void { + this.mso.skipTo('Done', data); + } +} From b9c530d54654abf02dcb59ea8f85f18aa81a177d Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 2 Dec 2024 10:37:12 -0700 Subject: [PATCH 05/12] refactor: comply with latest api spec --- messages/agent.test.run.md | 8 ++++---- src/agentTestCache.ts | 32 +++++++++++++++--------------- src/commands/agent/test/cancel.ts | 12 +++++------ src/commands/agent/test/results.ts | 1 + src/commands/agent/test/resume.ts | 14 ++++++------- src/commands/agent/test/run.ts | 29 ++++++++++++++------------- 6 files changed, 49 insertions(+), 47 deletions(-) diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 4b73b25..8100c6d 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -6,13 +6,13 @@ Start a test for an Agent. Start a test for an Agent, providing the AiEvalDefinitionVersion ID. Returns the job ID. -# flags.id.summary +# flags.name.summary -The AiEvalDefinitionVersion ID. +The name of the AiEvaluationDefinition to start. -# flags.id.description +# flags.name.description -The AiEvalDefinitionVersion ID. +The name of the AiEvaluationDefinition to start. # flags.wait.summary diff --git a/src/agentTestCache.ts b/src/agentTestCache.ts index a63f2db..bd06c85 100644 --- a/src/agentTestCache.ts +++ b/src/agentTestCache.ts @@ -10,7 +10,7 @@ import { Duration } from '@salesforce/kit'; type CacheContents = { aiEvaluationId: string; - jobId: string; + name: string; }; export class AgentTestCache extends TTLConfig { @@ -28,41 +28,41 @@ export class AgentTestCache extends TTLConfig }; } - public async createCacheEntry(jobId: string, aiEvaluationId: string): Promise { - if (!jobId) throw new SfError('Job ID is required to create a cache entry'); + public async createCacheEntry(aiEvaluationId: string, name: string): Promise { + if (!aiEvaluationId) throw new SfError('aiEvaluationId is required to create a cache entry'); - this.set(jobId, { aiEvaluationId, jobId }); + this.set(aiEvaluationId, { aiEvaluationId, name }); await this.write(); } - public async removeCacheEntry(jobId: string): Promise { - if (!jobId) throw new SfError('Job ID is required to remove a cache entry'); + public async removeCacheEntry(aiEvaluationId: string): Promise { + if (!aiEvaluationId) throw new SfError('aiEvaluationId is required to remove a cache entry'); - this.unset(jobId); + this.unset(aiEvaluationId); await this.write(); } public resolveFromCache(): CacheContents { const key = this.getLatestKey(); - if (!key) throw new SfError('Could not find a job ID to resume'); + if (!key) throw new SfError('Could not find an aiEvaluationId to resume'); return this.get(key); } public useIdOrMostRecent( - jobId: string | undefined, + aiEvaluationId: string | undefined, useMostRecent: boolean - ): { jobId: string; aiEvaluationId?: string } { - if (jobId && useMostRecent) { - throw new SfError('Cannot specify both a job ID and use most recent flag'); + ): { aiEvaluationId: string; name?: string } { + if (aiEvaluationId && useMostRecent) { + throw new SfError('Cannot specify both an aiEvaluationId and use most recent flag'); } - if (!jobId && !useMostRecent) { - throw new SfError('Must specify either a job ID or use most recent flag'); + if (!aiEvaluationId && !useMostRecent) { + throw new SfError('Must specify either an aiEvaluationId or use most recent flag'); } - if (jobId) { - return { jobId }; + if (aiEvaluationId) { + return { aiEvaluationId }; } return this.resolveFromCache(); diff --git a/src/commands/agent/test/cancel.ts b/src/commands/agent/test/cancel.ts index c16a337..55c14d8 100644 --- a/src/commands/agent/test/cancel.ts +++ b/src/commands/agent/test/cancel.ts @@ -14,7 +14,7 @@ Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.cancel'); export type AgentTestCancelResult = { - jobId: string; // AiEvaluation.Id + aiEvaluationId: string; // AiEvaluation.Id success: boolean; errorCode?: string; message?: string; @@ -44,20 +44,20 @@ export default class AgentTestCancel extends SfCommand { const { flags } = await this.parse(AgentTestCancel); const agentTestCache = await AgentTestCache.create(); - const { jobId } = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + const { aiEvaluationId } = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); - this.log(`Canceling tests for AiEvaluation Job: ${jobId}`); + this.log(`Canceling tests for AiEvaluation Job: ${aiEvaluationId}`); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const result = await agentTester.cancel(jobId); + const result = await agentTester.cancel(aiEvaluationId); if (result.success) { - await agentTestCache.removeCacheEntry(jobId); + await agentTestCache.removeCacheEntry(aiEvaluationId); } return { success: result.success, - jobId, + aiEvaluationId, }; } } diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index f21babb..39cea27 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -19,6 +19,7 @@ export default class AgentTestResults extends SfCommand public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly state = 'beta'; public static readonly flags = { 'target-org': Flags.requiredOrg(), diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 2f12820..a1c406e 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -15,7 +15,7 @@ Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); export type AgentTestResumeResult = { - jobId: string; + aiEvaluationId: string; }; export default class AgentTestResume extends SfCommand { @@ -50,21 +50,21 @@ export default class AgentTestResume extends SfCommand { const { flags } = await this.parse(AgentTestResume); const agentTestCache = await AgentTestCache.create(); - const { jobId, aiEvaluationId } = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + const { name, aiEvaluationId } = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); const mso = new TestStages({ - title: `Agent Test Run: ${aiEvaluationId ?? jobId}`, + title: `Agent Test Run: ${name ?? aiEvaluationId}`, jsonEnabled: this.jsonEnabled(), }); - mso.start({ id: jobId }); + mso.start({ id: aiEvaluationId }); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const completed = await mso.poll(agentTester, jobId, flags.wait); - if (completed) await agentTestCache.removeCacheEntry(jobId); + const completed = await mso.poll(agentTester, aiEvaluationId, flags.wait); + if (completed) await agentTestCache.removeCacheEntry(aiEvaluationId); mso.stop(); return { - jobId, + aiEvaluationId, }; } } diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index ce63ee5..1430c97 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -18,7 +18,7 @@ const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.r // TODO: this should include details and status export type AgentTestRunResult = { - jobId: string; // AiEvaluation.Id + aiEvaluationId: string; }; export default class AgentTestRun extends SfCommand { @@ -29,13 +29,11 @@ export default class AgentTestRun extends SfCommand { public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), - // This should probably be "test-name" - // Is it `AiEvaluationDefinition_My_first_test_v1`? or `"My first test v1"`? or `My_first_test_v1`? - id: Flags.string({ + name: Flags.string({ char: 'i', required: true, - summary: messages.getMessage('flags.id.summary'), - description: messages.getMessage('flags.id.description'), + summary: messages.getMessage('flags.name.summary'), + description: messages.getMessage('flags.name.description'), }), // we want to pass `undefined` to the API // eslint-disable-next-line sf-plugin/flag-min-max-default @@ -56,30 +54,33 @@ export default class AgentTestRun extends SfCommand { public async run(): Promise { const { flags } = await this.parse(AgentTestRun); - const mso = new TestStages({ title: `Agent Test Run: ${flags.id}`, jsonEnabled: this.jsonEnabled() }); + const mso = new TestStages({ title: `Agent Test Run: ${flags.name}`, jsonEnabled: this.jsonEnabled() }); mso.start(); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const response = await agentTester.start(flags.id); + const response = await agentTester.start(flags.name); - mso.update({ id: response.id }); + mso.update({ id: response.aiEvaluationId }); const agentTestCache = await AgentTestCache.create(); - await agentTestCache.createCacheEntry(response.id, flags.id); + await agentTestCache.createCacheEntry(response.aiEvaluationId, flags.name); if (flags.wait?.minutes) { - const completed = await mso.poll(agentTester, response.id, flags.wait); - if (completed) await agentTestCache.removeCacheEntry(response.id); + const completed = await mso.poll(agentTester, response.aiEvaluationId, flags.wait); + if (completed) await agentTestCache.removeCacheEntry(response.aiEvaluationId); } else { mso.stop(); this.log( - `Run ${colorize('dim', `sf agent test resume --job-id ${response.id}`)} to resuming watching this test.` + `Run ${colorize( + 'dim', + `sf agent test resume --job-id ${response.aiEvaluationId}` + )} to resuming watching this test.` ); } mso.stop(); return { - jobId: response.id, // AiEvaluation.Id; needed for getting status and stopping + aiEvaluationId: response.aiEvaluationId, // AiEvaluation.Id; needed for getting status and stopping }; } } From e422ca5436a9dc03fe42c8aef50486be6b8ddfd1 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 2 Dec 2024 11:17:57 -0700 Subject: [PATCH 06/12] feat: show passing and failing tests --- src/testStages.ts | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/testStages.ts b/src/testStages.ts index abbb513..4f2aa62 100644 --- a/src/testStages.ts +++ b/src/testStages.ts @@ -12,7 +12,13 @@ import { Lifecycle } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; import { Ux } from '@salesforce/sf-plugins-core'; -type Data = { id: string; status: string }; +type Data = { + id: string; + status: string; + totalTestCases: number; + passingTestCases: number; + failingTestCases: number; +}; const isTimeoutError = (e: unknown): e is { name: 'PollingClientTimeout' } => (e as { name: string })?.name === 'PollingClientTimeout'; @@ -34,6 +40,18 @@ export class TestStages { label: 'Status', get: (data): string | undefined => data?.status, }, + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Passing Tests', + get: (data): string | undefined => data?.passingTestCases?.toString(), + }, + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Failing Tests', + get: (data): string | undefined => data?.failingTestCases?.toString(), + }, ], postStagesBlock: [ { @@ -52,8 +70,15 @@ export class TestStages { public async poll(agentTester: AgentTester, id: string, wait: Duration): Promise { this.mso.skipTo('Polling for Test Results'); const lifecycle = Lifecycle.getInstance(); - lifecycle.on('AGENT_TEST_POLLING_EVENT', async (event: { status: string }) => - Promise.resolve(this.update({ status: event?.status })) + lifecycle.on( + 'AGENT_TEST_POLLING_EVENT', + async (event: { + status: string; + completedTestCases: number; + totalTestCases: number; + failingTestCases: number; + passingTestCases: number; + }) => Promise.resolve(this.update(event)) ); try { From 4da8b5f667a08f27b300e495be09204770acd7e5 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 2 Dec 2024 11:19:56 -0700 Subject: [PATCH 07/12] feat: show completion progress --- src/testStages.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/testStages.ts b/src/testStages.ts index 4f2aa62..274221d 100644 --- a/src/testStages.ts +++ b/src/testStages.ts @@ -40,6 +40,15 @@ export class TestStages { label: 'Status', get: (data): string | undefined => data?.status, }, + { + stage: 'Polling for Test Results', + type: 'dynamic-key-value', + label: 'Completed Tests', + get: (data): string | undefined => + data?.totalTestCases && data?.passingTestCases && data?.failingTestCases + ? `${data?.passingTestCases + data?.failingTestCases}/${data?.totalTestCases}` + : undefined, + }, { stage: 'Polling for Test Results', type: 'dynamic-key-value', From c10e5bf625b10e84a73e2808037927c18ea806ae Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 2 Dec 2024 12:47:01 -0700 Subject: [PATCH 08/12] chore: clean up --- src/commands/agent/generate/test.ts | 1 + src/commands/agent/test/cancel.ts | 3 ++- src/commands/agent/test/resume.ts | 1 + src/commands/agent/test/run.ts | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/commands/agent/generate/test.ts b/src/commands/agent/generate/test.ts index 956f4db..d7c54aa 100644 --- a/src/commands/agent/generate/test.ts +++ b/src/commands/agent/generate/test.ts @@ -19,6 +19,7 @@ export default class AgentGenerateTest extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly state = 'beta'; public static readonly flags = { 'target-org': Flags.requiredOrg(), diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index a1c406e..c1349b6 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -22,6 +22,7 @@ export default class AgentTestResume extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly state = 'beta'; public static readonly flags = { 'target-org': Flags.requiredOrg(), diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 1430c97..2b63e4b 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -25,6 +25,7 @@ export default class AgentTestRun extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly state = 'beta'; public static readonly flags = { 'target-org': Flags.requiredOrg(), From 477da6dc634eda1ff2ff581fdab89cfbbfada507 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Mon, 2 Dec 2024 13:56:56 -0700 Subject: [PATCH 09/12] test: initial tests --- command-snapshot.json | 30 +++- messages/agent.test.run.md | 6 +- package.json | 2 +- schemas/agent-generate-test.json | 16 ++ schemas/agent-test-cancel.json | 4 +- schemas/agent-test-results.json | 145 ++++++++++++++++++ schemas/agent-test-resume.json | 19 +++ schemas/agent-test-run.json | 12 +- src/commands/agent/test/results.ts | 4 +- src/commands/agent/test/resume.ts | 2 + src/commands/agent/test/run.ts | 17 +- test/agentTestCache.test.ts | 112 ++++++++++++++ test/commands/agent/generate/test.test.ts | 46 ------ test/commands/agent/test/cancel.nut.ts | 55 +++++++ test/commands/agent/test/results.nut.ts | 43 +++++- test/commands/agent/test/results.test.ts | 46 ------ test/commands/agent/test/resume.nut.ts | 42 ++++- test/commands/agent/test/resume.test.ts | 46 ------ test/commands/agent/test/run.nut.ts | 60 ++++++++ test/mocks/connect_agent-job-spec.json | 90 +++++++++++ test/mocks/einstein_ai-evaluations_runs.json | 4 + .../1.json | 4 + .../2.json | 4 + .../3.json | 4 + ...ations_runs_4KBSM000000003F4AQ_cancel.json | 3 + ...tions_runs_4KBSM000000003F4AQ_details.json | 82 ++++++++++ test/nut/agent-test-run.nut.ts | 37 ----- test/unit/agent-test-run.test.ts | 24 --- yarn.lock | 50 +++++- 29 files changed, 759 insertions(+), 250 deletions(-) create mode 100644 schemas/agent-generate-test.json create mode 100644 schemas/agent-test-results.json create mode 100644 schemas/agent-test-resume.json create mode 100644 test/agentTestCache.test.ts delete mode 100644 test/commands/agent/generate/test.test.ts create mode 100644 test/commands/agent/test/cancel.nut.ts delete mode 100644 test/commands/agent/test/results.test.ts delete mode 100644 test/commands/agent/test/resume.test.ts create mode 100644 test/commands/agent/test/run.nut.ts create mode 100644 test/mocks/connect_agent-job-spec.json create mode 100644 test/mocks/einstein_ai-evaluations_runs.json create mode 100644 test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json create mode 100644 test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json create mode 100644 test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json create mode 100644 test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_cancel.json create mode 100644 test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json delete mode 100644 test/nut/agent-test-run.nut.ts delete mode 100644 test/unit/agent-test-run.test.ts diff --git a/command-snapshot.json b/command-snapshot.json index 0dba124..b676e13 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -27,20 +27,44 @@ ], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:generate:test", + "flagAliases": [], + "flagChars": ["n"], + "flags": ["flags-dir", "json", "name"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:test:cancel", "flagAliases": [], "flagChars": ["i", "o", "r"], - "flags": ["flags-dir", "job-id", "json", "target-org", "use-most-recent"], + "flags": ["api-version", "flags-dir", "job-id", "json", "target-org", "use-most-recent"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:results", + "flagAliases": [], + "flagChars": ["i", "o", "r"], + "flags": ["api-version", "flags-dir", "job-id", "json", "result-format", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:resume", + "flagAliases": [], + "flagChars": ["i", "o", "r", "w"], + "flags": ["api-version", "flags-dir", "job-id", "json", "target-org", "use-most-recent", "wait"], "plugin": "@salesforce/plugin-agent" }, { "alias": [], "command": "agent:test:run", "flagAliases": [], - "flagChars": ["d", "i", "o", "r", "w"], - "flags": ["api-version", "flags-dir", "id", "json", "output-dir", "result-format", "target-org", "wait"], + "flagChars": ["n", "o", "r", "w"], + "flags": ["api-version", "flags-dir", "json", "name", "result-format", "target-org", "wait"], "plugin": "@salesforce/plugin-agent" } ] diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 8100c6d..d187b98 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -22,12 +22,8 @@ Number of minutes to wait for the command to complete and display results to the If the command continues to run after the wait period, the CLI returns control of the terminal window to you. -# flags.output-dir.summary - -Directory in which to store test run files. - # examples - Start a test for an Agent: - <%= config.bin %> <%= command.id %> --id AiEvalDefVerId + <%= config.bin %> <%= command.id %> --name AiEvalDefVerId diff --git a/package.json b/package.json index bc98ff7..e57134b 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "@inquirer/select": "^4.0.1", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.7.12", - "@salesforce/agents": "^0.1.4", + "@salesforce/agents": "^0.2.2", "@salesforce/core": "^8.8.0", "@salesforce/kit": "^3.2.1", "@salesforce/sf-plugins-core": "^12.1.0", diff --git a/schemas/agent-generate-test.json b/schemas/agent-generate-test.json new file mode 100644 index 0000000..a393999 --- /dev/null +++ b/schemas/agent-generate-test.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentGenerateTestResult", + "definitions": { + "AgentGenerateTestResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + } + }, + "required": ["path"], + "additionalProperties": false + } + } +} diff --git a/schemas/agent-test-cancel.json b/schemas/agent-test-cancel.json index 81f5cbf..bf11239 100644 --- a/schemas/agent-test-cancel.json +++ b/schemas/agent-test-cancel.json @@ -5,7 +5,7 @@ "AgentTestCancelResult": { "type": "object", "properties": { - "jobId": { + "aiEvaluationId": { "type": "string" }, "success": { @@ -18,7 +18,7 @@ "type": "string" } }, - "required": ["jobId", "success"], + "required": ["aiEvaluationId", "success"], "additionalProperties": false } } diff --git a/schemas/agent-test-results.json b/schemas/agent-test-results.json new file mode 100644 index 0000000..6914fb8 --- /dev/null +++ b/schemas/agent-test-results.json @@ -0,0 +1,145 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTestResultsResult", + "definitions": { + "AgentTestResultsResult": { + "$ref": "#/definitions/AgentTestDetailsResponse" + }, + "AgentTestDetailsResponse": { + "type": "object", + "properties": { + "status": { + "$ref": "#/definitions/TestStatus" + }, + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + }, + "errorMessage": { + "type": "string" + }, + "testCases": { + "type": "array", + "items": { + "$ref": "#/definitions/TestCaseResult" + } + } + }, + "required": ["status", "startTime", "testCases"], + "additionalProperties": false + }, + "TestStatus": { + "type": "string", + "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR"] + }, + "TestCaseResult": { + "type": "object", + "properties": { + "status": { + "$ref": "#/definitions/TestStatus" + }, + "number": { + "type": "string" + }, + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + }, + "generatedData": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "AGENT" + }, + "actionsSequence": { + "type": "array", + "items": { + "type": "string" + } + }, + "outcome": { + "type": "string", + "enum": ["Success", "Failure"] + }, + "topic": { + "type": "string" + }, + "inputTokensCount": { + "type": "string" + }, + "outputTokensCount": { + "type": "string" + } + }, + "required": ["type", "actionsSequence", "outcome", "topic", "inputTokensCount", "outputTokensCount"], + "additionalProperties": false + }, + "expectationResults": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "actualValue": { + "type": "string" + }, + "expectedValue": { + "type": "string" + }, + "score": { + "type": "number" + }, + "result": { + "type": "string", + "enum": ["Passed", "Failed"] + }, + "metricLabel": { + "type": "string", + "enum": ["Accuracy", "Precision"] + }, + "metricExplainability": { + "type": "string" + }, + "status": { + "$ref": "#/definitions/TestStatus" + }, + "startTime": { + "type": "string" + }, + "endTime": { + "type": "string" + }, + "errorCode": { + "type": "string" + }, + "errorMessage": { + "type": "string" + } + }, + "required": [ + "name", + "actualValue", + "expectedValue", + "score", + "result", + "metricLabel", + "metricExplainability", + "status", + "startTime" + ], + "additionalProperties": false + } + } + }, + "required": ["status", "number", "startTime", "generatedData", "expectationResults"], + "additionalProperties": false + } + } +} diff --git a/schemas/agent-test-resume.json b/schemas/agent-test-resume.json new file mode 100644 index 0000000..5f86d12 --- /dev/null +++ b/schemas/agent-test-resume.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTestResumeResult", + "definitions": { + "AgentTestResumeResult": { + "type": "object", + "properties": { + "aiEvaluationId": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "required": ["aiEvaluationId", "status"], + "additionalProperties": false + } + } +} diff --git a/schemas/agent-test-run.json b/schemas/agent-test-run.json index 5ac6a22..284c121 100644 --- a/schemas/agent-test-run.json +++ b/schemas/agent-test-run.json @@ -5,20 +5,14 @@ "AgentTestRunResult": { "type": "object", "properties": { - "jobId": { + "aiEvaluationId": { "type": "string" }, - "success": { - "type": "boolean" - }, - "errorCode": { - "type": "string" - }, - "message": { + "status": { "type": "string" } }, - "required": ["jobId", "success"], + "required": ["aiEvaluationId", "status"], "additionalProperties": false } } diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 39cea27..a009d83 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -7,13 +7,13 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester } from '@salesforce/agents'; +import { AgentTester, AgentTestDetailsResponse } from '@salesforce/agents'; import { resultFormatFlag } from '../../../flags.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.results'); -export type AgentTestResultsResult = Awaited>['response']; +export type AgentTestResultsResult = AgentTestDetailsResponse; export default class AgentTestResults extends SfCommand { public static readonly summary = messages.getMessage('summary'); diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index c1349b6..fc349c4 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -16,6 +16,7 @@ const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.r export type AgentTestResumeResult = { aiEvaluationId: string; + status: string; }; export default class AgentTestResume extends SfCommand { @@ -65,6 +66,7 @@ export default class AgentTestResume extends SfCommand { mso.stop(); return { + status: 'COMPLETED', aiEvaluationId, }; } diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 2b63e4b..c383b5b 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -19,6 +19,7 @@ const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.r // TODO: this should include details and status export type AgentTestRunResult = { aiEvaluationId: string; + status: string; }; export default class AgentTestRun extends SfCommand { @@ -31,7 +32,7 @@ export default class AgentTestRun extends SfCommand { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), name: Flags.string({ - char: 'i', + char: 'n', required: true, summary: messages.getMessage('flags.name.summary'), description: messages.getMessage('flags.name.description'), @@ -45,10 +46,6 @@ export default class AgentTestRun extends SfCommand { summary: messages.getMessage('flags.wait.summary'), description: messages.getMessage('flags.wait.description'), }), - 'output-dir': Flags.directory({ - char: 'd', - summary: messages.getMessage('flags.output-dir.summary'), - }), 'result-format': resultFormatFlag(), }; @@ -69,6 +66,11 @@ export default class AgentTestRun extends SfCommand { if (flags.wait?.minutes) { const completed = await mso.poll(agentTester, response.aiEvaluationId, flags.wait); if (completed) await agentTestCache.removeCacheEntry(response.aiEvaluationId); + mso.stop(); + return { + status: 'COMPLETED', + aiEvaluationId: response.aiEvaluationId, + }; } else { mso.stop(); this.log( @@ -79,9 +81,6 @@ export default class AgentTestRun extends SfCommand { ); } - mso.stop(); - return { - aiEvaluationId: response.aiEvaluationId, // AiEvaluation.Id; needed for getting status and stopping - }; + return response; } } diff --git a/test/agentTestCache.test.ts b/test/agentTestCache.test.ts new file mode 100644 index 0000000..11d758b --- /dev/null +++ b/test/agentTestCache.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { expect } from 'chai'; +import { SfError } from '@salesforce/core'; +import sinon from 'sinon'; +import { AgentTestCache } from '../src/agentTestCache.js'; + +describe('AgentTestCache', () => { + let cache: AgentTestCache; + + beforeEach(() => { + cache = new AgentTestCache(AgentTestCache.getDefaultOptions()); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('createCacheEntry', () => { + it('should create a cache entry', async () => { + const writeStub = sinon.stub(cache, 'write').resolves(); + await cache.createCacheEntry('123', 'testName'); + const entry = cache.get('123'); + expect(entry.aiEvaluationId).to.equal('123'); + expect(entry.name).to.equal('testName'); + expect(writeStub.calledOnce).to.be.true; + }); + + it('should throw an error if aiEvaluationId is not provided', async () => { + try { + await cache.createCacheEntry('', 'testName'); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).message).to.equal('aiEvaluationId is required to create a cache entry'); + } + }); + }); + + describe('removeCacheEntry', () => { + it('should remove a cache entry', async () => { + const writeStub = sinon.stub(cache, 'write').resolves(); + await cache.createCacheEntry('123', 'testName'); + await cache.removeCacheEntry('123'); + expect(cache.get('123')).to.be.undefined; + expect(writeStub.calledTwice).to.be.true; + }); + + it('should throw an error if aiEvaluationId is not provided', async () => { + try { + await cache.removeCacheEntry(''); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).message).to.equal('aiEvaluationId is required to remove a cache entry'); + } + }); + }); + + describe('resolveFromCache', () => { + it('should resolve the most recent cache entry', async () => { + sinon.stub(cache, 'getLatestKey').returns('123'); + await cache.createCacheEntry('123', 'testName'); + const result = cache.resolveFromCache(); + expect(result.aiEvaluationId).to.equal('123'); + expect(result.name).to.equal('testName'); + }); + + it('should throw an error if no cache entry is found', () => { + sinon.stub(cache, 'getLatestKey').returns(undefined); + try { + cache.resolveFromCache(); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).message).to.equal('Could not find an aiEvaluationId to resume'); + } + }); + }); + + describe('useIdOrMostRecent', () => { + it('should return the provided aiEvaluationId', () => { + const result = cache.useIdOrMostRecent('123', false); + expect(result).to.deep.equal({ aiEvaluationId: '123' }); + }); + + it('should return the most recent cache entry', async () => { + sinon.stub(cache, 'resolveFromCache').returns({ aiEvaluationId: '123', name: 'testName' }); + const result = cache.useIdOrMostRecent(undefined, true); + expect(result).to.deep.equal({ aiEvaluationId: '123', name: 'testName' }); + }); + + it('should throw an error if both aiEvaluationId and useMostRecent are provided', () => { + try { + cache.useIdOrMostRecent('123', true); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).message).to.equal('Cannot specify both an aiEvaluationId and use most recent flag'); + } + }); + + it('should throw an error if neither aiEvaluationId nor useMostRecent are provided', () => { + try { + cache.useIdOrMostRecent(undefined, false); + } catch (e) { + expect(e).to.be.instanceOf(SfError); + expect((e as SfError).message).to.equal('Must specify either an aiEvaluationId or use most recent flag'); + } + }); + }); +}); diff --git a/test/commands/agent/generate/test.test.ts b/test/commands/agent/generate/test.test.ts deleted file mode 100644 index 934f8c9..0000000 --- a/test/commands/agent/generate/test.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { TestContext } from '@salesforce/core/testSetup'; -import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; -import AgentGenerateTest from '../../../../src/commands/agent/generate/test.js'; - -describe('agent generate test', () => { - const $$ = new TestContext(); - let sfCommandStubs: ReturnType; - - beforeEach(() => { - sfCommandStubs = stubSfCommandUx($$.SANDBOX); - }); - - afterEach(() => { - $$.restore(); - }); - - it('runs hello', async () => { - await AgentGenerateTest.run([]); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('hello world'); - }); - - it('runs hello with --json and no provided name', async () => { - const result = await AgentGenerateTest.run([]); - expect(result.path).to.equal('src/commands/agent/generate/test.ts'); - }); - - it('runs hello world --name Astro', async () => { - await AgentGenerateTest.run(['--name', 'Astro']); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('hello Astro'); - }); -}); diff --git a/test/commands/agent/test/cancel.nut.ts b/test/commands/agent/test/cancel.nut.ts new file mode 100644 index 0000000..b6d5671 --- /dev/null +++ b/test/commands/agent/test/cancel.nut.ts @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { resolve } from 'node:path'; +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; +import { AgentTestRunResult } from '../../../../src/commands/agent/test/run.js'; +import { AgentTestCancelResult } from '../../../../src/commands/agent/test/cancel.js'; +import { AgentTestCache } from '../../../../src/agentTestCache.js'; + +describe('agent test cancel NUTs', () => { + let session: TestSession; + const mockDir = resolve('test/mocks'); + + before(async () => { + session = await TestSession.create({ + devhubAuthStrategy: 'AUTO', + project: { name: 'agentTestRun' }, + }); + }); + + after(async () => { + await session?.clean(); + }); + + it('should cancel async test run', async () => { + const runResult = execCmd( + `agent test run --name my_agent_tests --target-org ${session.hubOrg.username} --json`, + { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + } + ).jsonOutput; + + expect(runResult?.result.aiEvaluationId).to.be.ok; + + const output = execCmd( + `agent test cancel --job-id ${runResult?.result.aiEvaluationId} --target-org ${session.hubOrg.username} --json`, + { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + } + ).jsonOutput; + + expect(output?.result.success).to.be.true; + expect(output?.result.aiEvaluationId).to.equal('4KBSM000000003F4AQ'); + + // check that cache does not have an entry + const cache = await AgentTestCache.create(); + expect(() => cache.resolveFromCache()).to.throw('Could not find an aiEvaluationId to resume'); + }); +}); diff --git a/test/commands/agent/test/results.nut.ts b/test/commands/agent/test/results.nut.ts index c2d0a8e..5004e91 100644 --- a/test/commands/agent/test/results.nut.ts +++ b/test/commands/agent/test/results.nut.ts @@ -1,27 +1,56 @@ /* - * Copyright (c) 2023, salesforce.com, inc. + * Copyright (c) 2024, salesforce.com, inc. * All rights reserved. * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { resolve } from 'node:path'; import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { expect } from 'chai'; +import { AgentTestRunResult } from '../../../../src/commands/agent/test/run.js'; +import { AgentTestResultsResult } from '../../../../src/commands/agent/test/results.js'; +import { AgentTestCache } from '../../../../src/agentTestCache.js'; describe('agent test results NUTs', () => { let session: TestSession; + const mockDir = resolve('test/mocks'); before(async () => { - session = await TestSession.create({ devhubAuthStrategy: 'NONE' }); + session = await TestSession.create({ + devhubAuthStrategy: 'AUTO', + project: { name: 'agentTestRun' }, + }); }); after(async () => { await session?.clean(); }); - it('should display provided name', () => { - const name = 'World'; - const command = `agent test results --name ${name}`; - const output = execCmd(command, { ensureExitCode: 0 }).shellOutput.stdout; - expect(output).to.contain(name); + it('should get results of completed test run', async () => { + const runResult = execCmd( + `agent test run --name my_agent_tests --target-org ${session.hubOrg.username} --wait 5 --json`, + { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + } + ).jsonOutput; + + expect(runResult?.result.aiEvaluationId).to.be.ok; + expect(runResult?.result.status).to.equal('COMPLETED'); + + const output = execCmd( + `agent test results --job-id ${runResult?.result.aiEvaluationId} --target-org ${session.hubOrg.username} --json`, + { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + } + ).jsonOutput; + + expect(output?.result.status).to.equal('COMPLETED'); + expect(output?.result.testCases.length).to.equal(2); + + // check that cache does not have an entry + const cache = await AgentTestCache.create(); + expect(() => cache.resolveFromCache()).to.throw('Could not find an aiEvaluationId to resume'); }); }); diff --git a/test/commands/agent/test/results.test.ts b/test/commands/agent/test/results.test.ts deleted file mode 100644 index eb80975..0000000 --- a/test/commands/agent/test/results.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { TestContext } from '@salesforce/core/testSetup'; -import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; -import AgentTestResults from '../../../../src/commands/agent/test/results.js'; - -describe('agent test results', () => { - const $$ = new TestContext(); - let sfCommandStubs: ReturnType; - - beforeEach(() => { - sfCommandStubs = stubSfCommandUx($$.SANDBOX); - }); - - afterEach(() => { - $$.restore(); - }); - - it('runs hello', async () => { - await AgentTestResults.run([]); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('hello world'); - }); - - it('runs hello with --json and no provided name', async () => { - const result = await AgentTestResults.run([]); - expect(result.path).to.equal('src/commands/agent/test/results.ts'); - }); - - it('runs hello world --name Astro', async () => { - await AgentTestResults.run(['--name', 'Astro']); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('hello Astro'); - }); -}); diff --git a/test/commands/agent/test/resume.nut.ts b/test/commands/agent/test/resume.nut.ts index ba8a9bb..bc84d67 100644 --- a/test/commands/agent/test/resume.nut.ts +++ b/test/commands/agent/test/resume.nut.ts @@ -1,27 +1,55 @@ /* - * Copyright (c) 2023, salesforce.com, inc. + * Copyright (c) 2024, salesforce.com, inc. * All rights reserved. * Licensed under the BSD 3-Clause license. * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ +import { resolve } from 'node:path'; import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; import { expect } from 'chai'; +import { AgentTestRunResult } from '../../../../src/commands/agent/test/run.js'; +import { AgentTestResumeResult } from '../../../../src/commands/agent/test/resume.js'; +import { AgentTestCache } from '../../../../src/agentTestCache.js'; describe('agent test resume NUTs', () => { let session: TestSession; + const mockDir = resolve('test/mocks'); before(async () => { - session = await TestSession.create({ devhubAuthStrategy: 'NONE' }); + session = await TestSession.create({ + devhubAuthStrategy: 'AUTO', + project: { name: 'agentTestRun' }, + }); }); after(async () => { await session?.clean(); }); - it('should display provided name', () => { - const name = 'World'; - const command = `agent test resume --name ${name}`; - const output = execCmd(command, { ensureExitCode: 0 }).shellOutput.stdout; - expect(output).to.contain(name); + it('should resume async test run', async () => { + const runResult = execCmd( + `agent test run --name my_agent_tests --target-org ${session.hubOrg.username} --json`, + { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + } + ).jsonOutput; + + expect(runResult?.result.aiEvaluationId).to.be.ok; + + const output = execCmd( + `agent test resume --job-id ${runResult?.result.aiEvaluationId} --target-org ${session.hubOrg.username} --json`, + { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + } + ).jsonOutput; + + expect(output?.result.status).to.equal('COMPLETED'); + expect(output?.result.aiEvaluationId).to.equal('4KBSM000000003F4AQ'); + + // check that cache does not have an entry + const cache = await AgentTestCache.create(); + expect(() => cache.resolveFromCache()).to.throw('Could not find an aiEvaluationId to resume'); }); }); diff --git a/test/commands/agent/test/resume.test.ts b/test/commands/agent/test/resume.test.ts deleted file mode 100644 index 3c2e9b0..0000000 --- a/test/commands/agent/test/resume.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { TestContext } from '@salesforce/core/testSetup'; -import { expect } from 'chai'; -import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; -import AgentTestResume from '../../../../src/commands/agent/test/resume.js'; - -describe('agent test resume', () => { - const $$ = new TestContext(); - let sfCommandStubs: ReturnType; - - beforeEach(() => { - sfCommandStubs = stubSfCommandUx($$.SANDBOX); - }); - - afterEach(() => { - $$.restore(); - }); - - it('runs hello', async () => { - await AgentTestResume.run([]); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('hello world'); - }); - - it('runs hello with --json and no provided name', async () => { - const result = await AgentTestResume.run([]); - expect(result.path).to.equal('src/commands/agent/test/resume.ts'); - }); - - it('runs hello world --name Astro', async () => { - await AgentTestResume.run(['--name', 'Astro']); - const output = sfCommandStubs.log - .getCalls() - .flatMap((c) => c.args) - .join('\n'); - expect(output).to.include('hello Astro'); - }); -}); diff --git a/test/commands/agent/test/run.nut.ts b/test/commands/agent/test/run.nut.ts new file mode 100644 index 0000000..13c3a99 --- /dev/null +++ b/test/commands/agent/test/run.nut.ts @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ +import { resolve } from 'node:path'; +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; +import { AgentTestRunResult } from '../../../../src/commands/agent/test/run.js'; +import { AgentTestCache } from '../../../../src/agentTestCache.js'; + +describe('agent test run NUTs', () => { + let session: TestSession; + const mockDir = resolve('test/mocks'); + + before(async () => { + session = await TestSession.create({ + devhubAuthStrategy: 'AUTO', + project: { name: 'agentTestRun' }, + }); + }); + + after(async () => { + await session?.clean(); + }); + + it('should start async test run', async () => { + const name = 'my_agent_tests'; + const command = `agent test run --name ${name} --target-org ${session.hubOrg.username} --json`; + const output = execCmd(command, { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + }).jsonOutput; + expect(output?.result.status).to.equal('NEW'); + expect(output?.result.aiEvaluationId).to.equal('4KBSM000000003F4AQ'); + + // check cache for test run entry + const cache = await AgentTestCache.create(); + const testRun = cache.resolveFromCache(); + expect(testRun.aiEvaluationId).to.equal('4KBSM000000003F4AQ'); + expect(testRun.name).to.equal(name); + }); + + it('should poll for test run completion when --wait is used', async () => { + const name = 'my_agent_tests'; + const command = `agent test run --name ${name} --target-org ${session.hubOrg.username} --wait 5 --json`; + const output = execCmd(command, { + ensureExitCode: 0, + env: { ...process.env, SF_MOCK_DIR: mockDir }, + }).jsonOutput; + + expect(output?.result.status).to.equal('COMPLETED'); + expect(output?.result.aiEvaluationId).to.equal('4KBSM000000003F4AQ'); + + // check that cache does not have an entry + const cache = await AgentTestCache.create(); + expect(() => cache.resolveFromCache()).to.throw('Could not find an aiEvaluationId to resume'); + }); +}); diff --git a/test/mocks/connect_agent-job-spec.json b/test/mocks/connect_agent-job-spec.json new file mode 100644 index 0000000..5f32c3b --- /dev/null +++ b/test/mocks/connect_agent-job-spec.json @@ -0,0 +1,90 @@ +{ + "isSuccess": true, + "type": "customer_facing", + "role": "replace me", + "companyName": "replace me", + "companyDescription": "replace me", + "companyWebsite": "replace me", + "jobSpecs": [ + { + "jobTitle": "Guest_Experience_Enhancement", + "jobDescription": "Develop and implement entertainment programs to enhance guest experience." + }, + { + "jobTitle": "Event_Planning_and_Execution", + "jobDescription": "Plan, organize, and execute resort events and activities." + }, + { + "jobTitle": "Vendor_Management", + "jobDescription": "Coordinate with external vendors for event supplies and services." + }, + { + "jobTitle": "Staff_Training_and_Development", + "jobDescription": "Train and develop staff to deliver exceptional entertainment services." + }, + { + "jobTitle": "Budget_Management", + "jobDescription": "Manage budgets for entertainment activities and events." + }, + { + "jobTitle": "Guest_Feedback_Analysis", + "jobDescription": "Collect and analyze guest feedback to improve entertainment offerings." + }, + { + "jobTitle": "Marketing_Collaboration", + "jobDescription": "Work with marketing to promote events and entertainment activities." + }, + { + "jobTitle": "Technology_Integration", + "jobDescription": "Utilize technology to enhance guest engagement and streamline operations." + }, + { + "jobTitle": "Safety_and_Compliance", + "jobDescription": "Ensure all entertainment activities comply with safety regulations." + }, + { + "jobTitle": "Performance_Monitoring", + "jobDescription": "Monitor and evaluate the performance of entertainment programs." + }, + { + "jobTitle": "Community_Partnerships", + "jobDescription": "Build partnerships with local artists and performers." + }, + { + "jobTitle": "Inventory_Management", + "jobDescription": "Manage inventory of entertainment equipment and supplies." + }, + { + "jobTitle": "Custom_Experience_Creation", + "jobDescription": "Design personalized entertainment experiences for VIP guests." + }, + { + "jobTitle": "Data_Reporting", + "jobDescription": "Generate reports on entertainment program performance and guest satisfaction." + }, + { + "jobTitle": "Crisis_Management", + "jobDescription": "Develop plans to handle emergencies during entertainment events." + }, + { + "jobTitle": "Digital_Engagement", + "jobDescription": "Enhance online presence and engagement through social media." + }, + { + "jobTitle": "Salesforce_Integration", + "jobDescription": "Utilize Salesforce to track guest preferences and tailor entertainment." + }, + { + "jobTitle": "Trend_Analysis", + "jobDescription": "Stay updated on industry trends to keep entertainment offerings fresh." + }, + { + "jobTitle": "Cross_Department_Coordination", + "jobDescription": "Collaborate with other departments to ensure seamless guest experience." + }, + { + "jobTitle": "Resource_Optimization", + "jobDescription": "Optimize the use of resources to maximize guest satisfaction." + } + ] +} diff --git a/test/mocks/einstein_ai-evaluations_runs.json b/test/mocks/einstein_ai-evaluations_runs.json new file mode 100644 index 0000000..87f063c --- /dev/null +++ b/test/mocks/einstein_ai-evaluations_runs.json @@ -0,0 +1,4 @@ +{ + "aiEvaluationId": "4KBSM000000003F4AQ", + "status": "NEW" +} diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json new file mode 100644 index 0000000..daf2bbc --- /dev/null +++ b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json @@ -0,0 +1,4 @@ +{ + "status": "IN_PROGRESS", + "startTime": "2024-11-13T15:00:00.000Z" +} diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json new file mode 100644 index 0000000..daf2bbc --- /dev/null +++ b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json @@ -0,0 +1,4 @@ +{ + "status": "IN_PROGRESS", + "startTime": "2024-11-13T15:00:00.000Z" +} diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json new file mode 100644 index 0000000..d4f6503 --- /dev/null +++ b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json @@ -0,0 +1,4 @@ +{ + "status": "COMPLETED", + "startTime": "2024-11-13T15:00:00.000Z" +} diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_cancel.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_cancel.json new file mode 100644 index 0000000..5550c6d --- /dev/null +++ b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_cancel.json @@ -0,0 +1,3 @@ +{ + "success": true +} diff --git a/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json new file mode 100644 index 0000000..b895af0 --- /dev/null +++ b/test/mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json @@ -0,0 +1,82 @@ +{ + "status": "COMPLETED", + "startTime": "2024-11-28T12:00:00Z", + "endTime": "2024-11-28T12:05:00Z", + "errorMessage": null, + "testCases": [ + { + "status": "COMPLETED", + "number": 1, + "startTime": "2024-11-28T12:00:10Z", + "endTime": "2024-11-28T12:00:20Z", + "generatedData": { + "type": "AGENT", + "actionsSequence": ["Action1", "Action2"], + "outcome": "Success", + "topic": "Mathematics", + "inputTokensCount": 50, + "outputTokensCount": 55 + }, + "expectationResults": [ + { + "name": "topic_sequence_match", + "actualValue": "Result A", + "expectedValue": "Result A", + "score": 1.0, + "result": "Passed", + "metricLabel": "Accuracy", + "metricExplainability": "Measures the correctness of the result.", + "status": "Completed", + "startTime": "2024-11-28T12:00:12Z", + "endTime": "2024-11-28T12:00:13Z", + "errorCode": null, + "errorMessage": null + }, + { + "name": "action_sequence_match", + "actualValue": "Result B", + "expectedValue": "Result B", + "score": 0.9, + "result": "Passed", + "metricLabel": "Precision", + "metricExplainability": "Measures the precision of the result.", + "status": "Completed", + "startTime": "2024-11-28T12:00:14Z", + "endTime": "2024-11-28T12:00:15Z", + "errorCode": null, + "errorMessage": null + } + ] + }, + { + "status": "ERROR", + "number": 2, + "startTime": "2024-11-28T12:00:30Z", + "endTime": "2024-11-28T12:00:40Z", + "generatedData": { + "type": "AGENT", + "actionsSequence": ["Action3", "Action4"], + "outcome": "Failure", + "topic": "Physics", + "inputTokensCount": 60, + "outputTokensCount": 50 + }, + "expectationResults": [ + { + "name": "topic_sequence_match", + "actualValue": "Result C", + "expectedValue": "Result D", + "score": 0.5, + "result": "Failed", + "metricLabel": "Accuracy", + "metricExplainability": "Measures the correctness of the result.", + "status": "Completed", + "startTime": "2024-11-28T12:00:32Z", + "endTime": "2024-11-28T12:00:33Z", + "errorCode": null, + "errorMessage": null + } + ] + } + ] +} diff --git a/test/nut/agent-test-run.nut.ts b/test/nut/agent-test-run.nut.ts deleted file mode 100644 index 01a4f84..0000000 --- a/test/nut/agent-test-run.nut.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2021, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ - -import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; -import { expect } from 'chai'; -import { AgentTestRunResult } from '../../src/commands/agent/test/run.js'; - -let testSession: TestSession; - -describe('agent test run NUTs', () => { - before('prepare session', async () => { - testSession = await TestSession.create({ - devhubAuthStrategy: 'AUTO', - scratchOrgs: [ - { - edition: 'developer', - setDefault: true, - }, - ], - }); - }); - - after(async () => { - await testSession?.clean(); - }); - - it('should return a job ID', () => { - const result = execCmd('agent test run -i 4KBSM000000003F4AQ --json', { ensureExitCode: 0 }) - .jsonOutput?.result; - expect(result?.success).to.equal(true); - expect(result?.jobId).to.be.ok; - }); -}); diff --git a/test/unit/agent-test-run.test.ts b/test/unit/agent-test-run.test.ts deleted file mode 100644 index 0b13124..0000000 --- a/test/unit/agent-test-run.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ - -import { runCommand } from '@oclif/test'; -import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup'; -import { expect } from 'chai'; - -describe('agent run test', () => { - const $$ = new TestContext(); - const testOrg = new MockTestOrgData(); - - afterEach(() => { - $$.restore(); - }); - - it('runs agent run test', async () => { - const { stdout } = await runCommand(`agent:test:run -i the-id -o ${testOrg.username}`); - expect(stdout).to.include('Agent Test Run: the-id'); - }); -}); diff --git a/yarn.lock b/yarn.lock index 43fd884..8093fb0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1360,6 +1360,22 @@ strip-ansi "^7.1.0" wrap-ansi "^9.0.0" +"@oclif/table@^0.3.3": + version "0.3.5" + resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.3.5.tgz#118149eab364f3485eab5c9fd0d717c56082bacb" + integrity sha512-1IjoVz7WAdUdBW5vYIRc6wt9N7Ezwll6AtdmeqLQ8lUmB9gQJVyeb7dqXtUaUvIG7bZMvryfPe6Xibeo5FTCWA== + dependencies: + "@oclif/core" "^4" + "@types/react" "^18.3.12" + change-case "^5.4.4" + cli-truncate "^4.0.0" + ink "^5.0.1" + natural-orderby "^3.0.2" + object-hash "^3.0.0" + react "^18.3.1" + strip-ansi "^7.1.0" + wrap-ansi "^9.0.0" + "@oclif/test@^4.1.0": version "4.1.0" resolved "https://registry.yarnpkg.com/@oclif/test/-/test-4.1.0.tgz#7935e3707cf07480790139e02973196d18d16822" @@ -1373,13 +1389,16 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@salesforce/agents@^0.1.4": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.1.4.tgz#8a1660dbcf613401ba3a564038611a278b49ac2a" - integrity sha512-lUK1h13B93IGI6+mLbFBbM07kI0Yj5qmLuHaguGLU88/tsgLLMq2gpoENNBJXKxtg+0h9uwjt0/i44KwSxc75A== +"@salesforce/agents@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.2.2.tgz#c32e35e043f60ae6b4192448984342541bb0fc81" + integrity sha512-X5HGMHLFqK8rpXrNF4r/dYEdseu//0PnI6oGuWoVwnNPgJ4D0b3iIqTYeznpVEYqOa8wVZdceG2pZXd0GiO+9w== dependencies: - "@salesforce/core" "^8.5.2" + "@oclif/table" "^0.3.3" + "@salesforce/core" "^8.8.0" "@salesforce/kit" "^3.2.3" + "@salesforce/sf-plugins-core" "^12.1.0" + nock "^13.5.6" "@salesforce/cli-plugins-testkit@^5.3.35": version "5.3.35" @@ -1397,7 +1416,7 @@ strip-ansi "6.0.1" ts-retry-promise "^0.8.1" -"@salesforce/core@^8.5.1", "@salesforce/core@^8.5.2", "@salesforce/core@^8.5.7", "@salesforce/core@^8.6.2", "@salesforce/core@^8.6.3", "@salesforce/core@^8.8.0": +"@salesforce/core@^8.5.1", "@salesforce/core@^8.5.7", "@salesforce/core@^8.6.2", "@salesforce/core@^8.6.3", "@salesforce/core@^8.8.0": version "8.8.0" resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-8.8.0.tgz#849c07ea3a2548ca201fc0fe8baef9b36a462194" integrity sha512-HWGdRiy/MPCJ2KHz+W+cnqx0O9xhx9+QYvwP8bn9PE27wj0A/NjTi4xrqIWk1M+fE4dXHycE+8qPf4b540euvg== @@ -5010,6 +5029,11 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== +json-stringify-safe@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== + json5@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" @@ -5610,6 +5634,15 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" +nock@^13.5.6: + version "13.5.6" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.5.6.tgz#5e693ec2300bbf603b61dae6df0225673e6c4997" + integrity sha512-o2zOYiCpzRqSzPj0Zt/dQ/DqZeYoaQ7TUonc/xUPjCGl9WeHpNbxgVvOquXYAaJzI0M9BXV3HTzG0p8IUAbBTQ== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + propagate "^2.0.0" + node-fetch@^2.6.1, node-fetch@^2.6.9: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -6114,6 +6147,11 @@ process@^0.11.10: resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== +propagate@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== + proper-lockfile@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.2.tgz#c8b9de2af6b2f1601067f98e01ac66baa223141f" From 75d6403b466546181db2b697163b5b1b689a2266 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Thu, 5 Dec 2024 09:51:06 -0700 Subject: [PATCH 10/12] fix: keep up with agents lib --- command-snapshot.json | 6 +++--- package.json | 2 +- src/commands/agent/generate/spec.ts | 4 +--- src/commands/agent/test/results.ts | 8 +++++--- src/commands/agent/test/resume.ts | 11 +++++++++-- src/commands/agent/test/run.ts | 9 +++++++-- src/flags.ts | 1 - src/testStages.ts | 15 +++++++++------ yarn.lock | 26 +++++--------------------- 9 files changed, 40 insertions(+), 42 deletions(-) diff --git a/command-snapshot.json b/command-snapshot.json index b676e13..4031136 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -47,7 +47,7 @@ "alias": [], "command": "agent:test:results", "flagAliases": [], - "flagChars": ["i", "o", "r"], + "flagChars": ["i", "o"], "flags": ["api-version", "flags-dir", "job-id", "json", "result-format", "target-org"], "plugin": "@salesforce/plugin-agent" }, @@ -56,14 +56,14 @@ "command": "agent:test:resume", "flagAliases": [], "flagChars": ["i", "o", "r", "w"], - "flags": ["api-version", "flags-dir", "job-id", "json", "target-org", "use-most-recent", "wait"], + "flags": ["api-version", "flags-dir", "job-id", "json", "result-format", "target-org", "use-most-recent", "wait"], "plugin": "@salesforce/plugin-agent" }, { "alias": [], "command": "agent:test:run", "flagAliases": [], - "flagChars": ["n", "o", "r", "w"], + "flagChars": ["n", "o", "w"], "flags": ["api-version", "flags-dir", "json", "name", "result-format", "target-org", "wait"], "plugin": "@salesforce/plugin-agent" } diff --git a/package.json b/package.json index e57134b..492133c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "@inquirer/select": "^4.0.1", "@oclif/core": "^4", "@oclif/multi-stage-output": "^0.7.12", - "@salesforce/agents": "^0.2.2", + "@salesforce/agents": "^0.3.0", "@salesforce/core": "^8.8.0", "@salesforce/kit": "^3.2.1", "@salesforce/sf-plugins-core": "^12.1.0", diff --git a/src/commands/agent/generate/spec.ts b/src/commands/agent/generate/spec.ts index d067c17..93191fb 100644 --- a/src/commands/agent/generate/spec.ts +++ b/src/commands/agent/generate/spec.ts @@ -142,9 +142,7 @@ export default class AgentCreateSpec extends SfCommand { this.log(); this.styledHeader('Agent Details'); - const type = (await this.getFlagOrPrompt(flags.type, FLAGGABLE_PROMPTS.type)) as - | 'customer_facing' - | 'employee_facing'; + const type = (await this.getFlagOrPrompt(flags.type, FLAGGABLE_PROMPTS.type)) as 'customer' | 'internal'; const role = await this.getFlagOrPrompt(flags.role, FLAGGABLE_PROMPTS.role); const companyName = await this.getFlagOrPrompt(flags['company-name'], FLAGGABLE_PROMPTS['company-name']); const companyDescription = await this.getFlagOrPrompt( diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index a009d83..f67e27b 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -7,7 +7,7 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester, AgentTestDetailsResponse } from '@salesforce/agents'; +import { AgentTester, AgentTestDetailsResponse, humanFormat } from '@salesforce/agents'; import { resultFormatFlag } from '../../../flags.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); @@ -37,8 +37,10 @@ export default class AgentTestResults extends SfCommand const { flags } = await this.parse(AgentTestResults); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const { response, formatted } = await agentTester.details(flags['job-id'], flags['result-format']); - this.log(formatted); + const response = await agentTester.details(flags['job-id']); + if (flags['result-format'] === 'human') { + this.log(await humanFormat(flags['job-id'], response)); + } return response; } } diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index fc349c4..2087612 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -7,9 +7,10 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester } from '@salesforce/agents'; +import { AgentTester, humanFormat } from '@salesforce/agents'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; +import { resultFormatFlag } from '../../../flags.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.resume'); @@ -46,6 +47,7 @@ export default class AgentTestResume extends SfCommand { summary: messages.getMessage('flags.wait.summary'), description: messages.getMessage('flags.wait.description'), }), + 'result-format': resultFormatFlag(), }; public async run(): Promise { @@ -61,10 +63,15 @@ export default class AgentTestResume extends SfCommand { mso.start({ id: aiEvaluationId }); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const completed = await mso.poll(agentTester, aiEvaluationId, flags.wait); + const { completed, response } = await mso.poll(agentTester, aiEvaluationId, flags.wait); if (completed) await agentTestCache.removeCacheEntry(aiEvaluationId); mso.stop(); + + if (response && flags['result-format'] === 'human') { + this.log(await humanFormat(name ?? aiEvaluationId, response)); + } + return { status: 'COMPLETED', aiEvaluationId, diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index c383b5b..1b2a2c9 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -7,7 +7,7 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; -import { AgentTester } from '@salesforce/agents'; +import { AgentTester, humanFormat } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; import { resultFormatFlag } from '../../../flags.js'; import { AgentTestCache } from '../../../agentTestCache.js'; @@ -64,9 +64,14 @@ export default class AgentTestRun extends SfCommand { await agentTestCache.createCacheEntry(response.aiEvaluationId, flags.name); if (flags.wait?.minutes) { - const completed = await mso.poll(agentTester, response.aiEvaluationId, flags.wait); + const { completed, response: detailsResponse } = await mso.poll(agentTester, response.aiEvaluationId, flags.wait); if (completed) await agentTestCache.removeCacheEntry(response.aiEvaluationId); + mso.stop(); + + if (detailsResponse && flags['result-format'] === 'human') { + this.log(await humanFormat(flags.name, detailsResponse)); + } return { status: 'COMPLETED', aiEvaluationId: response.aiEvaluationId, diff --git a/src/flags.ts b/src/flags.ts index a2c0c35..418e308 100644 --- a/src/flags.ts +++ b/src/flags.ts @@ -18,6 +18,5 @@ export const resultFormatFlag = Flags.option({ // 'junit' ] as const, default: 'human', - char: 'r', summary: messages.getMessage('flags.result-format.summary'), }); diff --git a/src/testStages.ts b/src/testStages.ts index 274221d..8c0f9ec 100644 --- a/src/testStages.ts +++ b/src/testStages.ts @@ -7,7 +7,7 @@ import { colorize } from '@oclif/core/ux'; import { MultiStageOutput } from '@oclif/multi-stage-output'; -import { AgentTester } from '@salesforce/agents'; +import { AgentTestDetailsResponse, AgentTester } from '@salesforce/agents'; import { Lifecycle } from '@salesforce/core'; import { Duration } from '@salesforce/kit'; import { Ux } from '@salesforce/sf-plugins-core'; @@ -76,7 +76,11 @@ export class TestStages { this.mso.skipTo('Starting Tests', data); } - public async poll(agentTester: AgentTester, id: string, wait: Duration): Promise { + public async poll( + agentTester: AgentTester, + id: string, + wait: Duration + ): Promise<{ completed: boolean; response?: AgentTestDetailsResponse }> { this.mso.skipTo('Polling for Test Results'); const lifecycle = Lifecycle.getInstance(); lifecycle.on( @@ -91,16 +95,15 @@ export class TestStages { ); try { - const { formatted } = await agentTester.poll(id, { timeout: wait }); + const response = await agentTester.poll(id, { timeout: wait }); this.stop(); - this.ux.log(formatted); - return true; + return { completed: true, response }; } catch (e) { if (isTimeoutError(e)) { this.stop('async'); this.ux.log(`Client timed out after ${wait.minutes} minutes.`); this.ux.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); - return true; + return { completed: true }; } else { this.error(); throw e; diff --git a/yarn.lock b/yarn.lock index 8093fb0..663d365 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1344,23 +1344,7 @@ http-call "^5.2.2" lodash "^4.17.21" -"@oclif/table@^0.3.2": - version "0.3.3" - resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.3.3.tgz#5dc1c98cfa5415b131d77c85048df187fc241d12" - integrity sha512-sz6gGT1JAPP743vxl1491hxboIu0ZFHaP3gyvhz5Prgsuljp2NGyyu7JPEMeVImCnZ9N3K9cy3VXxRFEwRH/ig== - dependencies: - "@oclif/core" "^4" - "@types/react" "^18.3.12" - change-case "^5.4.4" - cli-truncate "^4.0.0" - ink "^5.0.1" - natural-orderby "^3.0.2" - object-hash "^3.0.0" - react "^18.3.1" - strip-ansi "^7.1.0" - wrap-ansi "^9.0.0" - -"@oclif/table@^0.3.3": +"@oclif/table@^0.3.2", "@oclif/table@^0.3.3": version "0.3.5" resolved "https://registry.yarnpkg.com/@oclif/table/-/table-0.3.5.tgz#118149eab364f3485eab5c9fd0d717c56082bacb" integrity sha512-1IjoVz7WAdUdBW5vYIRc6wt9N7Ezwll6AtdmeqLQ8lUmB9gQJVyeb7dqXtUaUvIG7bZMvryfPe6Xibeo5FTCWA== @@ -1389,10 +1373,10 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@salesforce/agents@^0.2.2": - version "0.2.2" - resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.2.2.tgz#c32e35e043f60ae6b4192448984342541bb0fc81" - integrity sha512-X5HGMHLFqK8rpXrNF4r/dYEdseu//0PnI6oGuWoVwnNPgJ4D0b3iIqTYeznpVEYqOa8wVZdceG2pZXd0GiO+9w== +"@salesforce/agents@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@salesforce/agents/-/agents-0.3.0.tgz#5f58d69eca1dde07daaf88bc2226b1a09e579666" + integrity sha512-BV/Fa+WN8IT5n+bsdDI8wga5dxjY9Rhu6eAvU3OCyRQ7F0nFd5uqLe2Ybo+0gLbGCvGCrV9gt8eJ5z4fsgLoDQ== dependencies: "@oclif/table" "^0.3.3" "@salesforce/core" "^8.8.0" From 38eef81f144739596fc696eccb7264d55cffb6e6 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Thu, 5 Dec 2024 09:57:29 -0700 Subject: [PATCH 11/12] fix: messages --- command-snapshot.json | 8 ------ messages/agent.generate.test.md | 19 ------------- messages/agent.test.cancel.md | 2 +- messages/agent.test.results.md | 12 ++++----- messages/agent.test.resume.md | 6 +---- schemas/agent-generate-test.json | 16 ----------- src/commands/agent/generate/test.ts | 42 ----------------------------- src/commands/agent/test/results.ts | 1 - 8 files changed, 8 insertions(+), 98 deletions(-) delete mode 100644 messages/agent.generate.test.md delete mode 100644 schemas/agent-generate-test.json delete mode 100644 src/commands/agent/generate/test.ts diff --git a/command-snapshot.json b/command-snapshot.json index 4031136..7ad7dc9 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -27,14 +27,6 @@ ], "plugin": "@salesforce/plugin-agent" }, - { - "alias": [], - "command": "agent:generate:test", - "flagAliases": [], - "flagChars": ["n"], - "flags": ["flags-dir", "json", "name"], - "plugin": "@salesforce/plugin-agent" - }, { "alias": [], "command": "agent:test:cancel", diff --git a/messages/agent.generate.test.md b/messages/agent.generate.test.md deleted file mode 100644 index fb78f84..0000000 --- a/messages/agent.generate.test.md +++ /dev/null @@ -1,19 +0,0 @@ -# summary - -Summary of a command. - -# description - -More information about a command. Don't repeat the summary. - -# flags.name.summary - -Description of a flag. - -# flags.name.description - -More information about a flag. Don't repeat the summary. - -# examples - -- <%= config.bin %> <%= command.id %> diff --git a/messages/agent.test.cancel.md b/messages/agent.test.cancel.md index d67f6ee..f9f8e91 100644 --- a/messages/agent.test.cancel.md +++ b/messages/agent.test.cancel.md @@ -18,4 +18,4 @@ Use the job ID of the most recent test evaluation. - Cancel a test for an Agent: - <%= config.bin %> <%= command.id %> --id AiEvalId + <%= config.bin %> <%= command.id %> --job-id AiEvalId diff --git a/messages/agent.test.results.md b/messages/agent.test.results.md index d93bb90..b46b851 100644 --- a/messages/agent.test.results.md +++ b/messages/agent.test.results.md @@ -1,19 +1,19 @@ # summary -Summary of a command. +Get the results of a test evaluation. # description -More information about a command. Don't repeat the summary. +Provide the AiEvaluation ID to get the results of a test evaluation. # flags.job-id.summary -Description of a flag. +The AiEvaluation ID. -# flags.job-id.description +# flags.use-most-recent.summary -More information about a flag. Don't repeat the summary. +Use the job ID of the most recent test evaluation. # examples -- <%= config.bin %> <%= command.id %> +- <%= config.bin %> <%= command.id %> --job-id AiEvalId diff --git a/messages/agent.test.resume.md b/messages/agent.test.resume.md index 2a2c78a..9c7122f 100644 --- a/messages/agent.test.resume.md +++ b/messages/agent.test.resume.md @@ -22,12 +22,8 @@ Number of minutes to wait for the command to complete and display results to the If the command continues to run after the wait period, the CLI returns control of the terminal window to you. -# flags.output-dir.summary - -Directory in which to store test run files. - # examples - Resume a test for an Agent: - <%= config.bin %> <%= command.id %> --id AiEvalId + <%= config.bin %> <%= command.id %> --job-id AiEvalId diff --git a/schemas/agent-generate-test.json b/schemas/agent-generate-test.json deleted file mode 100644 index a393999..0000000 --- a/schemas/agent-generate-test.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$ref": "#/definitions/AgentGenerateTestResult", - "definitions": { - "AgentGenerateTestResult": { - "type": "object", - "properties": { - "path": { - "type": "string" - } - }, - "required": ["path"], - "additionalProperties": false - } - } -} diff --git a/src/commands/agent/generate/test.ts b/src/commands/agent/generate/test.ts deleted file mode 100644 index d7c54aa..0000000 --- a/src/commands/agent/generate/test.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ - -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; - -Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); -const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.test'); - -export type AgentGenerateTestResult = { - path: string; -}; - -export default class AgentGenerateTest extends SfCommand { - public static readonly summary = messages.getMessage('summary'); - public static readonly description = messages.getMessage('description'); - public static readonly examples = messages.getMessages('examples'); - public static readonly state = 'beta'; - - public static readonly flags = { - name: Flags.string({ - summary: messages.getMessage('flags.name.summary'), - description: messages.getMessage('flags.name.description'), - char: 'n', - required: false, - }), - }; - - public async run(): Promise { - const { flags } = await this.parse(AgentGenerateTest); - - const name = flags.name ?? 'world'; - this.log(`hello ${name} from src/commands/agent/generate/test.ts`); - return { - path: 'src/commands/agent/generate/test.ts', - }; - } -} diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index f67e27b..5443fc4 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -26,7 +26,6 @@ export default class AgentTestResults extends SfCommand 'api-version': Flags.orgApiVersion(), 'job-id': Flags.string({ summary: messages.getMessage('flags.job-id.summary'), - description: messages.getMessage('flags.job-id.description'), char: 'i', required: true, }), From 470246875da56fc988db29bf905bad4582ba4e71 Mon Sep 17 00:00:00 2001 From: Mike Donnalley Date: Thu, 5 Dec 2024 10:05:14 -0700 Subject: [PATCH 12/12] test: remove old test --- test/commands/agent/generate/test.nut.ts | 27 ------------------------ 1 file changed, 27 deletions(-) delete mode 100644 test/commands/agent/generate/test.nut.ts diff --git a/test/commands/agent/generate/test.nut.ts b/test/commands/agent/generate/test.nut.ts deleted file mode 100644 index 98e4735..0000000 --- a/test/commands/agent/generate/test.nut.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023, salesforce.com, inc. - * All rights reserved. - * Licensed under the BSD 3-Clause license. - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - */ -import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; -import { expect } from 'chai'; - -describe('agent generate test NUTs', () => { - let session: TestSession; - - before(async () => { - session = await TestSession.create({ devhubAuthStrategy: 'NONE' }); - }); - - after(async () => { - await session?.clean(); - }); - - it('should display provided name', () => { - const name = 'World'; - const command = `agent generate test --name ${name}`; - const output = execCmd(command, { ensureExitCode: 0 }).shellOutput.stdout; - expect(output).to.contain(name); - }); -});