From d7db1ea7d5b6f5c86dbd3956919e6b722bc10a1c Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Wed, 30 Oct 2024 15:36:40 -0600 Subject: [PATCH 1/6] fix: agent test run --- command-snapshot.json | 2 +- .../{agent.run.test.md => agent.test.run.md} | 0 schemas/agent-test-run.json | 22 +++++++++++++++++++ .../agent/{run/test.ts => test/run.ts} | 10 ++++----- 4 files changed, 28 insertions(+), 6 deletions(-) rename messages/{agent.run.test.md => agent.test.run.md} (100%) create mode 100644 schemas/agent-test-run.json rename src/commands/agent/{run/test.ts => test/run.ts} (89%) diff --git a/command-snapshot.json b/command-snapshot.json index 624dfb0..33ccc6b 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -28,7 +28,7 @@ }, { "alias": [], - "command": "agent:run:test", + "command": "agent:test:run", "flagAliases": [], "flagChars": ["d", "i", "o", "w"], "flags": ["flags-dir", "id", "json", "output-dir", "target-org", "wait"], diff --git a/messages/agent.run.test.md b/messages/agent.test.run.md similarity index 100% rename from messages/agent.run.test.md rename to messages/agent.test.run.md diff --git a/schemas/agent-test-run.json b/schemas/agent-test-run.json new file mode 100644 index 0000000..5ad1bc7 --- /dev/null +++ b/schemas/agent-test-run.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTestRunResult", + "definitions": { + "AgentTestRunResult": { + "type": "object", + "properties": { + "buildVersion": { + "type": "number" + }, + "jobId": { + "type": "string" + }, + "errorRepresentation": { + "type": "string" + } + }, + "required": ["buildVersion", "jobId"], + "additionalProperties": false + } + } +} diff --git a/src/commands/agent/run/test.ts b/src/commands/agent/test/run.ts similarity index 89% rename from src/commands/agent/run/test.ts rename to src/commands/agent/test/run.ts index 7ae1d38..866b6b5 100644 --- a/src/commands/agent/run/test.ts +++ b/src/commands/agent/test/run.ts @@ -9,15 +9,15 @@ import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); -const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.run.test'); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); -export type AgentRunTestResult = { +export type AgentTestRunResult = { buildVersion: number; jobId: string; errorRepresentation?: string; }; -export default class AgentRunTest extends SfCommand { +export default class AgentTestRun extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); @@ -50,8 +50,8 @@ export default class AgentRunTest extends SfCommand { // ??? api-version or build-version ??? }; - public async run(): Promise { - const { flags } = await this.parse(AgentRunTest); + public async run(): Promise { + const { flags } = await this.parse(AgentTestRun); this.log(`Starting tests for AiEvalDefinitionVersion: ${flags.id}`); From 71b85f0ed28024a815ad6faba81e4d77842e48fe Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Wed, 30 Oct 2024 15:37:53 -0600 Subject: [PATCH 2/6] fix: agent test run --- schemas/agent-run-test.json | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 schemas/agent-run-test.json diff --git a/schemas/agent-run-test.json b/schemas/agent-run-test.json deleted file mode 100644 index 76435b9..0000000 --- a/schemas/agent-run-test.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$ref": "#/definitions/AgentRunTestResult", - "definitions": { - "AgentRunTestResult": { - "type": "object", - "properties": { - "buildVersion": { - "type": "number" - }, - "jobId": { - "type": "string" - }, - "errorRepresentation": { - "type": "string" - } - }, - "required": ["buildVersion", "jobId"], - "additionalProperties": false - } - } -} From a76728fe2fea58e5a61383aa9dbb1ab6757e85d6 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Wed, 30 Oct 2024 15:44:03 -0600 Subject: [PATCH 3/6] fix: test --- test/unit/{agent-run-test.test.ts => agent-test-run.test.ts} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename test/unit/{agent-run-test.test.ts => agent-test-run.test.ts} (87%) diff --git a/test/unit/agent-run-test.test.ts b/test/unit/agent-test-run.test.ts similarity index 87% rename from test/unit/agent-run-test.test.ts rename to test/unit/agent-test-run.test.ts index ddceb5f..0bb1cd1 100644 --- a/test/unit/agent-run-test.test.ts +++ b/test/unit/agent-test-run.test.ts @@ -7,7 +7,7 @@ import { MockTestOrgData, TestContext } from '@salesforce/core/testSetup'; import { expect } from 'chai'; import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; -import AgentRunTest from '../../src/commands/agent/run/test.js'; +import AgentTestRun from '../../src/commands/agent/test/run.js'; describe('agent run test', () => { const $$ = new TestContext(); @@ -23,7 +23,7 @@ describe('agent run test', () => { }); it('runs agent run test', async () => { - await AgentRunTest.run(['-i', 'the-id', '-o', testOrg.username]); + await AgentTestRun.run(['-i', 'the-id', '-o', testOrg.username]); const output = sfCommandStubs.log .getCalls() .flatMap((c) => c.args) From 29983c2f70347aff24ad1b78effe801b15bee9a3 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Thu, 31 Oct 2024 16:48:20 -0600 Subject: [PATCH 4/6] fix: add agent test cancel --- command-snapshot.json | 8 +++++ messages/agent.test.cancel.md | 21 +++++++++++ schemas/agent-test-cancel.json | 25 +++++++++++++ schemas/agent-test-run.json | 13 ++++--- src/commands/agent/test/cancel.ts | 58 +++++++++++++++++++++++++++++++ src/commands/agent/test/run.ts | 14 +++++--- 6 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 messages/agent.test.cancel.md create mode 100644 schemas/agent-test-cancel.json create mode 100644 src/commands/agent/test/cancel.ts diff --git a/command-snapshot.json b/command-snapshot.json index 33ccc6b..138b240 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -26,6 +26,14 @@ ], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:test:cancel", + "flagAliases": [], + "flagChars": ["i", "o", "r"], + "flags": ["flags-dir", "job-id", "json", "target-org", "use-most-recent"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:test:run", diff --git a/messages/agent.test.cancel.md b/messages/agent.test.cancel.md new file mode 100644 index 0000000..5d6f229 --- /dev/null +++ b/messages/agent.test.cancel.md @@ -0,0 +1,21 @@ +# summary + +Cancel a running test for an Agent. + +# description + +Cancel a running test for an Agent, providing the AiEvaluation ID. + +# flags.id.summary + +The AiEvaluation ID. + +# flags.use-most-recent.summary + +Use the job ID of the most recent test evaluation. + +# examples + +- Cancel a test for an Agent: + + <%= config.bin %> <%= command.id %> --id AiEvalId diff --git a/schemas/agent-test-cancel.json b/schemas/agent-test-cancel.json new file mode 100644 index 0000000..81f5cbf --- /dev/null +++ b/schemas/agent-test-cancel.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTestCancelResult", + "definitions": { + "AgentTestCancelResult": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "success": { + "type": "boolean" + }, + "errorCode": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": ["jobId", "success"], + "additionalProperties": false + } + } +} diff --git a/schemas/agent-test-run.json b/schemas/agent-test-run.json index 5ad1bc7..5ac6a22 100644 --- a/schemas/agent-test-run.json +++ b/schemas/agent-test-run.json @@ -5,17 +5,20 @@ "AgentTestRunResult": { "type": "object", "properties": { - "buildVersion": { - "type": "number" - }, "jobId": { "type": "string" }, - "errorRepresentation": { + "success": { + "type": "boolean" + }, + "errorCode": { + "type": "string" + }, + "message": { "type": "string" } }, - "required": ["buildVersion", "jobId"], + "required": ["jobId", "success"], "additionalProperties": false } } diff --git a/src/commands/agent/test/cancel.ts b/src/commands/agent/test/cancel.ts new file mode 100644 index 0000000..a4e1f2c --- /dev/null +++ b/src/commands/agent/test/cancel.ts @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.cancel'); + +export type AgentTestCancelResult = { + jobId: string; // AiEvaluation.Id + success: boolean; + errorCode?: string; + message?: string; +}; + +export default class AgentTestCancel extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'job-id': Flags.string({ + char: 'i', + required: true, + summary: messages.getMessage('flags.id.summary'), + }), + 'use-most-recent': Flags.boolean({ + char: 'r', + summary: messages.getMessage('flags.use-most-recent.summary'), + exactlyOne: ['use-most-recent', 'job-id'], + }), + // + // Future flags: + // ??? api-version ??? + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTestCancel); + + this.log(`Canceling tests for AiEvaluation Job: ${flags['job-id']}`); + + // Call SF Eval Connect API passing AiEvaluation.Id + // POST to /einstein/ai-evaluations/{aiEvaluationId}/stop + + // Returns: AiEvaluation.Id + + return { + success: true, + jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id + }; + } +} diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 866b6b5..2a55c7a 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -12,9 +12,10 @@ Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.run'); export type AgentTestRunResult = { - buildVersion: number; - jobId: string; - errorRepresentation?: string; + jobId: string; // AiEvaluation.Id + success: boolean; + errorCode?: string; + message?: string; }; export default class AgentTestRun extends SfCommand { @@ -56,10 +57,13 @@ export default class AgentTestRun extends SfCommand { this.log(`Starting tests for AiEvalDefinitionVersion: ${flags.id}`); // Call SF Eval Connect API passing AiEvalDefinitionVersion.Id + // POST to /einstein/ai-evaluations/{aiEvalDefinitionVersionId}/start + + // Returns: AiEvaluation.Id return { - buildVersion: 62.0, // looks like API version - jobId: '4KBSM000000003F4AQ', // evaluationJobId; needed for getting status and stopping + success: true, + jobId: '4KBSM000000003F4AQ', // AiEvaluation.Id; needed for getting status and stopping }; } } From 1563e12031d59f2ef44e3e5281489ec588f9b225 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Fri, 1 Nov 2024 10:47:04 -0600 Subject: [PATCH 5/6] fix: add a nut --- test/nut/agent-test-run.nut.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test/nut/agent-test-run.nut.ts diff --git a/test/nut/agent-test-run.nut.ts b/test/nut/agent-test-run.nut.ts new file mode 100644 index 0000000..9be6099 --- /dev/null +++ b/test/nut/agent-test-run.nut.ts @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021, salesforce.com, inc. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import { expect } from 'chai'; +import { AgentTestRunResult } from '../../src/commands/agent/test/run.js'; + +let testSession: TestSession; + +describe('agent test run NUTs', () => { + before('prepare session', async () => { + testSession = await TestSession.create(); + }); + + after(async () => { + await testSession?.clean(); + }); + + it('should return a job ID', () => { + const result = execCmd('agent test run -i 4KBSM000000003F4AQ --json', { ensureExitCode: 0 }) + .jsonOutput?.result; + expect(result?.success).to.equal(true); + expect(result?.jobId).to.be.ok; + }); +}); From 6dddb6118c9b529eb2266b7dcd81b53e7c300076 Mon Sep 17 00:00:00 2001 From: Steve Hetzel Date: Fri, 1 Nov 2024 12:03:59 -0600 Subject: [PATCH 6/6] fix: nut needs an org --- test/nut/agent-test-run.nut.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/nut/agent-test-run.nut.ts b/test/nut/agent-test-run.nut.ts index 9be6099..01a4f84 100644 --- a/test/nut/agent-test-run.nut.ts +++ b/test/nut/agent-test-run.nut.ts @@ -13,7 +13,15 @@ let testSession: TestSession; describe('agent test run NUTs', () => { before('prepare session', async () => { - testSession = await TestSession.create(); + testSession = await TestSession.create({ + devhubAuthStrategy: 'AUTO', + scratchOrgs: [ + { + edition: 'developer', + setDefault: true, + }, + ], + }); }); after(async () => {