fix: require all expectations

salesforcecli · Dec 17, 2024 · bfde4f5 · bfde4f5
1 parent c9104e7
commit bfde4f5
Show file tree

Hide file tree

Showing 2 changed files with 76 additions and 33 deletions.
diff --git a/src/commands/agent/generate/testset.ts b/src/commands/agent/generate/testset.ts
@@ -9,19 +9,17 @@ import { mkdir, writeFile } from 'node:fs/promises';
 import { SfCommand } from '@salesforce/sf-plugins-core';
 import { Messages } from '@salesforce/core';
 import input from '@inquirer/input';
-import select from '@inquirer/select';
 import confirm from '@inquirer/confirm';
 import { theme } from '../../../inquirer-theme.js';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
 const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.generate.testset');
 
-type ExpectationType = 'topic_sequence_match' | 'action_sequence_match' | 'bot_response_rating';
-
 export type TestSetInputs = {
   utterance: string;
-  expectationType: ExpectationType;
-  expectedValue: string;
+  actionSequenceExpectedValue: string;
+  botRatingExpectedValue: string;
+  topicSequenceExpectedValue: string;
 };
 
 async function promptForTestCase(): Promise<TestSetInputs> {
@@ -31,55 +29,73 @@ async function promptForTestCase(): Promise<TestSetInputs> {
     theme,
   });
 
-  const expectationType = await select<ExpectationType>({
-    message: 'What type of expectation would you like to test for the utterance?',
-    choices: ['topic_sequence_match', 'action_sequence_match', 'bot_response_rating'],
+  const topicSequenceExpectedValue = await input({
+    message: 'What is the expected value for the topic expectation?',
+    validate: (d: string): boolean | string => {
+      if (!d.length) {
+        return 'expected value cannot be empty';
+      }
+      return true;
+    },
     theme,
   });
 
-  const expectedValue = await input({
-    message: 'What is the expected value for the expectation?',
+  const actionSequenceExpectedValue = await input({
+    message: 'What is the expected value for the action expectation?',
     validate: (d: string): boolean | string => {
       if (!d.length) {
         return 'expected value cannot be empty';
       }
 
-      if (expectationType === 'action_sequence_match') {
-        return d.split(',').length > 1 || 'expected value must be a comma-separated list of actions';
+      return d.split(',').length > 1 || 'expected value must be a comma-separated list of actions';
+    },
+    theme,
+  });
+
+  const botRatingExpectedValue = await input({
+    message: 'What is the expected value for the bot rating expectation?',
+    validate: (d: string): boolean | string => {
+      if (!d.length) {
+        return 'expected value cannot be empty';
       }
 
-      return true;
+      return (Number(d) >= 0 && Number(d) <= 5) || 'expected value must be a number between 0 and 5';
     },
     theme,
   });
 
   return {
     utterance,
-    expectationType,
-    expectedValue,
+    actionSequenceExpectedValue,
+    botRatingExpectedValue,
+    topicSequenceExpectedValue,
   };
 }
 
 export function constructTestSetXML(testCases: TestSetInputs[]): string {
   const tab = '  ';
   let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<AiEvaluationTestSet>\n${tab}<subjectType>AGENT</subjectType>\n`;
   testCases.forEach((testCase, i) => {
-    const expectedValue =
-      testCase.expectationType === 'action_sequence_match'
-        ? `[${testCase.expectedValue
-            .split(',')
-            .map((v) => `"${v}"`)
-            .join(',')}]`
-        : testCase.expectedValue;
     xml += `  <testCase>
     <number>${i + 1}</number>
     <inputs>
       <utterance>${testCase.utterance}</utterance>
     </inputs>
     <expectations>
       <expectation>
-        <name>${testCase.expectationType}</name>
-        <expectedValue>${expectedValue}</expectedValue>
+        <name>topic_sequence_match</name>
+        <expectedValue>${testCase.topicSequenceExpectedValue}</expectedValue>
+      </expectation>
+      <expectation>
+        <name>action_sequence_match</name>
+        <expectedValue>${`[${testCase.actionSequenceExpectedValue
+          .split(',')
+          .map((v) => `"${v}"`)
+          .join(',')}]`}</expectedValue>
+      </expectation>
+      <expectation>
+        <name>bot_response_rating</name>
+        <expectedValue>${testCase.botRatingExpectedValue}</expectedValue>
       </expectation>
     </expectations>
   </testCase>\n`;

diff --git a/test/commands/agent/generate/testset.test.ts b/test/commands/agent/generate/testset.test.ts
@@ -12,18 +12,21 @@ describe('constructTestSetXML', () => {
     const testCases = [
       {
         utterance: 'hello',
-        expectationType: 'topic_sequence_match',
-        expectedValue: 'greeting',
+        actionSequenceExpectedValue: 'foo,bar',
+        botRatingExpectedValue: 'baz',
+        topicSequenceExpectedValue: 'qux',
       },
       {
         utterance: 'goodbye',
-        expectationType: 'action_sequence_match',
-        expectedValue: 'farewell,seeya',
+        actionSequenceExpectedValue: 'foo,bar',
+        botRatingExpectedValue: 'baz',
+        topicSequenceExpectedValue: 'qux',
       },
       {
         utterance: 'how are you',
-        expectationType: 'bot_response_rating',
-        expectedValue: '.5',
+        actionSequenceExpectedValue: 'foo,bar',
+        botRatingExpectedValue: 'baz',
+        topicSequenceExpectedValue: 'qux',
       },
     ] satisfies TestSetInputs[];
 
@@ -40,7 +43,15 @@ describe('constructTestSetXML', () => {
     <expectations>
       <expectation>
         <name>topic_sequence_match</name>
-        <expectedValue>greeting</expectedValue>
+        <expectedValue>qux</expectedValue>
+      </expectation>
+      <expectation>
+        <name>action_sequence_match</name>
+        <expectedValue>["foo","bar"]</expectedValue>
+      </expectation>
+      <expectation>
+        <name>bot_response_rating</name>
+        <expectedValue>baz</expectedValue>
       </expectation>
     </expectations>
   </testCase>
@@ -50,9 +61,17 @@ describe('constructTestSetXML', () => {
       <utterance>goodbye</utterance>
     </inputs>
     <expectations>
+      <expectation>
+        <name>topic_sequence_match</name>
+        <expectedValue>qux</expectedValue>
+      </expectation>
       <expectation>
         <name>action_sequence_match</name>
-        <expectedValue>["farewell","seeya"]</expectedValue>
+        <expectedValue>["foo","bar"]</expectedValue>
+      </expectation>
+      <expectation>
+        <name>bot_response_rating</name>
+        <expectedValue>baz</expectedValue>
       </expectation>
     </expectations>
   </testCase>
@@ -62,9 +81,17 @@ describe('constructTestSetXML', () => {
       <utterance>how are you</utterance>
     </inputs>
     <expectations>
+      <expectation>
+        <name>topic_sequence_match</name>
+        <expectedValue>qux</expectedValue>
+      </expectation>
+      <expectation>
+        <name>action_sequence_match</name>
+        <expectedValue>["foo","bar"]</expectedValue>
+      </expectation>
       <expectation>
         <name>bot_response_rating</name>
-        <expectedValue>.5</expectedValue>
+        <expectedValue>baz</expectedValue>
       </expectation>
     </expectations>
   </testCase>