JS[major]: Evaluate

.vscode/settings.json

@@ -1,4 +1,9 @@
 {
+  "cSpell.words": ["atee"],
+  "python.testing.pytestArgs": ["python"],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true,
+  "python.formatting.provider": "black",
   "eslint.workingDirectories": [
     "./js"
   ]

js/README.md

@@ -51,7 +51,7 @@ yarn add langchain
 Tracing can be activated by setting the following environment variables or by manually specifying the LangChainTracer.
 
 ```typescript
-process.env["LANGCHAIN_TRACING_V2"] = "true";
+process.env["LANGSMITH_TRACING"] = "true";
 process.env["LANGCHAIN_ENDPOINT"] = "https://api.smith.langchain.com";
 process.env["LANGCHAIN_API_KEY"] = "<YOUR-LANGSMITH-API-KEY>";
 // process.env["LANGCHAIN_PROJECT"] = "My Project Name"; // Optional: "default" is used if not set

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.1.21",
+  "version": "0.1.22",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [
@@ -195,4 +195,4 @@
     },
     "./package.json": "./package.json"
   }
-}
+}

js/src/client.ts

@@ -33,8 +33,13 @@
   getRuntimeEnvironment,
 } from "./utils/env.js";
 
-import { RunEvaluator } from "./evaluation/evaluator.js";
+import {
+  EvaluationResult,
+  EvaluationResults,
+  RunEvaluator,
+} from "./evaluation/evaluator.js";
 import { __version__ } from "./index.js";
+import { assertUuid } from "./utils/_uuid.js";
 
 interface ClientConfig {
   apiUrl?: string;
@@ -152,7 +157,6 @@
   treeFilter?: string;
   /**
    * The values to include in the response.
-   *
    */
   select?: string[];
 }
@@ -174,14 +178,15 @@
 
 interface FeedbackCreate {
   id: string;
-  run_id: string;
+  run_id: string | null;
   key: string;
   score?: ScoreType;
   value?: ValueType;
   correction?: object | null;
   comment?: string | null;
   feedback_source?: feedback_source | KVMap | null;
   feedbackConfig?: FeedbackConfig;
+  session_id?: string;
 }
 
 interface FeedbackUpdate {
@@ -317,12 +322,6 @@
     .replace(/^'(.*)'$/, "$1");
 }
 
-function assertUuid(str: string): void {
-  if (!uuid.validate(str)) {
-    throw new Error(`Invalid UUID: ${str}`);
-  }
-}
-
 const handle429 = async (response?: Response) => {
   if (response?.status === 429) {
     const retryAfter =
@@ -2193,7 +2192,7 @@
   }
 
   public async createFeedback(
-    runId: string,
+    runId: string | null,
     key: string,
     {
       score,
@@ -2205,6 +2204,7 @@
       sourceRunId,
       feedbackId,
       feedbackConfig,
+      projectId,
     }: {
       score?: ScoreType;
       value?: ValueType;
@@ -2216,8 +2216,15 @@
       sourceRunId?: string;
       feedbackId?: string;
       eager?: boolean;
+      projectId?: string;
     }
   ): Promise<Feedback> {
+    if (!runId && !projectId) {
+      throw new Error("One of runId or projectId must be provided");
+    }
+    if (runId && projectId) {
+      throw new Error("Only one of runId or projectId can be provided");
+    }
     const feedback_source: feedback_source = {
       type: feedbackSourceType ?? "api",
       metadata: sourceInfo ?? {},
@@ -2245,6 +2252,7 @@
       comment,
       feedback_source: feedback_source,
       feedbackConfig,
+      session_id: projectId,
     };
     const url = `${this.apiUrl}/feedback`;
     const response = await this.caller.call(fetch, url, {
@@ -2430,4 +2438,49 @@
       yield* tokens;
     }
   }
+
+  _selectEvalResults(
+    results: EvaluationResult | EvaluationResults
+  ): Array<EvaluationResult> {
+    let results_: Array<EvaluationResult>;
+    if ("results" in results) {
+      results_ = results.results;
+    } else {
+      results_ = [results];
+    }
+    return results_;
+  }
+
+  public async logEvaluationFeedback(
+    evaluatorResponse: EvaluationResult | EvaluationResults,
+    run?: Run,
+    sourceInfo?: { [key: string]: any }
+  ): Promise<EvaluationResult[]> {
+    const results: Array<EvaluationResult> =
+      this._selectEvalResults(evaluatorResponse);
+    for (const res of results) {
+      let sourceInfo_ = sourceInfo || {};
+      if (res.evaluatorInfo) {
+        sourceInfo_ = { ...res.evaluatorInfo, ...sourceInfo_ };
+      }
+      let runId_: string | null = null;
+      if (res.targetRunId) {
+        runId_ = res.targetRunId;
+      } else if (run) {
+        runId_ = run.id;
+      }
+
+      await this.createFeedback(runId_, res.key, {
+        score: res.score,
+        value: res.value,
+        comment: res.comment,
+        correction: res.correction,
+        sourceInfo: sourceInfo_,
+        sourceRunId: res.sourceRunId,
+        feedbackConfig: res.feedbackConfig as FeedbackConfig | undefined,
+        feedbackSourceType: "model",
+      });
+    }
+    return results;
+  }
 }