Merge branch 'main' into nc/batch-trace-combine

.github/workflows/js_test.yml

@@ -104,5 +104,7 @@ jobs:
         run: yarn install --immutable
       - name: Build
         run: yarn run build
+      - name: Check version
+        run: yarn run check-version
       - name: Test
         run: yarn run test

.github/workflows/release_js.yml

@@ -1,4 +1,4 @@
-name: Release
+name: JS Release
 
 on:
   push:
@@ -29,6 +29,8 @@ jobs:
         run: cd js && yarn install --immutable
       - name: Build
         run: cd js && yarn run build
+      - name: Check version
+        run: cd js && yarn run check-version
       - name: Publish package to NPM
         run: |
           cd js

.gitignore

@@ -159,3 +159,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 .DS_Store
+
+.envrc

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.0.53",
+  "version": "0.0.61",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "files": [
     "dist/",
@@ -28,6 +28,8 @@
   "types": "./dist/index.d.ts",
   "scripts": {
     "build": "yarn clean && yarn build:esm && yarn build:cjs && node scripts/create-entrypoints.js && node scripts/create-cli.js",
+    "bump-version": "node scripts/bump-version.js",
+    "check-version": "node scripts/check-version.js",
     "clean": "rm -rf dist/ && node scripts/create-entrypoints.js clean",
     "build:esm": "tsc --outDir dist/ && rm -rf dist/tests dist/**/tests",
     "build:cjs": "tsc --outDir dist-cjs/ -p tsconfig.cjs.json && node scripts/move-cjs-to-dist.js && rm -r dist-cjs",
@@ -119,4 +121,4 @@
     },
     "./package.json": "./package.json"
   }
-}
+}

js/scripts/bump-version.js

@@ -0,0 +1,21 @@
+import { readFileSync, writeFileSync } from 'fs';
+import process from 'process';
+const packageJson = JSON.parse(readFileSync('package.json'));
+
+let newVersion;
+if (process.argv.length > 2) {
+    newVersion = process.argv[2];
+} else {
+    const versionParts = packageJson.version.split('.');
+    versionParts[2] = parseInt(versionParts[2]) + 1;
+    newVersion = versionParts.join('.');
+}
+console.log(`Bumping version to ${newVersion}`);
+
+packageJson.version = newVersion;
+writeFileSync('package.json', JSON.stringify(packageJson, null, 2));
+
+const indexFilePath = 'src/index.ts';
+let indexFileContent = readFileSync(indexFilePath, 'utf-8');
+indexFileContent = indexFileContent.replace(/export const __version__ = "[0-9]+\.[0-9]+\.[0-9]+";/g, `export const __version__ = "${newVersion}";`);
+writeFileSync(indexFilePath, indexFileContent);

js/scripts/check-version.js

@@ -0,0 +1,17 @@
+import { readFileSync } from "fs";
+import { join } from "path";
+const indexFilePath = "src/index.ts";
+const packageJson = JSON.parse(readFileSync("package.json"));
+let indexFileContent = readFileSync(indexFilePath, "utf-8");
+
+const packageVersion = packageJson.version;
+const indexVersion = indexFileContent.match(
+  /__version__\s*=\s*['"]([^'"]+)['"]/
+)[1];
+
+if (packageVersion !== indexVersion) {
+  throw new Error(
+    `Version mismatch! package.json version: ${packageVersion}, index.ts version: ${indexVersion}`
+  );
+}
+console.log(`Version check passed: ${packageVersion} === ${indexVersion}`);

js/src/cli/docker-compose.yaml

@@ -1,4 +1,4 @@
-version: "3"
+version: "4"
 services:
   langchain-playground:
     image: langchain/${_LANGSMITH_IMAGE_PREFIX-}langchainplus-playground:${_LANGSMITH_IMAGE_VERSION:-latest}
@@ -25,6 +25,7 @@ services:
     depends_on:
       - langchain-db
       - langchain-redis
+      - clickhouse-setup
   langchain-queue:
     image: langchain/${_LANGSMITH_IMAGE_PREFIX-}langchainplus-backend:${_LANGSMITH_IMAGE_VERSION:-latest}
     environment:
@@ -67,6 +68,27 @@ services:
       - 63791:6379
     volumes:
       - langchain-redis-data:/data
+  langchain-clickhouse:
+    image: clickhouse/clickhouse-server:23.9
+    user: "101:101"
+    restart: always
+    environment:
+      - CLICKHOUSE_DB=default
+      - CLICKHOUSE_USER=default
+      - CLICKHOUSE_PASSWORD=password
+    volumes:
+      - langchain-clickhouse-data:/var/lib/clickhouse
+      - ./users.xml:/etc/clickhouse-server/users.d/users.xml
+    ports:
+      - 8124:8123
+      - 9001:9000
+  clickhouse-setup:
+    image: langchain/${_LANGSMITH_IMAGE_PREFIX-}langchainplus-backend:${_LANGSMITH_IMAGE_VERSION:-latest}
+    depends_on:
+      - langchain-clickhouse
+    restart: "no"
+    entrypoint: [ "bash", "-c", "sleep 60 && migrate -source file://clickhouse/migrations -database 'clickhouse://langchain-clickhouse:9000?username=default&password=password&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree' up"]
 volumes:
   langchain-db-data:
   langchain-redis-data:
+  langchain-clickhouse-data:

js/src/cli/users.xml

@@ -0,0 +1,10 @@
+<clickhouse>
+    <users>
+        <default>
+            <access_management>1</access_management>
+            <named_collection_control>1</named_collection_control>
+            <show_named_collections>1</show_named_collections>
+            <show_named_collections_secrets>1</show_named_collections_secrets>
+        </default>
+    </users>
+</clickhouse>

js/src/client.ts

@@ -23,9 +23,14 @@ import {
   convertLangChainMessageToExample,
   isLangChainMessage,
 } from "./utils/messages.js";
-import { getEnvironmentVariable, getRuntimeEnvironment } from "./utils/env.js";
+import {
+  getEnvironmentVariable,
+  getLangChainEnvVarsMetadata,
+  getRuntimeEnvironment,
+} from "./utils/env.js";
 
 import { RunEvaluator } from "./evaluation/evaluator.js";
+import { __version__ } from "./index.js";
 
 interface ClientConfig {
   apiUrl?: string;
@@ -98,6 +103,7 @@ interface CreateRunParams {
   child_runs?: RunCreate[];
   parent_run_id?: string;
   project_name?: string;
+  revision_id?: string;
 }
 
 interface projectOptions {
@@ -204,7 +210,7 @@ export class Client {
     const apiKey = getEnvironmentVariable("LANGCHAIN_API_KEY");
     const apiUrl =
       getEnvironmentVariable("LANGCHAIN_ENDPOINT") ??
-      (apiKey ? "https://api.smith.langchain.com" : "http://localhost:1984");
+      "https://api.smith.langchain.com";
     return {
       apiUrl: apiUrl,
       apiKey: apiKey,
@@ -227,7 +233,10 @@ export class Client {
     } else if (isLocalhost(this.apiUrl)) {
       this.webUrl = "http://localhost";
       return "http://localhost";
-    } else if (this.apiUrl.includes("/api")) {
+    } else if (
+      this.apiUrl.includes("/api") &&
+      !this.apiUrl.split(".", 1)[0].endsWith("api")
+    ) {
       this.webUrl = this.apiUrl.replace("/api", "");
       return this.webUrl;
     } else if (this.apiUrl.split(".", 1)[0].includes("dev")) {
@@ -240,7 +249,9 @@ export class Client {
   }
 
   private get headers(): { [header: string]: string } {
-    const headers: { [header: string]: string } = {};
+    const headers: { [header: string]: string } = {
+      "User-Agent": `langsmith-js/${__version__}`,
+    };
     if (this.apiKey) {
       headers["x-api-key"] = `${this.apiKey}`;
     }
@@ -310,14 +321,14 @@ export class Client {
   private async *_getCursorPaginatedList<T>(
     path: string,
     body: Record<string, any> | null = null,
-    requestMethod: string = "POST",
-    dataKey: string = "runs"
+    requestMethod = "POST",
+    dataKey = "runs"
   ): AsyncIterable<T[]> {
-    let bodyParams = body ? { ...body } : {};
+    const bodyParams = body ? { ...body } : {};
     while (true) {
       const response = await this.caller.call(fetch, `${this.apiUrl}${path}`, {
         method: requestMethod,
-        headers: this.headers,
+        headers: { ...this.headers, "Content-Type": "application/json" },
         signal: AbortSignal.timeout(this.timeout_ms),
         body: JSON.stringify(bodyParams),
       });
@@ -343,7 +354,9 @@ export class Client {
   public async createRun(run: CreateRunParams): Promise<void> {
     const headers = { ...this.headers, "Content-Type": "application/json" };
     const extra = run.extra ?? {};
+    const metadata = extra.metadata;
     const runtimeEnv = await getRuntimeEnvironment();
+    const envVars = getLangChainEnvVarsMetadata();
     const session_name = run.project_name;
     delete run.project_name;
     const runCreate: RunCreate = {
@@ -355,6 +368,13 @@ export class Client {
           ...runtimeEnv,
           ...extra.runtime,
         },
+        metadata: {
+          ...envVars,
+          ...(envVars.revision_id || run.revision_id
+            ? { revision_id: run.revision_id ?? envVars.revision_id }
+            : {}),
+          ...metadata,
+        },
       },
     };
     runCreate.inputs = hideInputs(runCreate.inputs);
@@ -512,7 +532,10 @@ export class Client {
       limit,
     };
 
-    for await (const runs of this._getCursorPaginatedList<Run>("/runs", body)) {
+    for await (const runs of this._getCursorPaginatedList<Run>(
+      "/runs/query",
+      body
+    )) {
       yield* runs;
     }
   }
@@ -1280,7 +1303,12 @@ export class Client {
     {
       sourceInfo,
       loadChildRuns,
-    }: { sourceInfo?: KVMap; loadChildRuns: boolean } = { loadChildRuns: false }
+      referenceExample,
+    }: {
+      sourceInfo?: KVMap;
+      loadChildRuns: boolean;
+      referenceExample?: Example;
+    } = { loadChildRuns: false }
   ): Promise<Feedback> {
     let run_: Run;
     if (typeof run === "string") {
@@ -1290,7 +1318,6 @@ export class Client {
     } else {
       throw new Error(`Invalid run type: ${typeof run}`);
     }
-    let referenceExample: Example | undefined = undefined;
     if (
       run_.reference_example_id !== null &&
       run_.reference_example_id !== undefined
@@ -1302,13 +1329,15 @@ export class Client {
     if (feedbackResult.evaluatorInfo) {
       sourceInfo_ = { ...sourceInfo_, ...feedbackResult.evaluatorInfo };
     }
-    return await this.createFeedback(run_.id, feedbackResult.key, {
-      score: feedbackResult.score,
-      value: feedbackResult.value,
-      comment: feedbackResult.comment,
-      correction: feedbackResult.correction,
+    const runId = feedbackResult.targetRunId ?? run_.id;
+    return await this.createFeedback(runId, feedbackResult.key, {
+      score: feedbackResult?.score,
+      value: feedbackResult?.value,
+      comment: feedbackResult?.comment,
+      correction: feedbackResult?.correction,
       sourceInfo: sourceInfo_,
       feedbackSourceType: "model",
+      sourceRunId: feedbackResult?.sourceRunId,
     });
   }
 

js/src/evaluation/evaluator.ts

@@ -1,13 +1,45 @@
-import { Example, KVMap, Run, ScoreType, ValueType } from "../schemas.js";
+import { Example, Run, ScoreType, ValueType } from "../schemas.js";
 
-export interface EvaluationResult {
+/**
+ * Represents the result of an evaluation.
+ */
+export type EvaluationResult = {
+  /**
+   * The key associated with the evaluation result.
+   */
   key: string;
+  /**
+   * The score of the evaluation result.
+   */
   score?: ScoreType;
+  /**
+   * The value of the evaluation result.
+   */
   value?: ValueType;
+  /**
+   * A comment associated with the evaluation result.
+   */
   comment?: string;
-  correction?: object;
-  evaluatorInfo?: KVMap;
-}
+  /**
+   * A correction record associated with the evaluation result.
+   */
+  correction?: Record<string, unknown>;
+  /**
+   * Information about the evaluator.
+   */
+  evaluatorInfo?: Record<string, unknown>;
+  /**
+   * The source run ID of the evaluation result.
+   * If set, a link to the source run will be available in the UI.
+   */
+  sourceRunId?: string;
+  /**
+   * The target run ID of the evaluation result.
+   * If this is not set, the target run ID is assumed to be
+   * the root of the trace.
+   */
+  targetRunId?: string;
+};
 
 export interface RunEvaluator {
   evaluateRun(run: Run, example?: Example): Promise<EvaluationResult>;

js/src/evaluation/string_evaluator.ts

@@ -6,7 +6,7 @@ export interface GradingFunctionResult {
   score?: ScoreType;
   value?: ValueType;
   comment?: string;
-  correction?: object;
+  correction?: Record<string, unknown>;
 }
 
 export interface GradingFunctionParams {

js/src/index.ts

@@ -3,3 +3,6 @@ export { Client } from "./client.js";
 export { Dataset, Example, TracerSession, Run, Feedback } from "./schemas.js";
 
 export { RunTree, RunTreeConfig } from "./run_trees.js";
+
+// Update using yarn bump-version
+export const __version__ = "0.0.61";

js/src/schemas.ts

@@ -200,6 +200,7 @@ export interface Example extends BaseExample {
   id: string;
   created_at: string;
   modified_at: string;
+  source_run_id?: string;
   runs: Run[];
 }