Merge branch 'main' into bagatur/py_exp_log

.github/workflows/build_langsmith_pyo3_wheels.yml

@@ -0,0 +1,16 @@
+name: Build langsmith_pyo3 wheels
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  hello-world:
+    runs-on: ubuntu-20.04
+    steps:
+      - run: echo 'hello world'

.github/workflows/py-bench.yml

@@ -43,7 +43,7 @@ jobs:
         run: |
           {
             echo 'OUTPUT<<EOF'
-            make -s benchmark
+            make -s benchmark-fast
             echo EOF
           } >> "$GITHUB_OUTPUT"
       - name: Compare benchmarks

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.2.4",
+  "version": "0.2.7",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "[email protected]",
   "files": [
@@ -114,7 +114,7 @@
     "@faker-js/faker": "^8.4.1",
     "@jest/globals": "^29.5.0",
     "@langchain/core": "^0.3.14",
-    "@langchain/langgraph": "^0.2.18",
+    "@langchain/langgraph": "^0.2.20",
     "@langchain/openai": "^0.3.11",
     "@opentelemetry/sdk-trace-base": "^1.26.0",
     "@opentelemetry/sdk-trace-node": "^1.26.0",
@@ -133,6 +133,7 @@
     "eslint-plugin-prettier": "^4.2.1",
     "jest": "^29.5.0",
     "langchain": "^0.3.3",
+    "node-fetch": "^2.7.0",
     "openai": "^4.67.3",
     "prettier": "^2.8.8",
     "ts-jest": "^29.1.0",

js/src/client.ts

@@ -838,7 +838,7 @@ export class Client {
             this._serverInfo = await this._getServerInfo();
           } catch (e) {
             console.warn(
-              `[WARNING]: LangSmith failed to fetch info on supported operations. Falling back to single calls and default limits.`
+              `[WARNING]: LangSmith failed to fetch info on supported operations. Falling back to batch operations and default limits.`
             );
           }
         }
@@ -956,22 +956,6 @@ export class Client {
     if (!rawBatch.post.length && !rawBatch.patch.length) {
       return;
     }
-    const serverInfo = await this._ensureServerInfo();
-    if (serverInfo.version === undefined) {
-      this.autoBatchTracing = false;
-      for (const preparedCreateParam of rawBatch.post) {
-        await this.createRun(preparedCreateParam as CreateRunParams);
-      }
-      for (const preparedUpdateParam of rawBatch.patch) {
-        if (preparedUpdateParam.id !== undefined) {
-          await this.updateRun(
-            preparedUpdateParam.id,
-            preparedUpdateParam as UpdateRunParams
-          );
-        }
-      }
-      return;
-    }
     const batchChunks = {
       post: [] as (typeof rawBatch)["post"],
       patch: [] as (typeof rawBatch)["patch"],
@@ -1166,33 +1150,51 @@ export class Client {
 
   private async _sendMultipartRequest(parts: MultipartPart[], context: string) {
     try {
-      const formData = new FormData();
+      // Create multipart form data manually using Blobs
+      const boundary =
+        "----LangSmithFormBoundary" + Math.random().toString(36).slice(2);
+      const chunks: Blob[] = [];
+
       for (const part of parts) {
-        formData.append(part.name, part.payload);
+        // Add field boundary
+        chunks.push(new Blob([`--${boundary}\r\n`]));
+        chunks.push(
+          new Blob([
+            `Content-Disposition: form-data; name="${part.name}"\r\n`,
+            `Content-Type: ${part.payload.type}\r\n\r\n`,
+          ])
+        );
+        chunks.push(part.payload);
+        chunks.push(new Blob(["\r\n"]));
       }
-      // Log the form data
-      await this.batchIngestCaller.call(
+
+      // Add final boundary
+      chunks.push(new Blob([`--${boundary}--\r\n`]));
+
+      // Combine all chunks into a single Blob
+      const body = new Blob(chunks);
+
+      // Convert Blob to ArrayBuffer for compatibility
+      const arrayBuffer = await body.arrayBuffer();
+
+      const res = await this.batchIngestCaller.call(
         _getFetchImplementation(),
         `${this.apiUrl}/runs/multipart`,
         {
           method: "POST",
           headers: {
             ...this.headers,
+            "Content-Type": `multipart/form-data; boundary=${boundary}`,
           },
-          body: formData,
+          body: arrayBuffer,
           signal: AbortSignal.timeout(this.timeout_ms),
           ...this.fetchOptions,
         }
       );
-    } catch (e) {
-      let errorMessage = "Failed to multipart ingest runs";
-      // eslint-disable-next-line no-instanceof/no-instanceof
-      if (e instanceof Error) {
-        errorMessage += `: ${e.stack || e.message}`;
-      } else {
-        errorMessage += `: ${String(e)}`;
-      }
-      console.warn(`${errorMessage.trim()}\n\nContext: ${context}`);
+      await raiseForStatus(res, "ingest multipart runs", true);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (e: any) {
+      console.warn(`${e.message.trim()}\n\nContext: ${context}`);
     }
   }
 

js/src/evaluation/_runner.ts

@@ -18,18 +18,18 @@ import { LangSmithConflictError } from "../utils/error.js";
 import { v4 as uuidv4 } from "uuid";
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-type TargetT<TInput = any, TOutput = KVMap> =
+export type TargetT<TInput = any, TOutput = KVMap> =
   | ((input: TInput, config?: KVMap) => Promise<TOutput>)
   | ((input: TInput, config?: KVMap) => TOutput)
   | { invoke: (input: TInput, config?: KVMap) => TOutput }
   | { invoke: (input: TInput, config?: KVMap) => Promise<TOutput> };
 
 // Data format: dataset-name, dataset_id, or examples
-type DataT = string | AsyncIterable<Example> | Example[];
+export type DataT = string | AsyncIterable<Example> | Example[];
 
 // Summary evaluator runs over the whole dataset
 // and reports aggregate metric(s)
-type SummaryEvaluatorT =
+export type SummaryEvaluatorT =
   | ((
       runs: Array<Run>,
       examples: Array<Example>
@@ -40,7 +40,7 @@ type SummaryEvaluatorT =
     ) => EvaluationResult | EvaluationResults);
 
 // Row-level evaluator
-type EvaluatorT =
+export type EvaluatorT =
   | RunEvaluator
   | ((run: Run, example?: Example) => EvaluationResult | EvaluationResults)
   | ((
@@ -128,7 +128,7 @@ export function evaluate(
   return _evaluate(target, options);
 }
 
-interface ExperimentResultRow {
+export interface ExperimentResultRow {
   run: Run;
   example: Example;
   evaluationResults: EvaluationResults;

js/src/evaluation/index.ts

@@ -5,5 +5,13 @@ export {
   GradingFunctionParams,
   GradingFunctionResult,
 } from "./string_evaluator.js";
-export { evaluate, type EvaluateOptions } from "./_runner.js";
+export {
+  evaluate,
+  type EvaluateOptions,
+  type TargetT,
+  type DataT,
+  type SummaryEvaluatorT,
+  type EvaluatorT,
+  type ExperimentResultRow,
+} from "./_runner.js";
 export { evaluateComparative } from "./evaluate_comparative.js";

js/src/index.ts

@@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.2.4";
+export const __version__ = "0.2.7";

js/src/tests/batch_client.int.test.ts

@@ -2,6 +2,7 @@ import { v4 as uuidv4 } from "uuid";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { fileURLToPath } from "node:url";
+import nodeFetch from "node-fetch";
 
 import { Client } from "../client.js";
 import { RunTree, convertToDottedOrderFormat } from "../run_trees.js";
@@ -11,6 +12,7 @@ import {
   waitUntilRunFound,
 } from "./utils.js";
 import { traceable } from "../traceable.js";
+import { overrideFetchImplementation } from "../singletons/fetch.js";
 
 test.concurrent(
   "Test persist update run",
@@ -229,6 +231,7 @@ test.concurrent(
       outputs: { output: ["Hi"] },
       dotted_order: dottedOrder,
       trace_id: runId,
+      end_time: Math.floor(new Date().getTime() / 1000),
     });
 
     await Promise.all([
@@ -282,3 +285,32 @@ test.skip("very large runs", async () => {
 
   await langchainClient.deleteProject({ projectName });
 }, 180_000);
+
+test("multipart should work with overridden node-fetch", async () => {
+  overrideFetchImplementation(nodeFetch);
+
+  const langchainClient = new Client({
+    autoBatchTracing: true,
+    timeout_ms: 120_000,
+  });
+
+  const projectName = "__test_node_fetch" + uuidv4().substring(0, 4);
+  await deleteProject(langchainClient, projectName);
+
+  await traceable(
+    async () => {
+      return "testing with node fetch";
+    },
+    {
+      project_name: projectName,
+      client: langchainClient,
+      tracingEnabled: true,
+    }
+  )();
+
+  await langchainClient.awaitPendingTraceBatches();
+
+  await Promise.all([waitUntilProjectFound(langchainClient, projectName)]);
+
+  await langchainClient.deleteProject({ projectName });
+});