diff --git a/js/.gitignore b/js/.gitignore index e758389d2..4b11d6959 100644 --- a/js/.gitignore +++ b/js/.gitignore @@ -59,6 +59,10 @@ Chinook_Sqlite.sql /langchain.js /langchain.d.ts /langchain.d.cts +/vercel.cjs +/vercel.js +/vercel.d.ts +/vercel.d.cts /wrappers.cjs /wrappers.js /wrappers.d.ts diff --git a/js/package.json b/js/package.json index 488285d6a..53d29968c 100644 --- a/js/package.json +++ b/js/package.json @@ -33,6 +33,10 @@ "langchain.js", "langchain.d.ts", "langchain.d.cts", + "vercel.cjs", + "vercel.js", + "vercel.d.ts", + "vercel.d.cts", "wrappers.cjs", "wrappers.js", "wrappers.d.ts", @@ -223,6 +227,15 @@ "import": "./langchain.js", "require": "./langchain.cjs" }, + "./vercel": { + "types": { + "import": "./vercel.d.ts", + "require": "./vercel.d.cts", + "default": "./vercel.d.ts" + }, + "import": "./vercel.js", + "require": "./vercel.cjs" + }, "./wrappers": { "types": { "import": "./wrappers.d.ts", diff --git a/js/scripts/create-entrypoints.js b/js/scripts/create-entrypoints.js index a3487f756..9cce2ab22 100644 --- a/js/scripts/create-entrypoints.js +++ b/js/scripts/create-entrypoints.js @@ -14,6 +14,7 @@ const entrypoints = { "evaluation/langchain": "evaluation/langchain", schemas: "schemas", langchain: "langchain", + vercel: "vercel", wrappers: "wrappers/index", anonymizer: "anonymizer/index", "wrappers/openai": "wrappers/openai", diff --git a/js/src/tests/utils/iterator.ts b/js/src/tests/utils/iterator.ts deleted file mode 100644 index 4734369ba..000000000 --- a/js/src/tests/utils/iterator.ts +++ /dev/null @@ -1,7 +0,0 @@ -export async function gatherIterator( - i: AsyncIterable | Promise> -): Promise> { - const out: T[] = []; - for await (const item of await i) out.push(item); - return out; -} diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel.int.test.ts similarity index 67% rename from js/src/tests/vercel_exporter.int.test.ts rename to js/src/tests/vercel.int.test.ts index f3f6616ca..8741ad4b4 100644 --- a/js/src/tests/vercel_exporter.int.test.ts +++ b/js/src/tests/vercel.int.test.ts @@ -1,42 +1,33 @@ -import { openai } from "@ai-sdk/openai"; - import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; -import { z } from "zod"; -import { LangSmithAISDKExporter } from "../wrappers/vercel.js"; + +import { + generateText, + streamText, + generateObject, + streamObject, + tool, +} from "ai"; +import { openai } from "@ai-sdk/openai"; + import { v4 as uuid } from "uuid"; -import { generateText, streamText, generateObject, streamObject } from "ai"; -import { tool } from "ai"; -import { gatherIterator } from "./utils/iterator.js"; +import { z } from "zod"; +import { AISDKExporter } from "../vercel.js"; import { Client } from "../index.js"; -import { waitUntilRunFound } from "./utils.js"; -import { getCurrentRunTree, traceable } from "../traceable.js"; - -const getTelemetrySettings = (runId?: string) => { - const metadata: Record = { - userId: "123", - language: "english", - }; - - if (runId) metadata["langsmith:runId"] = runId; - return { - isEnabled: true, - functionId: "functionId", - metadata, - }; -}; +import { traceable } from "../traceable.js"; +import { waitUntilRunFound, toArray } from "./utils.js"; const client = new Client(); // Not using @opentelemetry/sdk-node because we need to force flush // the spans to ensure they are sent to LangSmith between tests const provider = new NodeTracerProvider(); provider.addSpanProcessor( - new BatchSpanProcessor(new LangSmithAISDKExporter({ client })) + new BatchSpanProcessor(new AISDKExporter({ client })) ); provider.register(); test("generateText", async () => { - const traceId = uuid(); + const runId = uuid(); await generateText({ model: openai("gpt-4o-mini"), @@ -60,19 +51,23 @@ test("generateText", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), maxSteps: 10, }); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("generateText with image", async () => { - const traceId = uuid(); + const runId = uuid(); await generateText({ model: openai("gpt-4o-mini"), messages: [ @@ -90,18 +85,22 @@ test("generateText with image", async () => { ], }, ], - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), }); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("streamText", async () => { - const traceId = uuid(); + const runId = uuid(); const result = await streamText({ model: openai("gpt-4o-mini"), messages: [ @@ -124,20 +123,24 @@ test("streamText", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), maxSteps: 10, }); - await gatherIterator(result.fullStream); + await toArray(result.fullStream); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("generateObject", async () => { - const traceId = uuid(); + const runId = uuid(); await generateObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -147,18 +150,22 @@ test("generateObject", async () => { }), }), prompt: "What's the weather in Prague?", - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), }); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("streamObject", async () => { - const traceId = uuid(); + const runId = uuid(); const result = await streamObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -168,15 +175,19 @@ test("streamObject", async () => { }), }), prompt: "What's the weather in Prague?", - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), }); - await gatherIterator(result.partialObjectStream); + await toArray(result.partialObjectStream); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("traceable", async () => { @@ -184,11 +195,6 @@ test("traceable", async () => { const wrappedText = traceable( async (content: string) => { - const runTree = getCurrentRunTree(); - const headers = runTree.toHeaders(); - - const telemetry = getTelemetrySettings(); - const { text } = await generateText({ model: openai("gpt-4o-mini"), messages: [{ role: "user", content }], @@ -206,14 +212,10 @@ test("traceable", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: { - ...telemetry, - metadata: { - ...telemetry.metadata, - "langsmith:trace": headers["langsmith-trace"], - "langsmith:baggage": headers["baggage"], - }, - }, + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), maxSteps: 10, }); diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/vercel.ts similarity index 87% rename from js/src/wrappers/vercel/exporter.ts rename to js/src/vercel.ts index bb75fb6d6..e85db5ee7 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/vercel.ts @@ -1,12 +1,28 @@ -import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; -import type { AISDKSpan } from "./exporter.types.js"; -import { Client, RunTree } from "../../index.js"; -import { KVMap, RunCreate } from "../../schemas.js"; +import type { + CoreAssistantMessage, + CoreMessage, + ToolCallPart, + generateText, +} from "ai"; +import type { AISDKSpan } from "./vercel.types.js"; +import { Client, RunTree } from "./index.js"; +import { KVMap, RunCreate } from "./schemas.js"; import { v5 as uuid5 } from "uuid"; +import { getCurrentRunTree } from "./singletons/traceable.js"; // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; +type AITelemetrySettings = Exclude< + Parameters[0]["experimental_telemetry"], + undefined +>; + +interface TelemetrySettings extends AITelemetrySettings { + /** ID of the run sent to LangSmith */ + runId?: string; +} + type LangChainMessageFields = { content: | string @@ -201,14 +217,25 @@ function convertToTimestamp([seconds, nanoseconds]: [ const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47"; -const RUN_ID_METADATA_KEY = "ai.telemetry.metadata.langsmith:runId"; -const TRACE_METADATA_KEY = "ai.telemetry.metadata.langsmith:trace"; -const BAGGAGE_METADATA_KEY = "ai.telemetry.metadata.langsmith:baggage"; +const RUN_ID_METADATA_KEY = { + input: "langsmith:runId", + output: "ai.telemetry.metadata.langsmith:runId", +}; + +const TRACE_METADATA_KEY = { + input: "langsmith:trace", + output: "ai.telemetry.metadata.langsmith:trace", +}; + +const BAGGAGE_METADATA_KEY = { + input: "langsmith:baggage", + output: "ai.telemetry.metadata.langsmith:baggage", +}; const RESERVED_METADATA_KEYS = [ - RUN_ID_METADATA_KEY, - TRACE_METADATA_KEY, - BAGGAGE_METADATA_KEY, + RUN_ID_METADATA_KEY.output, + TRACE_METADATA_KEY.output, + BAGGAGE_METADATA_KEY.output, ]; interface RunTask { @@ -225,7 +252,7 @@ type InteropType = | { type: "manual"; userTraceId: string } | undefined; -export class LangSmithAISDKExporter { +export class AISDKExporter { private client: Client; private traceByMap: Record< string, @@ -241,38 +268,62 @@ export class LangSmithAISDKExporter { this.client = args?.client ?? new Client(); } - /** @internal */ - protected parseInteropFromMetadata(span: AISDKSpan): InteropType { - let userTraceId: string | undefined = undefined; + static getSettings(settings: TelemetrySettings) { + const { runId, ...rest } = settings; + const metadata = { ...rest?.metadata }; + if (runId != null) metadata[RUN_ID_METADATA_KEY.input] = runId; + + // attempt to obtain the run tree if used within a traceable function + let defaultEnabled = true; + try { + const runTree = getCurrentRunTree(); + const headers = runTree.toHeaders(); + metadata[TRACE_METADATA_KEY.input] = headers["langsmith-trace"]; + metadata[BAGGAGE_METADATA_KEY.input] = headers["baggage"]; + + // honor the tracingEnabled flag if coming from traceable + if (runTree.tracingEnabled != null) { + defaultEnabled = runTree.tracingEnabled; + } + } catch { + // pass + } if ( - RUN_ID_METADATA_KEY in span.attributes && - typeof span.attributes[RUN_ID_METADATA_KEY] === "string" && - span.attributes[RUN_ID_METADATA_KEY] + metadata[RUN_ID_METADATA_KEY.input] && + metadata[TRACE_METADATA_KEY.input] ) { - userTraceId = span.attributes[RUN_ID_METADATA_KEY]; + throw new Error( + "Cannot provide `runId` when used within traceable function." + ); } - if ( - TRACE_METADATA_KEY in span.attributes && - typeof span.attributes[TRACE_METADATA_KEY] === "string" && - span.attributes[TRACE_METADATA_KEY] - ) { - if (userTraceId) { - throw new Error( - "Cannot provide both `langsmith:runId` and `langsmith:trace` metadata keys." - ); - } + return { ...rest, isEnabled: rest.isEnabled ?? defaultEnabled, metadata }; + } + + /** @internal */ + protected parseInteropFromMetadata(span: AISDKSpan): InteropType { + const getKey = (key: string): string | undefined => { + const attributes = span.attributes as Record; - const baggage = - BAGGAGE_METADATA_KEY in span.attributes && - typeof span.attributes[BAGGAGE_METADATA_KEY] === "string" - ? span.attributes[BAGGAGE_METADATA_KEY] - : ""; + return key in attributes && typeof attributes[key] === "string" + ? (attributes[key] as string) + : undefined; + }; + + const userTraceId = getKey(RUN_ID_METADATA_KEY.output) || undefined; + const parentTrace = getKey(TRACE_METADATA_KEY.output) || undefined; + + if (parentTrace && userTraceId) { + throw new Error( + `Cannot provide both "${RUN_ID_METADATA_KEY.input}" and "${TRACE_METADATA_KEY.input}" metadata keys.` + ); + } + if (parentTrace) { const parentRunTree = RunTree.fromHeaders({ - "langsmith-trace": span.attributes[TRACE_METADATA_KEY], - baggage, + "langsmith-trace": parentTrace, + baggage: getKey(BAGGAGE_METADATA_KEY.output) || "", }); if (!parentRunTree) diff --git a/js/src/wrappers/vercel/exporter.types.ts b/js/src/vercel.types.ts similarity index 100% rename from js/src/wrappers/vercel/exporter.types.ts rename to js/src/vercel.types.ts diff --git a/js/src/wrappers/vercel.ts b/js/src/wrappers/vercel.ts index e143647a4..dc022d7c8 100644 --- a/js/src/wrappers/vercel.ts +++ b/js/src/wrappers/vercel.ts @@ -107,5 +107,3 @@ export const wrapAISDKModel = ( }, }); }; - -export { LangSmithAISDKExporter } from "./vercel/exporter.js"; diff --git a/js/tsconfig.json b/js/tsconfig.json index a58d12a7b..b778ed83f 100644 --- a/js/tsconfig.json +++ b/js/tsconfig.json @@ -40,6 +40,7 @@ "src/evaluation/langchain.ts", "src/schemas.ts", "src/langchain.ts", + "src/vercel.ts", "src/wrappers/index.ts", "src/anonymizer/index.ts", "src/wrappers/openai.ts",