From d420b3696243f6e75b7843e800b69a563a8be647 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Thu, 17 Oct 2024 19:32:36 +0200 Subject: [PATCH 01/36] feat(vercel): add OTEL based LangSmith trace exporter --- js/package.json | 1 + js/src/wrappers/vercel.ts | 2 + js/src/wrappers/vercel/exporter.ts | 414 +++++++++++++++++++++++ js/src/wrappers/vercel/exporter.types.ts | 231 +++++++++++++ js/yarn.lock | 29 ++ 5 files changed, 677 insertions(+) create mode 100644 js/src/wrappers/vercel/exporter.ts create mode 100644 js/src/wrappers/vercel/exporter.types.ts diff --git a/js/package.json b/js/package.json index fde21f5b0..4b67df723 100644 --- a/js/package.json +++ b/js/package.json @@ -112,6 +112,7 @@ "@langchain/core": "^0.3.1", "@langchain/langgraph": "^0.2.3", "@langchain/openai": "^0.3.0", + "@opentelemetry/sdk-trace-base": "^1.26.0", "@tsconfig/recommended": "^1.0.2", "@types/jest": "^29.5.1", "@typescript-eslint/eslint-plugin": "^5.59.8", diff --git a/js/src/wrappers/vercel.ts b/js/src/wrappers/vercel.ts index dc022d7c8..e143647a4 100644 --- a/js/src/wrappers/vercel.ts +++ b/js/src/wrappers/vercel.ts @@ -107,3 +107,5 @@ export const wrapAISDKModel = ( }, }); }; + +export { LangSmithAISDKExporter } from "./vercel/exporter.js"; diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts new file mode 100644 index 000000000..9e414b242 --- /dev/null +++ b/js/src/wrappers/vercel/exporter.ts @@ -0,0 +1,414 @@ +import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; +import type { ExportResult } from "@opentelemetry/core"; +import type { CoreAssistantMessage, CoreMessage } from "ai"; +import type { AISDKSpan } from "./exporter.types.js"; +import type { + BaseMessageFields, + MessageContentText, +} from "@langchain/core/messages"; +import { Client, ClientConfig, RunTree, RunTreeConfig } from "../../index.js"; +import { KVMap } from "../../schemas.js"; +import { AsyncLocalStorageProviderSingleton } from "../../singletons/traceable.js"; + +function assertNever(x: never): never { + throw new Error("Unreachable state: " + x); +} + +// TODO: remove dependency on @langchain/core +type $SmithMessage = { type: string; data: BaseMessageFields } | CoreMessage; + +function convertCoreToSmith( + message: CoreMessage +): $SmithMessage | $SmithMessage[] { + if (message.role === "assistant") { + const data: BaseMessageFields = { content: message.content }; + + if (Array.isArray(message.content)) { + data.content = message.content.map((part) => { + if (part.type === "text") { + return { + type: "text", + text: part.text, + } satisfies MessageContentText; + } + + if (part.type === "tool-call") { + return { + type: "tool_use", + name: part.toolName, + id: part.toolCallId, + input: part.args, + }; + } + + return part; + }); + + const toolCalls = message.content.filter( + (part) => part.type === "tool-call" + ); + + if (toolCalls.length > 0) { + data.additional_kwargs ??= {}; + data.additional_kwargs.tool_calls = toolCalls.map((part) => { + return { + id: part.toolCallId, + type: "function", + function: { + name: part.toolName, + id: part.toolCallId, + arguments: JSON.stringify(part.args), + }, + }; + }); + } + } + + return { type: "ai", data }; + } + + if (message.role === "user") { + // TODO: verify user content + return { type: "human", data: { content: message.content } }; + } + + if (message.role === "system") { + return { type: "system", data: { content: message.content } }; + } + + if (message.role === "tool") { + const res = message.content.map((toolCall) => { + return { + type: "tool", + data: { + content: JSON.stringify(toolCall.result), + name: toolCall.toolName, + tool_call_id: toolCall.toolCallId, + }, + }; + }); + if (res.length === 1) return res[0]; + return res; + } + + return message as any; +} + +const tryJson = ( + str: + | string + | number + | boolean + | Array + | Array + | Array + | undefined +) => { + try { + if (!str) return str; + if (typeof str !== "string") return str; + return JSON.parse(str); + } catch { + return str; + } +}; + +const sortByHrTime = (a: ReadableSpan, b: ReadableSpan) => { + return ( + Math.sign(a.startTime[0] - b.startTime[0]) || + Math.sign(a.startTime[1] - b.startTime[1]) + ); +}; + +export class LangSmithAISDKExporter implements SpanExporter { + private client: Client; + + constructor(config?: ClientConfig) { + this.client = new Client(config); + } + + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void + ): void { + const runTreeMap: Record = {}; + const sortedSpans = [...spans].sort(sortByHrTime) as AISDKSpan[]; + + for (const span of sortedSpans) { + const spanId = span.spanContext().spanId; + const parentSpanId = span.parentSpanId; + let parentRunTree = parentSpanId ? runTreeMap[parentSpanId] : null; + + if (parentRunTree == null) { + try { + parentRunTree = + AsyncLocalStorageProviderSingleton.getInstance().getStore() ?? null; + } catch { + // pass + } + } + + const toRunTree = (rawConfig: RunTreeConfig) => { + const aiMetadata = Object.keys(span.attributes) + .filter((key) => key.startsWith("ai.telemetry.metadata.")) + .reduce((acc, key) => { + acc[key.slice("ai.telemetry.metadata.".length)] = + span.attributes[key as keyof typeof span.attributes]; + + return acc; + }, {} as Record); + + if ( + ("ai.telemetry.functionId" in span.attributes && + span.attributes["ai.telemetry.functionId"]) || + ("resource.name" in span.attributes && + span.attributes["resource.name"]) + ) { + aiMetadata["functionId"] = + span.attributes["ai.telemetry.functionId"] || + span.attributes["resource.name"]; + } + + const config: RunTreeConfig = { + ...rawConfig, + metadata: { + ...rawConfig.metadata, + ...aiMetadata, + "ai.operationId": span.attributes["ai.operationId"], + }, + start_time: +( + String(span.startTime[0]) + String(span.startTime[1]).slice(0, 3) + ), + end_time: +( + String(span.endTime[0]) + String(span.endTime[1]).slice(0, 3) + ), + client: this.client, + }; + return parentRunTree?.createChild(config) ?? new RunTree(config); + }; + + switch (span.name) { + case "ai.generateText.doGenerate": + case "ai.generateText": + case "ai.streamText.doStream": + case "ai.streamText": { + const inputs = ((): KVMap | undefined => { + if ("ai.prompt.messages" in span.attributes) { + return { + messages: tryJson( + span.attributes["ai.prompt.messages"] + ).flatMap((i: CoreMessage) => convertCoreToSmith(i)), + }; + } + + if ("ai.prompt" in span.attributes) { + const input = tryJson(span.attributes["ai.prompt"]); + + if ( + typeof input === "object" && + input != null && + "messages" in input && + Array.isArray(input.messages) + ) { + return { + messages: input.messages.flatMap((i: CoreMessage) => + convertCoreToSmith(i) + ), + }; + } + + return { input }; + } + + return undefined; + })(); + + const outputs = ((): KVMap | undefined => { + let result: KVMap | undefined = undefined; + if (span.attributes["ai.response.toolCalls"]) { + result = { + llm_output: convertCoreToSmith({ + role: "assistant", + content: tryJson(span.attributes["ai.response.toolCalls"]), + } satisfies CoreAssistantMessage), + }; + } else if (span.attributes["ai.response.text"]) { + result = { + llm_output: convertCoreToSmith({ + role: "assistant", + content: span.attributes["ai.response.text"], + }), + }; + } + + if (span.attributes["ai.usage.completionTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["completion_tokens"] = + span.attributes["ai.usage.completionTokens"]; + } + + if (span.attributes["ai.usage.promptTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["prompt_tokens"] = + span.attributes["ai.usage.promptTokens"]; + } + + return result; + })(); + + // TODO: add first_token_time + runTreeMap[spanId] = toRunTree({ + run_type: "llm", + name: span.attributes["ai.model.provider"], + inputs, + outputs, + metadata: { + ls_provider: span.attributes["ai.model.provider"] + .split(".") + .at(0), + ls_model_type: span.attributes["ai.model.provider"] + .split(".") + .at(1), + ls_model_name: span.attributes["ai.model.id"], + }, + extra: { batch_size: 1 }, + }); + break; + } + + case "ai.toolCall": { + const args = tryJson(span.attributes["ai.toolCall.args"]); + let inputs: KVMap | undefined = { args }; + + if (typeof args === "object" && args != null) { + inputs = args; + } + + const output = tryJson(span.attributes["ai.toolCall.result"]); + let outputs: KVMap | undefined = { output }; + + if (typeof output === "object" && output != null) { + outputs = output; + } + + runTreeMap[spanId] = toRunTree({ + run_type: "tool", + name: span.attributes["ai.toolCall.name"], + inputs, + outputs, + }); + break; + } + + case "ai.streamObject": + case "ai.streamObject.doStream": + case "ai.generateObject": + case "ai.generateObject.doGenerate": { + const inputs = ((): KVMap | undefined => { + if ("ai.prompt.messages" in span.attributes) { + return { + messages: tryJson( + span.attributes["ai.prompt.messages"] + ).flatMap((i: CoreMessage) => convertCoreToSmith(i)), + }; + } + + if ("ai.prompt" in span.attributes) { + return { input: span.attributes["ai.prompt"] }; + } + + return undefined; + })(); + + const outputs = ((): KVMap | undefined => { + let result: KVMap | undefined = undefined; + + if (span.attributes["ai.response.object"]) { + result = { + output: tryJson(span.attributes["ai.response.object"]), + }; + } + + if (span.attributes["ai.usage.completionTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["completion_tokens"] = + span.attributes["ai.usage.completionTokens"]; + } + + if (span.attributes["ai.usage.promptTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["prompt_tokens"] = + +span.attributes["ai.usage.promptTokens"]; + } + + return result; + })(); + + runTreeMap[spanId] = toRunTree({ + run_type: "llm", + name: span.attributes["ai.model.provider"], + inputs, + outputs, + metadata: { + ls_provider: span.attributes["ai.model.provider"] + .split(".") + .at(0), + ls_model_type: span.attributes["ai.model.provider"] + .split(".") + .at(1), + ls_model_name: span.attributes["ai.model.id"], + }, + extra: { batch_size: 1 }, + }); + break; + } + + case "ai.embed": { + runTreeMap[spanId] = toRunTree({ + run_type: "chain", + name: span.attributes["ai.model.provider"], + inputs: { value: span.attributes["ai.value"] }, + outputs: { embedding: span.attributes["ai.embedding"] }, + }); + break; + } + case "ai.embed.doEmbed": + case "ai.embedMany": + case "ai.embedMany.doEmbed": { + runTreeMap[spanId] = toRunTree({ + run_type: "chain", + name: span.attributes["ai.model.provider"], + inputs: { values: span.attributes["ai.values"] }, + outputs: { embeddings: span.attributes["ai.embeddings"] }, + }); + break; + } + + default: + assertNever(span); + } + } + + Promise.all( + Object.values(runTreeMap).map((runTree) => runTree.postRun()) + ).then( + () => resultCallback({ code: 0 }), + (error) => resultCallback({ code: 1, error }) + ); + } + + async shutdown(): Promise { + // pass + } + async forceFlush?(): Promise { + // pass + } +} diff --git a/js/src/wrappers/vercel/exporter.types.ts b/js/src/wrappers/vercel/exporter.types.ts new file mode 100644 index 000000000..8e3d2a32b --- /dev/null +++ b/js/src/wrappers/vercel/exporter.types.ts @@ -0,0 +1,231 @@ +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; + +// eslint-disable-next-line @typescript-eslint/ban-types +type AnyString = string & {}; + +interface TypedReadableSpan + extends Omit { + name: Name; + attributes: Attributes; +} + +interface BaseLLMSpanAttributes { + "ai.model.id": string; + "ai.model.provider": string; + + "ai.usage.promptTokens": number; + "ai.usage.completionTokens": number; + + "ai.telemetry.functionId"?: string; + "resource.name"?: string; +} + +interface CallLLMSpanAttributes extends BaseLLMSpanAttributes { + "ai.response.model": string; + "ai.response.id": string; + "ai.response.timestamp": number; +} + +interface BaseEmbedSpanAttributes { + "ai.model.id": string; + "ai.model.provider": string; + "ai.usage.tokens": number; + + "ai.telemetry.functionId"?: string; + "resource.name"?: string; +} + +export type ToolCallSpan = TypedReadableSpan< + "ai.toolCall", + { + "operation.name": "ai.toolCall"; + "ai.operationId": "ai.toolCall"; + "ai.toolCall.name": string; + "ai.toolCall.id": string; + "ai.toolCall.args": string; + "ai.toolCall.result"?: string; + } +>; + +export type GenerateTextSpan = TypedReadableSpan< + "ai.generateText", + BaseLLMSpanAttributes & { + "operation.name": "ai.generateText"; + "ai.operationId": "ai.generateText"; + "ai.prompt": string; + "ai.response.text": string; + "ai.response.toolCalls": string; + "ai.response.finishReason": string; + "ai.settings.maxSteps": number; + } +>; + +export type DoGenerateTextSpan = TypedReadableSpan< + "ai.generateText.doGenerate", + CallLLMSpanAttributes & { + "operation.name": "ai.generateText.doGenerate"; + "ai.operationId": "ai.generateText.doGenerate"; + "ai.prompt.format": string; + "ai.prompt.messages": string; + "ai.response.text": string; + "ai.response.toolCalls": string; + "ai.response.finishReason": string; + } +>; + +export type StreamTextSpan = TypedReadableSpan< + "ai.streamText", + BaseLLMSpanAttributes & { + "operation.name": "ai.streamText"; + "ai.operationId": "ai.streamText"; + "ai.prompt": string; + "ai.response.text": string; + "ai.response.toolCalls": string; + "ai.response.finishReason": string; + "ai.settings.maxSteps": number; + } +>; + +export type DoStreamTextSpan = TypedReadableSpan< + "ai.streamText.doStream", + CallLLMSpanAttributes & { + "operation.name": "ai.streamText.doStream"; + "ai.operationId": "ai.streamText.doStream"; + "ai.prompt.format": string; + "ai.prompt.messages": string; + "ai.response.text": string; + "ai.response.toolCalls": string; + "ai.response.msToFirstChunk": number; + "ai.response.msToFinish": number; + "ai.response.avgCompletionTokensPerSecond": number; + "ai.response.finishReason": string; + } +>; + +export type GenerateObjectSpan = TypedReadableSpan< + "ai.generateObject", + BaseLLMSpanAttributes & { + "operation.name": "ai.generateObject"; + "ai.operationId": "ai.generateObject"; + "ai.prompt": string; + + "ai.schema": string; + "ai.schema.name": string; + "ai.schema.description": string; + + "ai.response.object": string; + + "ai.settings.mode": "json" | AnyString; + "ai.settings.output": "object" | "no-schema" | AnyString; + } +>; +export type DoGenerateObjectSpan = TypedReadableSpan< + "ai.generateObject.doGenerate", + CallLLMSpanAttributes & { + "operation.name": "ai.generateObject.doGenerate"; + "ai.operationId": "ai.generateObject.doGenerate"; + + "ai.prompt.format": string; + "ai.prompt.messages": string; + + "ai.response.object": string; + "ai.response.finishReason": string; + + "ai.settings.mode": "json" | AnyString; + "ai.settings.output": "object" | "no-schema" | AnyString; + } +>; + +export type StreamObjectSpan = TypedReadableSpan< + "ai.streamObject", + BaseLLMSpanAttributes & { + "operation.name": "ai.streamObject"; + "ai.operationId": "ai.streamObject"; + "ai.prompt": string; + + "ai.schema": string; + "ai.schema.name": string; + "ai.schema.description": string; + + "ai.response.object": string; + + "ai.settings.mode": "json" | AnyString; + "ai.settings.output": "object" | "no-schema" | AnyString; + } +>; +export type DoStreamObjectSpan = TypedReadableSpan< + "ai.streamObject.doStream", + CallLLMSpanAttributes & { + "operation.name": "ai.streamObject.doStream"; + "ai.operationId": "ai.streamObject.doStream"; + + "ai.prompt.format": string; + "ai.prompt.messages": string; + + "ai.response.object": string; + "ai.response.finishReason": string; + "ai.response.msToFirstChunk": number; + + "ai.settings.mode": "json" | AnyString; + } +>; + +export type EmbedSpan = TypedReadableSpan< + "ai.embed", + BaseEmbedSpanAttributes & { + "operation.name": "ai.embed"; + "ai.operationId": "ai.embed"; + + // TODO: is this correct? + "ai.value": string; + "ai.embedding": string; + } +>; + +export type DoEmbedSpan = TypedReadableSpan< + "ai.embed.doEmbed", + BaseEmbedSpanAttributes & { + "operation.name": "ai.embed.doEmbed"; + "ai.operationId": "ai.embed.doEmbed"; + + "ai.values": string[]; + "ai.embeddings": string[]; + } +>; + +export type EmbedManySpan = TypedReadableSpan< + "ai.embedMany", + BaseEmbedSpanAttributes & { + "operation.name": "ai.embedMany"; + "ai.operationId": "ai.embedMany"; + + "ai.values": string[]; + "ai.embeddings": string[]; + } +>; + +export type DoEmbedManySpan = TypedReadableSpan< + "ai.embedMany.doEmbed", + BaseEmbedSpanAttributes & { + "operation.name": "ai.embedMany.doEmbed"; + "ai.operationId": "ai.embedMany.doEmbed"; + + "ai.values": string[]; + "ai.embeddings": string[]; + } +>; + +export type AISDKSpan = + | ToolCallSpan + | GenerateTextSpan + | DoGenerateTextSpan + | StreamTextSpan + | DoStreamTextSpan + | GenerateObjectSpan + | DoGenerateObjectSpan + | StreamObjectSpan + | DoStreamObjectSpan + | EmbedSpan + | DoEmbedSpan + | EmbedManySpan + | DoEmbedManySpan; diff --git a/js/yarn.lock b/js/yarn.lock index 2a3feae33..531ebf491 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1445,6 +1445,35 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== +"@opentelemetry/core@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.26.0.tgz#7d84265aaa850ed0ca5813f97d831155be42b328" + integrity sha512-1iKxXXE8415Cdv0yjG3G6hQnB5eVEsJce3QaawX8SjDn0mAS0ZM8fAbZZJD4ajvhC15cePvosSCut404KrIIvQ== + dependencies: + "@opentelemetry/semantic-conventions" "1.27.0" + +"@opentelemetry/resources@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.26.0.tgz#da4c7366018bd8add1f3aa9c91c6ac59fd503cef" + integrity sha512-CPNYchBE7MBecCSVy0HKpUISEeJOniWqcHaAHpmasZ3j9o6V3AyBzhRc90jdmemq0HOxDr6ylhUbDhBqqPpeNw== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/semantic-conventions" "1.27.0" + +"@opentelemetry/sdk-trace-base@^1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.26.0.tgz#0c913bc6d2cfafd901de330e4540952269ae579c" + integrity sha512-olWQldtvbK4v22ymrKLbIcBi9L2SpMO84sCPY54IVsJhP9fRsxJT194C/AVaAuJzLE30EdhhM1VmvVYR7az+cw== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/semantic-conventions" "1.27.0" + +"@opentelemetry/semantic-conventions@1.27.0": + version "1.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz#1a857dcc95a5ab30122e04417148211e6f945e6c" + integrity sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg== + "@sinclair/typebox@^0.25.16": version "0.25.24" resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz" From 59a6ed5537ee63eb04bd0245005df562f26dabbe Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Thu, 17 Oct 2024 19:37:32 +0200 Subject: [PATCH 02/36] Fix build --- js/src/wrappers/vercel/exporter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 9e414b242..db21bb3f6 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -1,6 +1,6 @@ import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; import type { ExportResult } from "@opentelemetry/core"; -import type { CoreAssistantMessage, CoreMessage } from "ai"; +import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; import type { AISDKSpan } from "./exporter.types.js"; import type { BaseMessageFields, @@ -45,7 +45,7 @@ function convertCoreToSmith( }); const toolCalls = message.content.filter( - (part) => part.type === "tool-call" + (part): part is ToolCallPart => part.type === "tool-call" ); if (toolCalls.length > 0) { From 228aea2b79c62031a3b133f6c19019bb2bcc5fcc Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 21 Oct 2024 22:50:32 +0200 Subject: [PATCH 03/36] Update eslint rule to permit underscore prefix unused variables --- js/.eslintrc.cjs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/js/.eslintrc.cjs b/js/.eslintrc.cjs index da4c3ecb4..f54109db0 100644 --- a/js/.eslintrc.cjs +++ b/js/.eslintrc.cjs @@ -30,7 +30,18 @@ module.exports = { "@typescript-eslint/no-shadow": 0, "@typescript-eslint/no-empty-interface": 0, "@typescript-eslint/no-use-before-define": ["error", "nofunc"], - "@typescript-eslint/no-unused-vars": ["warn", { args: "none" }], + "@typescript-eslint/no-unused-vars": [ + "warn", + { + args: "none", + argsIgnorePattern: "^_", + caughtErrors: "all", + caughtErrorsIgnorePattern: "^_", + destructuredArrayIgnorePattern: "^_", + varsIgnorePattern: "^_", + ignoreRestSiblings: true, + }, + ], "@typescript-eslint/no-floating-promises": "error", "@typescript-eslint/no-misused-promises": "error", camelcase: 0, From 3f027b88818281689a2c00b9234244d4a0319852 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 21 Oct 2024 22:50:48 +0200 Subject: [PATCH 04/36] Update packages --- js/package.json | 5 +- js/yarn.lock | 524 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 460 insertions(+), 69 deletions(-) diff --git a/js/package.json b/js/package.json index 4b67df723..d1822eb5f 100644 --- a/js/package.json +++ b/js/package.json @@ -105,19 +105,20 @@ "uuid": "^10.0.0" }, "devDependencies": { - "@ai-sdk/openai": "^0.0.40", + "@ai-sdk/openai": "^0.0.68", "@babel/preset-env": "^7.22.4", "@faker-js/faker": "^8.4.1", "@jest/globals": "^29.5.0", "@langchain/core": "^0.3.1", "@langchain/langgraph": "^0.2.3", "@langchain/openai": "^0.3.0", + "@opentelemetry/sdk-node": "^0.53.0", "@opentelemetry/sdk-trace-base": "^1.26.0", "@tsconfig/recommended": "^1.0.2", "@types/jest": "^29.5.1", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.59.8", - "ai": "^3.2.37", + "ai": "^3.4.17", "babel-jest": "^29.5.0", "cross-env": "^7.0.3", "dotenv": "^16.1.3", diff --git a/js/yarn.lock b/js/yarn.lock index 531ebf491..480daf0d0 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -2,72 +2,75 @@ # yarn lockfile v1 -"@ai-sdk/openai@^0.0.40": - version "0.0.40" - resolved "https://registry.yarnpkg.com/@ai-sdk/openai/-/openai-0.0.40.tgz#227df69c8edf8b26b17f78ae55daa03e58a58870" - integrity sha512-9Iq1UaBHA5ZzNv6j3govuKGXrbrjuWvZIgWNJv4xzXlDMHu9P9hnqlBr/Aiay54WwCuTVNhTzAUTfFgnTs2kbQ== +"@ai-sdk/openai@^0.0.68": + version "0.0.68" + resolved "https://registry.yarnpkg.com/@ai-sdk/openai/-/openai-0.0.68.tgz#7507534a217355273651ad2ea0fffd6e208587ea" + integrity sha512-WSzB7qpBTrnYvFbnBBmIsw1G8GM04JRMr+I7B5T7msgZfleG4cTvVrn9A1HeHHw9TmbKiaCKJrEZH4V0lb7jNQ== dependencies: - "@ai-sdk/provider" "0.0.14" - "@ai-sdk/provider-utils" "1.0.5" + "@ai-sdk/provider" "0.0.24" + "@ai-sdk/provider-utils" "1.0.20" -"@ai-sdk/provider-utils@1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@ai-sdk/provider-utils/-/provider-utils-1.0.5.tgz#765c60871019ded104d79b4cea0805ba563bb5aa" - integrity sha512-XfOawxk95X3S43arn2iQIFyWGMi0DTxsf9ETc6t7bh91RPWOOPYN1tsmS5MTKD33OGJeaDQ/gnVRzXUCRBrckQ== +"@ai-sdk/provider-utils@1.0.20": + version "1.0.20" + resolved "https://registry.yarnpkg.com/@ai-sdk/provider-utils/-/provider-utils-1.0.20.tgz#46175945dc32ad2d76cb5447738bcac3ad59dbcb" + integrity sha512-ngg/RGpnA00eNOWEtXHenpX1MsM2QshQh4QJFjUfwcqHpM5kTfG7je7Rc3HcEDP+OkRVv2GF+X4fC1Vfcnl8Ow== dependencies: - "@ai-sdk/provider" "0.0.14" + "@ai-sdk/provider" "0.0.24" eventsource-parser "1.1.2" nanoid "3.3.6" secure-json-parse "2.7.0" -"@ai-sdk/provider@0.0.14": - version "0.0.14" - resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-0.0.14.tgz#a07569c39a8828aa8312cf1ac6f35ce6ee1b2fce" - integrity sha512-gaQ5Y033nro9iX1YUjEDFDRhmMcEiCk56LJdIUbX5ozEiCNCfpiBpEqrjSp/Gp5RzBS2W0BVxfG7UGW6Ezcrzg== +"@ai-sdk/provider@0.0.24": + version "0.0.24" + resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-0.0.24.tgz#e794f4255a833c47aeffcd8f6808a79b2a6b1f06" + integrity sha512-XMsNGJdGO+L0cxhhegtqZ8+T6nn4EoShS819OvCgI2kLbYTIvk0GWFGD0AXJmxkxs3DrpsJxKAFukFR7bvTkgQ== dependencies: json-schema "0.4.0" -"@ai-sdk/react@0.0.30": - version "0.0.30" - resolved "https://registry.yarnpkg.com/@ai-sdk/react/-/react-0.0.30.tgz#51d586141a81d7f9b76798922b206e8c6faf04dc" - integrity sha512-VnHYRzwhiM4bZdL9DXwJltN8Qnz1MkFdRTa1y7KdmHSJ18ebCNWmPO5XJhnZiQdEXHYmrzZ3WiVt2X6pxK07FA== +"@ai-sdk/react@0.0.64": + version "0.0.64" + resolved "https://registry.yarnpkg.com/@ai-sdk/react/-/react-0.0.64.tgz#921d1dc53c98b7c3488a2099d2b67f6573c83e92" + integrity sha512-4LN2vleyA6rYHZ4Rk9CdxnJgaVkNPJDD4Wx1brUhc5RvUxj3TODcm2UwGOR/mxv4pcydtZGELfQQs/i/tkAUCw== dependencies: - "@ai-sdk/provider-utils" "1.0.5" - "@ai-sdk/ui-utils" "0.0.20" + "@ai-sdk/provider-utils" "1.0.20" + "@ai-sdk/ui-utils" "0.0.46" swr "2.2.5" -"@ai-sdk/solid@0.0.23": - version "0.0.23" - resolved "https://registry.yarnpkg.com/@ai-sdk/solid/-/solid-0.0.23.tgz#712cf1a02bfc337806c5c1b486d16252bec57a15" - integrity sha512-GMojG2PsqwnOGfx7C1MyQPzPBIlC44qn3ykjp9OVnN2Fu47mcFp3QM6gwWoHwNqi7FQDjRy+s/p+8EqYIQcAwg== +"@ai-sdk/solid@0.0.50": + version "0.0.50" + resolved "https://registry.yarnpkg.com/@ai-sdk/solid/-/solid-0.0.50.tgz#a7a30959a97c472a7bae38986958c5164aa2c487" + integrity sha512-JF+KKOgGAgcROgae6FU+hAtxMRhR896SzwI3H1h5hFOZrjqYeYzemJoKzA5MR5IBnPSK4FzEjunc8G5L67TyzQ== dependencies: - "@ai-sdk/provider-utils" "1.0.5" - "@ai-sdk/ui-utils" "0.0.20" + "@ai-sdk/provider-utils" "1.0.20" + "@ai-sdk/ui-utils" "0.0.46" -"@ai-sdk/svelte@0.0.24": - version "0.0.24" - resolved "https://registry.yarnpkg.com/@ai-sdk/svelte/-/svelte-0.0.24.tgz#2519b84a0c104c82d5e48d3b8e9350e9dd4af6cf" - integrity sha512-ZjzzvfYLE01VTO0rOZf6z9sTGhJhe6IYZMxQiM3P+zemufRYe57NDcLYEb6h+2qhvU6Z+k/Q+Nh/spAt0JzGUg== +"@ai-sdk/svelte@0.0.52": + version "0.0.52" + resolved "https://registry.yarnpkg.com/@ai-sdk/svelte/-/svelte-0.0.52.tgz#3b1ee970ce870a5b565807d88b701185afabcd4b" + integrity sha512-ZGd81ruVuqpOh1Suma+HwBMBywcOV0IUzi96Q3knIoZIz99sVwebSKH8ExMofXm49bQdCTRa73Wn8sTs6QDIYg== dependencies: - "@ai-sdk/provider-utils" "1.0.5" - "@ai-sdk/ui-utils" "0.0.20" + "@ai-sdk/provider-utils" "1.0.20" + "@ai-sdk/ui-utils" "0.0.46" sswr "2.1.0" -"@ai-sdk/ui-utils@0.0.20": - version "0.0.20" - resolved "https://registry.yarnpkg.com/@ai-sdk/ui-utils/-/ui-utils-0.0.20.tgz#c68968185a7cc33f7d98d13999731e1c7b672cbb" - integrity sha512-6MRWigzXfuxUcAYEFMLP6cLbALJkg12Iz1Sl+wuPMpB6aw7di2ePiTuNakFUYjgP7TNsW4UxzpypBqqJ1KNB0A== +"@ai-sdk/ui-utils@0.0.46": + version "0.0.46" + resolved "https://registry.yarnpkg.com/@ai-sdk/ui-utils/-/ui-utils-0.0.46.tgz#72311a1917a370074089cc6dd8c982d272f6b836" + integrity sha512-ZG/wneyJG+6w5Nm/hy1AKMuRgjPQToAxBsTk61c9sVPUTaxo+NNjM2MhXQMtmsja2N5evs8NmHie+ExEgpL3cA== dependencies: - "@ai-sdk/provider-utils" "1.0.5" + "@ai-sdk/provider" "0.0.24" + "@ai-sdk/provider-utils" "1.0.20" + json-schema "0.4.0" secure-json-parse "2.7.0" + zod-to-json-schema "3.23.2" -"@ai-sdk/vue@0.0.24": - version "0.0.24" - resolved "https://registry.yarnpkg.com/@ai-sdk/vue/-/vue-0.0.24.tgz#2e72f7e755850ed51540f9a7b25dc6b228a8647a" - integrity sha512-0S+2dVSui6LFgaWoFx+3h5R7GIP9MxdJo63tFuLvgyKr2jmpo5S5kGcWl95vNdzKDqaesAXfOnky+tn5A2d49A== +"@ai-sdk/vue@0.0.55": + version "0.0.55" + resolved "https://registry.yarnpkg.com/@ai-sdk/vue/-/vue-0.0.55.tgz#3da3466418a3e105dd96bdee7217bd2d94a5cb61" + integrity sha512-NZ89CeRPO3D9GjI7GmK3vC+YXjsaWi3iCIvxlGqfQYt0JFKcjgM6dfeq8Nkk+qWI9OoxoOhV/yQdqWQKPv3RRg== dependencies: - "@ai-sdk/provider-utils" "1.0.5" - "@ai-sdk/ui-utils" "0.0.20" + "@ai-sdk/provider-utils" "1.0.20" + "@ai-sdk/ui-utils" "0.0.46" swrv "1.0.4" "@ampproject/remapping@^2.2.0": @@ -1096,6 +1099,24 @@ resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-8.4.1.tgz#5d5e8aee8fce48f5e189bf730ebd1f758f491451" integrity sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg== +"@grpc/grpc-js@^1.7.1": + version "1.12.2" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.12.2.tgz#97eda82dd49bb9c24eaf6434ea8d7de446e95aac" + integrity sha512-bgxdZmgTrJZX50OjyVwz3+mNEnCTNkh3cIqGPWVNeW9jX6bn1ZkU80uPd+67/ZpIJIjRQ9qaHCjhavyoWYxumg== + dependencies: + "@grpc/proto-loader" "^0.7.13" + "@js-sdsl/ordered-map" "^4.4.2" + +"@grpc/proto-loader@^0.7.13": + version "0.7.13" + resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.13.tgz#f6a44b2b7c9f7b609f5748c6eac2d420e37670cf" + integrity sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw== + dependencies: + lodash.camelcase "^4.3.0" + long "^5.0.0" + protobufjs "^7.2.5" + yargs "^17.7.2" + "@humanwhocodes/config-array@^0.11.8": version "0.11.8" resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz" @@ -1368,6 +1389,11 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" +"@js-sdsl/ordered-map@^4.4.2": + version "4.4.2" + resolved "https://registry.yarnpkg.com/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz#9299f82874bab9e4c7f9c48d865becbfe8d6907c" + integrity sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw== + "@langchain/core@^0.3.1": version "0.3.1" resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.3.1.tgz#f06206809575b2a95eaef609b3273842223c0786" @@ -1440,11 +1466,23 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@opentelemetry/api@1.9.0": +"@opentelemetry/api-logs@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.53.0.tgz#c478cbd8120ec2547b64edfa03a552cfe42170be" + integrity sha512-8HArjKx+RaAI8uEIgcORbZIPklyh1YLjPSBus8hjRmvLi6DeFzgOcdZ7KwPabKj8mXF8dX0hyfAyGfycz0DbFw== + dependencies: + "@opentelemetry/api" "^1.0.0" + +"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.0.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== +"@opentelemetry/context-async-hooks@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-1.26.0.tgz#fa92f722cf685685334bba95f258d3ef9fce60f6" + integrity sha512-HedpXXYzzbaoutw6DFLWLDket2FwLkLpil4hGCZ1xYEIMTcivdfwEOISgdbLEWyG3HW52gTq2V9mOVJrONgiwg== + "@opentelemetry/core@1.26.0": version "1.26.0" resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.26.0.tgz#7d84265aaa850ed0ca5813f97d831155be42b328" @@ -1452,6 +1490,142 @@ dependencies: "@opentelemetry/semantic-conventions" "1.27.0" +"@opentelemetry/exporter-logs-otlp-grpc@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.53.0.tgz#cc2514acbff2a41fa428c8b8e81ca386027890e2" + integrity sha512-x5ygAQgWAQOI+UOhyV3z9eW7QU2dCfnfOuIBiyYmC2AWr74f6x/3JBnP27IAcEx6aihpqBYWKnpoUTztkVPAZw== + dependencies: + "@grpc/grpc-js" "^1.7.1" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-grpc-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + "@opentelemetry/sdk-logs" "0.53.0" + +"@opentelemetry/exporter-logs-otlp-http@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.53.0.tgz#1b4a152ea427ec4581532880fd0d620cc559cb11" + integrity sha512-cSRKgD/n8rb+Yd+Cif6EnHEL/VZg1o8lEcEwFji1lwene6BdH51Zh3feAD9p2TyVoBKrl6Q9Zm2WltSp2k9gWQ== + dependencies: + "@opentelemetry/api-logs" "0.53.0" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + "@opentelemetry/sdk-logs" "0.53.0" + +"@opentelemetry/exporter-logs-otlp-proto@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.53.0.tgz#5227efbd9ced9f8f5878dc3e60fc86707f42f5f5" + integrity sha512-jhEcVL1deeWNmTUP05UZMriZPSWUBcfg94ng7JuBb1q2NExgnADQFl1VQQ+xo62/JepK+MxQe4xAwlsDQFbISA== + dependencies: + "@opentelemetry/api-logs" "0.53.0" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-logs" "0.53.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + +"@opentelemetry/exporter-trace-otlp-grpc@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.53.0.tgz#716429f58e71e101fc1fa79b3634083faf7f76da" + integrity sha512-m6KSh6OBDwfDjpzPVbuJbMgMbkoZfpxYH2r262KckgX9cMYvooWXEKzlJYsNDC6ADr28A1rtRoUVRwNfIN4tUg== + dependencies: + "@grpc/grpc-js" "^1.7.1" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-grpc-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + +"@opentelemetry/exporter-trace-otlp-http@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.53.0.tgz#48e46c4573a35d31c14e6bc44635923e32970b9a" + integrity sha512-m7F5ZTq+V9mKGWYpX8EnZ7NjoqAU7VemQ1E2HAG+W/u0wpY1x0OmbxAXfGKFHCspdJk8UKlwPGrpcB8nay3P8A== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + +"@opentelemetry/exporter-trace-otlp-proto@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.53.0.tgz#a5cf9ddd02f71c1cff7f425f2c138f056cfb3683" + integrity sha512-T/bdXslwRKj23S96qbvGtaYOdfyew3TjPEKOk5mHjkCmkVl1O9C/YMdejwSsdLdOq2YW30KjR9kVi0YMxZushQ== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + +"@opentelemetry/exporter-zipkin@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.26.0.tgz#7f7aa5f72f2048ff1316e006b14cce4182b408c3" + integrity sha512-PW5R34n3SJHO4t0UetyHKiXL6LixIqWN6lWncg3eRXhKuT30x+b7m5sDJS0kEWRfHeS+kG7uCw2vBzmB2lk3Dw== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + "@opentelemetry/semantic-conventions" "1.27.0" + +"@opentelemetry/instrumentation@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.53.0.tgz#e6369e4015eb5112468a4d45d38dcada7dad892d" + integrity sha512-DMwg0hy4wzf7K73JJtl95m/e0boSoWhH07rfvHvYzQtBD3Bmv0Wc1x733vyZBqmFm8OjJD0/pfiUg1W3JjFX0A== + dependencies: + "@opentelemetry/api-logs" "0.53.0" + "@types/shimmer" "^1.2.0" + import-in-the-middle "^1.8.1" + require-in-the-middle "^7.1.1" + semver "^7.5.2" + shimmer "^1.2.1" + +"@opentelemetry/otlp-exporter-base@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.53.0.tgz#dfe51874b869c687c3cb463b70cddda7de282762" + integrity sha512-UCWPreGQEhD6FjBaeDuXhiMf6kkBODF0ZQzrk/tuQcaVDJ+dDQ/xhJp192H9yWnKxVpEjFrSSLnpqmX4VwX+eA== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-transformer" "0.53.0" + +"@opentelemetry/otlp-grpc-exporter-base@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.53.0.tgz#6c5ba207352e23d45dc75473e157ef27a1a7f4c8" + integrity sha512-F7RCN8VN+lzSa4fGjewit8Z5fEUpY/lmMVy5EWn2ZpbAabg3EE3sCLuTNfOiooNGnmvzimUPruoeqeko/5/TzQ== + dependencies: + "@grpc/grpc-js" "^1.7.1" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/otlp-exporter-base" "0.53.0" + "@opentelemetry/otlp-transformer" "0.53.0" + +"@opentelemetry/otlp-transformer@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-transformer/-/otlp-transformer-0.53.0.tgz#55d435db5ed5cf56b99c010827294dd4921c45c2" + integrity sha512-rM0sDA9HD8dluwuBxLetUmoqGJKSAbWenwD65KY9iZhUxdBHRLrIdrABfNDP7aiTjcgK8XFyTn5fhDz7N+W6DA== + dependencies: + "@opentelemetry/api-logs" "0.53.0" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-logs" "0.53.0" + "@opentelemetry/sdk-metrics" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + protobufjs "^7.3.0" + +"@opentelemetry/propagator-b3@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-b3/-/propagator-b3-1.26.0.tgz#3ebbeff26a3fb81e8be011666ea6d07ff3e4fba7" + integrity sha512-vvVkQLQ/lGGyEy9GT8uFnI047pajSOVnZI2poJqVGD3nJ+B9sFGdlHNnQKophE3lHfnIH0pw2ubrCTjZCgIj+Q== + dependencies: + "@opentelemetry/core" "1.26.0" + +"@opentelemetry/propagator-jaeger@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.26.0.tgz#096ac03d754204921cd5a886c77b5c9bd4677cd7" + integrity sha512-DelFGkCdaxA1C/QA0Xilszfr0t4YbGd3DjxiCDPh34lfnFr+VkkrjV9S8ZTJvAzfdKERXhfOxIKBoGPJwoSz7Q== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/resources@1.26.0": version "1.26.0" resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.26.0.tgz#da4c7366018bd8add1f3aa9c91c6ac59fd503cef" @@ -1460,7 +1634,46 @@ "@opentelemetry/core" "1.26.0" "@opentelemetry/semantic-conventions" "1.27.0" -"@opentelemetry/sdk-trace-base@^1.26.0": +"@opentelemetry/sdk-logs@0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-logs/-/sdk-logs-0.53.0.tgz#ec8b69278c4e683c13c58ed4285a47c27f5799c6" + integrity sha512-dhSisnEgIj/vJZXZV6f6KcTnyLDx/VuQ6l3ejuZpMpPlh9S1qMHiZU9NMmOkVkwwHkMy3G6mEBwdP23vUZVr4g== + dependencies: + "@opentelemetry/api-logs" "0.53.0" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/resources" "1.26.0" + +"@opentelemetry/sdk-metrics@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics/-/sdk-metrics-1.26.0.tgz#37bb0afb1d4447f50aab9cdd05db6f2d8b86103e" + integrity sha512-0SvDXmou/JjzSDOjUmetAAvcKQW6ZrvosU0rkbDGpXvvZN+pQF6JbK/Kd4hNdK4q/22yeruqvukXEJyySTzyTQ== + dependencies: + "@opentelemetry/core" "1.26.0" + "@opentelemetry/resources" "1.26.0" + +"@opentelemetry/sdk-node@^0.53.0": + version "0.53.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-node/-/sdk-node-0.53.0.tgz#0d25a142009792f9a4d7d69ab243a225c229643b" + integrity sha512-0hsxfq3BKy05xGktwG8YdGdxV978++x40EAKyKr1CaHZRh8uqVlXnclnl7OMi9xLMJEcXUw7lGhiRlArFcovyg== + dependencies: + "@opentelemetry/api-logs" "0.53.0" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/exporter-logs-otlp-grpc" "0.53.0" + "@opentelemetry/exporter-logs-otlp-http" "0.53.0" + "@opentelemetry/exporter-logs-otlp-proto" "0.53.0" + "@opentelemetry/exporter-trace-otlp-grpc" "0.53.0" + "@opentelemetry/exporter-trace-otlp-http" "0.53.0" + "@opentelemetry/exporter-trace-otlp-proto" "0.53.0" + "@opentelemetry/exporter-zipkin" "1.26.0" + "@opentelemetry/instrumentation" "0.53.0" + "@opentelemetry/resources" "1.26.0" + "@opentelemetry/sdk-logs" "0.53.0" + "@opentelemetry/sdk-metrics" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + "@opentelemetry/sdk-trace-node" "1.26.0" + "@opentelemetry/semantic-conventions" "1.27.0" + +"@opentelemetry/sdk-trace-base@1.26.0", "@opentelemetry/sdk-trace-base@^1.26.0": version "1.26.0" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.26.0.tgz#0c913bc6d2cfafd901de330e4540952269ae579c" integrity sha512-olWQldtvbK4v22ymrKLbIcBi9L2SpMO84sCPY54IVsJhP9fRsxJT194C/AVaAuJzLE30EdhhM1VmvVYR7az+cw== @@ -1469,11 +1682,76 @@ "@opentelemetry/resources" "1.26.0" "@opentelemetry/semantic-conventions" "1.27.0" +"@opentelemetry/sdk-trace-node@1.26.0": + version "1.26.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.26.0.tgz#169ef4fc058e82a12460da18cedaf6e4615fc617" + integrity sha512-Fj5IVKrj0yeUwlewCRwzOVcr5avTuNnMHWf7GPc1t6WaT78J6CJyF3saZ/0RkZfdeNO8IcBl/bNcWMVZBMRW8Q== + dependencies: + "@opentelemetry/context-async-hooks" "1.26.0" + "@opentelemetry/core" "1.26.0" + "@opentelemetry/propagator-b3" "1.26.0" + "@opentelemetry/propagator-jaeger" "1.26.0" + "@opentelemetry/sdk-trace-base" "1.26.0" + semver "^7.5.2" + "@opentelemetry/semantic-conventions@1.27.0": version "1.27.0" resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz#1a857dcc95a5ab30122e04417148211e6f945e6c" integrity sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg== +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@sinclair/typebox@^0.25.16": version "0.25.24" resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz" @@ -1613,6 +1891,13 @@ resolved "https://registry.npmjs.org/@types/node/-/node-20.2.5.tgz" integrity sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ== +"@types/node@>=13.7.0": + version "22.7.7" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.7.tgz#6cd9541c3dccb4f7e8b141b491443f4a1570e307" + integrity sha512-SRxCrrg9CL/y54aiMCG3edPKdprgMVGDXjA3gB8UmmBW5TcXzRUYAh8EWzTnSJFAd1rgImPELza+A3bJ+qxz8Q== + dependencies: + undici-types "~6.19.2" + "@types/node@^18.11.18": version "18.19.17" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.17.tgz#a581a9fb4b2cfdbc61f008804f4436b2d5c40354" @@ -1640,6 +1925,11 @@ resolved "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz" integrity sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw== +"@types/shimmer@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@types/shimmer/-/shimmer-1.2.0.tgz#9b706af96fa06416828842397a70dfbbf1c14ded" + integrity sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg== + "@types/stack-utils@^2.0.0": version "2.0.1" resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz" @@ -1753,6 +2043,11 @@ abort-controller@^3.0.0: dependencies: event-target-shim "^5.0.0" +acorn-import-attributes@^1.9.5: + version "1.9.5" + resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" + integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== + acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" @@ -1768,6 +2063,11 @@ acorn@^8.4.1, acorn@^8.8.0: resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.8.2: + version "8.13.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.13.0.tgz#2a30d670818ad16ddd6a35d3842dacec9e5d7ca3" + integrity sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w== + agentkeepalive@^4.2.1: version "4.5.0" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" @@ -1775,25 +2075,25 @@ agentkeepalive@^4.2.1: dependencies: humanize-ms "^1.2.1" -ai@^3.2.37: - version "3.2.37" - resolved "https://registry.yarnpkg.com/ai/-/ai-3.2.37.tgz#148ed3124e6b0a01c703597471718520ef1c498d" - integrity sha512-waqKYZOE1zJwKEHx69R4v/xNG0a1o0He8TDgX29hUu36Zk0yrBJoVSlXbC9KoFuxW4eRpt+gZv1kqd1nVc1CGg== - dependencies: - "@ai-sdk/provider" "0.0.14" - "@ai-sdk/provider-utils" "1.0.5" - "@ai-sdk/react" "0.0.30" - "@ai-sdk/solid" "0.0.23" - "@ai-sdk/svelte" "0.0.24" - "@ai-sdk/ui-utils" "0.0.20" - "@ai-sdk/vue" "0.0.24" +ai@^3.4.17: + version "3.4.17" + resolved "https://registry.yarnpkg.com/ai/-/ai-3.4.17.tgz#9c5bbce9a2a2fdb49058ded31f0d5ba9f8531bfd" + integrity sha512-QZc+NgNlzPT34ZTHaCGGXVJ+stbMLj98hwq+vJaIzD1lns6HlDatrmlFjJsYYf8FtnfqGV7yPNu8DrH8a274vA== + dependencies: + "@ai-sdk/provider" "0.0.24" + "@ai-sdk/provider-utils" "1.0.20" + "@ai-sdk/react" "0.0.64" + "@ai-sdk/solid" "0.0.50" + "@ai-sdk/svelte" "0.0.52" + "@ai-sdk/ui-utils" "0.0.46" + "@ai-sdk/vue" "0.0.55" "@opentelemetry/api" "1.9.0" eventsource-parser "1.1.2" json-schema "0.4.0" jsondiffpatch "0.6.0" nanoid "3.3.6" secure-json-parse "2.7.0" - zod-to-json-schema "3.22.5" + zod-to-json-schema "3.23.2" ajv@^6.10.0, ajv@^6.12.4: version "6.12.6" @@ -2129,6 +2429,11 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz" integrity sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA== +cjs-module-lexer@^1.2.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz#707413784dbb3a72aa11c2f2b042a0bef4004170" + integrity sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA== + client-only@^0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/client-only/-/client-only-0.0.1.tgz#38bba5d403c41ab150bff64a95c85013cf73bca1" @@ -2246,6 +2551,13 @@ debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: dependencies: ms "2.1.2" +debug@^4.3.5: + version "4.3.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" + integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== + dependencies: + ms "^2.1.3" + decamelize@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -3000,7 +3312,7 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -hasown@^2.0.0: +hasown@^2.0.0, hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== @@ -3037,6 +3349,16 @@ import-fresh@^3.0.0, import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" +import-in-the-middle@^1.8.1: + version "1.11.2" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.11.2.tgz#dd848e72b63ca6cd7c34df8b8d97fc9baee6174f" + integrity sha512-gK6Rr6EykBcc6cVWRSBR5TWf8nn6hZMYSRYqCcHa0l0d1fPK7JSYo6+Mlmck76jIX9aL/IZ71c06U2VpFwl1zA== + dependencies: + acorn "^8.8.2" + acorn-import-attributes "^1.9.5" + cjs-module-lexer "^1.2.2" + module-details-from-path "^1.0.3" + import-local@^3.0.2: version "3.1.0" resolved "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz" @@ -3113,6 +3435,13 @@ is-core-module@^2.11.0: dependencies: has "^1.0.3" +is-core-module@^2.13.0: + version "2.15.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" + integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== + dependencies: + hasown "^2.0.2" + is-date-object@^1.0.1: version "1.0.5" resolved "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz" @@ -3774,6 +4103,11 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lodash.camelcase@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" + integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== + lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz" @@ -3789,6 +4123,11 @@ lodash.merge@^4.6.2: resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== +long@^5.0.0: + version "5.2.3" + resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" + integrity sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q== + lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" @@ -3869,12 +4208,17 @@ minimist@^1.2.0, minimist@^1.2.6: resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +module-details-from-path@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" + integrity sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A== + ms@2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0, ms@^2.1.1: +ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -4185,6 +4529,24 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" +protobufjs@^7.2.5, protobufjs@^7.3.0: + version "7.4.0" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.4.0.tgz#7efe324ce9b3b61c82aae5de810d287bc08a248a" + integrity sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + punycode@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz" @@ -4269,6 +4631,15 @@ require-directory@^2.1.1: resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== +require-in-the-middle@^7.1.1: + version "7.4.0" + resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-7.4.0.tgz#606977820d4b5f9be75e5a108ce34cfed25b3bb4" + integrity sha512-X34iHADNbNDfr6OTStIAHWSAvvKQRYgLO6duASaVf7J2VA3lvmNYboAHOuLC2huav1IwgZJtyEcJCKVzFxOSMQ== + dependencies: + debug "^4.3.5" + module-details-from-path "^1.0.3" + resolve "^1.22.8" + resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz" @@ -4300,6 +4671,15 @@ resolve@^1.14.2, resolve@^1.20.0, resolve@^1.22.1: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" +resolve@^1.22.8: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + retry@^0.13.1: version "0.13.1" resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz" @@ -4350,7 +4730,7 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.6.3: +semver@^7.5.2, semver@^7.6.3: version "7.6.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== @@ -4379,6 +4759,11 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== +shimmer@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" + integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" @@ -4702,6 +5087,11 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz" @@ -4871,7 +5261,7 @@ yargs-parser@^21.0.1, yargs-parser@^21.1.1: resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== -yargs@^17.3.1: +yargs@^17.3.1, yargs@^17.7.2: version "17.7.2" resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== @@ -4894,10 +5284,10 @@ yocto-queue@^0.1.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== -zod-to-json-schema@3.22.5: - version "3.22.5" - resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.22.5.tgz#3646e81cfc318dbad2a22519e5ce661615418673" - integrity sha512-+akaPo6a0zpVCCseDed504KBJUQpEW5QZw7RMneNmKw+fGaML1Z9tUNLnHHAC8x6dzVRO1eB2oEMyZRnuBZg7Q== +zod-to-json-schema@3.23.2: + version "3.23.2" + resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.2.tgz#bc7e379c8050462538383e382964c03d8fe008f9" + integrity sha512-uSt90Gzc/tUfyNqxnjlfBs8W6WSGpNBv0rVsNxP/BVSMHMKGdthPYff4xtCHYloJGM0CFxFsb3NbC0eqPhfImw== zod-to-json-schema@^3.22.3: version "3.22.4" From 4194ccadc16c470074ef9b305136fb86fc387096 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 21 Oct 2024 23:06:57 +0200 Subject: [PATCH 05/36] Obtain the client from RunTree, make sure to clear out any pending batches --- js/src/wrappers/vercel/exporter.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index db21bb3f6..dce4728ae 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -121,10 +121,10 @@ const sortByHrTime = (a: ReadableSpan, b: ReadableSpan) => { }; export class LangSmithAISDKExporter implements SpanExporter { - private client: Client; + private client: Client | undefined; - constructor(config?: ClientConfig) { - this.client = new Client(config); + constructor(args?: { client?: Client }) { + this.client = args?.client; } export( @@ -184,7 +184,11 @@ export class LangSmithAISDKExporter implements SpanExporter { ), client: this.client, }; - return parentRunTree?.createChild(config) ?? new RunTree(config); + const runTree = + parentRunTree?.createChild(config) ?? new RunTree(config); + this.client ??= runTree.client; + + return runTree; }; switch (span.name) { @@ -406,9 +410,9 @@ export class LangSmithAISDKExporter implements SpanExporter { } async shutdown(): Promise { - // pass + await this.client?.awaitPendingTraceBatches(); } async forceFlush?(): Promise { - // pass + await this.client?.awaitPendingTraceBatches(); } } From e9bd78c685320689f7a7aaf1559c844376a16575 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 21 Oct 2024 23:07:20 +0200 Subject: [PATCH 06/36] Handle image output --- js/src/wrappers/vercel/exporter.ts | 87 +++++++++++++++++++++++++----- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index dce4728ae..1802a6bb0 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -2,11 +2,7 @@ import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; import type { ExportResult } from "@opentelemetry/core"; import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; import type { AISDKSpan } from "./exporter.types.js"; -import type { - BaseMessageFields, - MessageContentText, -} from "@langchain/core/messages"; -import { Client, ClientConfig, RunTree, RunTreeConfig } from "../../index.js"; +import { Client, RunTree, RunTreeConfig } from "../../index.js"; import { KVMap } from "../../schemas.js"; import { AsyncLocalStorageProviderSingleton } from "../../singletons/traceable.js"; @@ -14,14 +10,54 @@ function assertNever(x: never): never { throw new Error("Unreachable state: " + x); } -// TODO: remove dependency on @langchain/core -type $SmithMessage = { type: string; data: BaseMessageFields } | CoreMessage; +// eslint-disable-next-line @typescript-eslint/ban-types +type AnyString = string & {}; +type LangChainMessageFields = { + content: + | string + | Array< + | { type: "text"; text: string } + | { + type: "image_url"; + image_url: + | string + | { + url: string; + detail?: "auto" | "low" | "high" | AnyString; + }; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | (Record & { type?: "text" | "image_url" | string }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any + | (Record & { type?: never }) + >; + name?: string; + id?: string; + additional_kwargs?: { + tool_calls?: { + id: string; + function: { arguments: string; name: string }; + type: "function"; + index?: number; + }[]; + [key: string]: unknown; + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + response_metadata?: Record; +}; +type LangChainLikeMessage = { type: string; data: LangChainMessageFields }; + +// Attempt to convert CoreMessage to a LangChain-compatible format +// which allows us to render messages more nicely in LangSmith function convertCoreToSmith( message: CoreMessage -): $SmithMessage | $SmithMessage[] { +): + | LangChainLikeMessage + | CoreMessage + | Array { if (message.role === "assistant") { - const data: BaseMessageFields = { content: message.content }; + const data: LangChainMessageFields = { content: message.content }; if (Array.isArray(message.content)) { data.content = message.content.map((part) => { @@ -29,7 +65,8 @@ function convertCoreToSmith( return { type: "text", text: part.text, - } satisfies MessageContentText; + ...part.experimental_providerMetadata, + }; } if (part.type === "tool-call") { @@ -38,6 +75,7 @@ function convertCoreToSmith( name: part.toolName, id: part.toolCallId, input: part.args, + ...part.experimental_providerMetadata, }; } @@ -68,8 +106,31 @@ function convertCoreToSmith( } if (message.role === "user") { - // TODO: verify user content - return { type: "human", data: { content: message.content } }; + const data: LangChainMessageFields = { content: message.content }; + + if (Array.isArray(message.content)) { + data.content = message.content.map((part) => { + if (part.type === "text") { + return { + type: "text", + text: part.text, + ...part.experimental_providerMetadata, + }; + } + + if (part.type === "image") { + return { + type: "image_url", + image_url: part.image, + ...part.experimental_providerMetadata, + }; + } + + return part; + }); + } + + return { type: "human", data }; } if (message.role === "system") { @@ -91,7 +152,7 @@ function convertCoreToSmith( return res; } - return message as any; + return message; } const tryJson = ( From c168d9c69777cd7924fb6856f3af5b345d900070 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 21 Oct 2024 23:07:48 +0200 Subject: [PATCH 07/36] Fix JSON data being sent serialized --- js/src/wrappers/vercel/exporter.ts | 12 +++++++----- js/src/wrappers/vercel/exporter.types.ts | 1 - 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 1802a6bb0..4f2b6131b 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -383,7 +383,7 @@ export class LangSmithAISDKExporter implements SpanExporter { } if ("ai.prompt" in span.attributes) { - return { input: span.attributes["ai.prompt"] }; + return { input: tryJson(span.attributes["ai.prompt"]) }; } return undefined; @@ -440,8 +440,8 @@ export class LangSmithAISDKExporter implements SpanExporter { runTreeMap[spanId] = toRunTree({ run_type: "chain", name: span.attributes["ai.model.provider"], - inputs: { value: span.attributes["ai.value"] }, - outputs: { embedding: span.attributes["ai.embedding"] }, + inputs: { value: tryJson(span.attributes["ai.value"]) }, + outputs: { embedding: tryJson(span.attributes["ai.embedding"]) }, }); break; } @@ -451,8 +451,10 @@ export class LangSmithAISDKExporter implements SpanExporter { runTreeMap[spanId] = toRunTree({ run_type: "chain", name: span.attributes["ai.model.provider"], - inputs: { values: span.attributes["ai.values"] }, - outputs: { embeddings: span.attributes["ai.embeddings"] }, + inputs: { values: span.attributes["ai.values"].map(tryJson) }, + outputs: { + embeddings: span.attributes["ai.embeddings"].map(tryJson), + }, }); break; } diff --git a/js/src/wrappers/vercel/exporter.types.ts b/js/src/wrappers/vercel/exporter.types.ts index 8e3d2a32b..95315ae09 100644 --- a/js/src/wrappers/vercel/exporter.types.ts +++ b/js/src/wrappers/vercel/exporter.types.ts @@ -176,7 +176,6 @@ export type EmbedSpan = TypedReadableSpan< "operation.name": "ai.embed"; "ai.operationId": "ai.embed"; - // TODO: is this correct? "ai.value": string; "ai.embedding": string; } From aae789a4355e4f5298da1f70cd9b405f71b1d224 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Mon, 21 Oct 2024 23:08:10 +0200 Subject: [PATCH 08/36] Call all of the AI SDK methods --- js/src/tests/vercel_exporter.int.test.ts | 211 +++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 js/src/tests/vercel_exporter.int.test.ts diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel_exporter.int.test.ts new file mode 100644 index 000000000..18e5e3650 --- /dev/null +++ b/js/src/tests/vercel_exporter.int.test.ts @@ -0,0 +1,211 @@ +import { openai } from "@ai-sdk/openai"; +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { z } from "zod"; +import { LangSmithAISDKExporter } from "../wrappers/vercel.js"; + +import { + generateText, + streamText, + generateObject, + streamObject, + embed, + embedMany, +} from "ai"; +import { tool } from "ai"; + +const telemetrySettings = { + isEnabled: true, + functionId: "functionId", + metadata: { + userId: "123", + language: "english", + }, +}; + +test.concurrent("generateText", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + function getOrders(userId: string) { + return `User ${userId} has the following orders: 1`; + } + + function getTrackingInformation(orderId: string) { + return `Here is the tracking information for ${orderId}`; + } + + await generateText({ + model: openai("gpt-4o-mini"), + messages: [ + { + role: "user", + content: "What are my orders and where are they? My user ID is 123", + }, + ], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => getOrders(userId), + }), + viewTrackingInformation: tool({ + description: "view tracking information for a specific order", + parameters: z.object({ orderId: z.string() }), + execute: async ({ orderId }) => getTrackingInformation(orderId), + }), + }, + experimental_telemetry: telemetrySettings, + maxSteps: 10, + }); + + await sdk.shutdown(); +}); + +test.concurrent("streamText", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + function getOrders(userId: string) { + return `User ${userId} has the following orders: 1`; + } + + function getTrackingInformation(orderId: string) { + return `Here is the tracking information for ${orderId}`; + } + + const result = await streamText({ + model: openai("gpt-4o-mini"), + messages: [ + { + role: "user", + content: "What are my orders and where are they? My user ID is 123", + }, + ], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => getOrders(userId), + }), + viewTrackingInformation: tool({ + description: "view tracking information for a specific order", + parameters: z.object({ orderId: z.string() }), + execute: async ({ orderId }) => getTrackingInformation(orderId), + }), + }, + experimental_telemetry: { isEnabled: true }, + maxSteps: 10, + }); + + for await (const _stream of result.fullStream) { + // consume + } + + await sdk.shutdown(); +}); + +test.concurrent("generateObject", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + await generateObject({ + model: openai("gpt-4o-mini", { structuredOutputs: true }), + schema: z.object({ + recipe: z.object({ + city: z.string(), + unit: z.union([z.literal("celsius"), z.literal("fahrenheit")]), + }), + }), + prompt: "What's the weather in Prague?", + experimental_telemetry: telemetrySettings, + }); + + await sdk.shutdown(); +}); + +test.concurrent("streamObject", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + const result = await streamObject({ + model: openai("gpt-4o-mini", { structuredOutputs: true }), + schema: z.object({ + recipe: z.object({ + city: z.string(), + unit: z.union([z.literal("celsius"), z.literal("fahrenheit")]), + }), + }), + prompt: "What's the weather in Prague?", + experimental_telemetry: telemetrySettings, + }); + + for await (const _partialObject of result.partialObjectStream) { + // pass + } + + await sdk.shutdown(); +}); + +test.concurrent("embed", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + await embed({ + model: openai.embedding("text-embedding-3-small"), + value: "prague castle at sunset", + experimental_telemetry: telemetrySettings, + }); + + await sdk.shutdown(); +}); + +test.concurrent("embedMany", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + await embedMany({ + model: openai.embedding("text-embedding-3-small"), + values: [ + "a peaceful meadow with wildflowers", + "bustling city street at rush hour", + "prague castle at sunset", + ], + experimental_telemetry: telemetrySettings, + }); + + await sdk.shutdown(); +}); + +test.concurrent("generateText with image", async () => { + const traceExporter = new LangSmithAISDKExporter(); + const sdk = new NodeSDK({ traceExporter }); + sdk.start(); + + await generateText({ + model: openai("gpt-4o-mini"), + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "What's in this picture?", + }, + { + type: "image", + image: new URL("https://picsum.photos/200/300"), + }, + ], + }, + ], + experimental_telemetry: telemetrySettings, + }); + + await sdk.shutdown(); +}); From 997204c6c5e4662cdbb0480ed4daafe9ccb6cdde Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Tue, 22 Oct 2024 16:19:46 +0200 Subject: [PATCH 09/36] Fix ordering of recorded runs, buffer runs --- js/src/tests/utils/iterator.ts | 7 + js/src/tests/vercel_exporter.int.test.ts | 135 ++--- js/src/wrappers/vercel/exporter.ts | 596 ++++++++++++++--------- 3 files changed, 421 insertions(+), 317 deletions(-) create mode 100644 js/src/tests/utils/iterator.ts diff --git a/js/src/tests/utils/iterator.ts b/js/src/tests/utils/iterator.ts new file mode 100644 index 000000000..4734369ba --- /dev/null +++ b/js/src/tests/utils/iterator.ts @@ -0,0 +1,7 @@ +export async function gatherIterator( + i: AsyncIterable | Promise> +): Promise> { + const out: T[] = []; + for await (const item of await i) out.push(item); + return out; +} diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel_exporter.int.test.ts index 18e5e3650..7662e5434 100644 --- a/js/src/tests/vercel_exporter.int.test.ts +++ b/js/src/tests/vercel_exporter.int.test.ts @@ -12,6 +12,7 @@ import { embedMany, } from "ai"; import { tool } from "ai"; +import { gatherIterator } from "./utils/iterator.js"; const telemetrySettings = { isEnabled: true, @@ -22,19 +23,11 @@ const telemetrySettings = { }, }; -test.concurrent("generateText", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - - function getOrders(userId: string) { - return `User ${userId} has the following orders: 1`; - } - - function getTrackingInformation(orderId: string) { - return `Here is the tracking information for ${orderId}`; - } +const traceExporter = new LangSmithAISDKExporter(); +const sdk = new NodeSDK({ traceExporter }); +sdk.start(); +test("generateText", async () => { await generateText({ model: openai("gpt-4o-mini"), messages: [ @@ -47,34 +40,48 @@ test.concurrent("generateText", async () => { listOrders: tool({ description: "list all orders", parameters: z.object({ userId: z.string() }), - execute: async ({ userId }) => getOrders(userId), + execute: async ({ userId }) => + `User ${userId} has the following orders: 1`, }), viewTrackingInformation: tool({ description: "view tracking information for a specific order", parameters: z.object({ orderId: z.string() }), - execute: async ({ orderId }) => getTrackingInformation(orderId), + execute: async ({ orderId }) => + `Here is the tracking information for ${orderId}`, }), }, experimental_telemetry: telemetrySettings, maxSteps: 10, }); - await sdk.shutdown(); + await traceExporter.forceFlush?.(); }); -test.concurrent("streamText", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - - function getOrders(userId: string) { - return `User ${userId} has the following orders: 1`; - } +test("generateText with image", async () => { + await generateText({ + model: openai("gpt-4o-mini"), + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "What's in this picture?", + }, + { + type: "image", + image: new URL("https://picsum.photos/200/300"), + }, + ], + }, + ], + experimental_telemetry: telemetrySettings, + }); - function getTrackingInformation(orderId: string) { - return `Here is the tracking information for ${orderId}`; - } + await traceExporter.forceFlush?.(); +}); +test("streamText", async () => { const result = await streamText({ model: openai("gpt-4o-mini"), messages: [ @@ -87,30 +94,25 @@ test.concurrent("streamText", async () => { listOrders: tool({ description: "list all orders", parameters: z.object({ userId: z.string() }), - execute: async ({ userId }) => getOrders(userId), + execute: async ({ userId }) => + `User ${userId} has the following orders: 1`, }), viewTrackingInformation: tool({ description: "view tracking information for a specific order", parameters: z.object({ orderId: z.string() }), - execute: async ({ orderId }) => getTrackingInformation(orderId), + execute: async ({ orderId }) => + `Here is the tracking information for ${orderId}`, }), }, experimental_telemetry: { isEnabled: true }, maxSteps: 10, }); - for await (const _stream of result.fullStream) { - // consume - } - - await sdk.shutdown(); + await gatherIterator(result.fullStream); + await traceExporter.forceFlush?.(); }); -test.concurrent("generateObject", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - +test("generateObject", async () => { await generateObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -123,14 +125,10 @@ test.concurrent("generateObject", async () => { experimental_telemetry: telemetrySettings, }); - await sdk.shutdown(); + await traceExporter.forceFlush?.(); }); -test.concurrent("streamObject", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - +test("streamObject", async () => { const result = await streamObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -143,32 +141,21 @@ test.concurrent("streamObject", async () => { experimental_telemetry: telemetrySettings, }); - for await (const _partialObject of result.partialObjectStream) { - // pass - } - - await sdk.shutdown(); + await gatherIterator(result.partialObjectStream); + await traceExporter.forceFlush?.(); }); -test.concurrent("embed", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - +test("embed", async () => { await embed({ model: openai.embedding("text-embedding-3-small"), value: "prague castle at sunset", experimental_telemetry: telemetrySettings, }); - await sdk.shutdown(); + await traceExporter.forceFlush?.(); }); -test.concurrent("embedMany", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - +test("embedMany", async () => { await embedMany({ model: openai.embedding("text-embedding-3-small"), values: [ @@ -179,33 +166,9 @@ test.concurrent("embedMany", async () => { experimental_telemetry: telemetrySettings, }); - await sdk.shutdown(); + await traceExporter.forceFlush?.(); }); -test.concurrent("generateText with image", async () => { - const traceExporter = new LangSmithAISDKExporter(); - const sdk = new NodeSDK({ traceExporter }); - sdk.start(); - - await generateText({ - model: openai("gpt-4o-mini"), - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: "What's in this picture?", - }, - { - type: "image", - image: new URL("https://picsum.photos/200/300"), - }, - ], - }, - ], - experimental_telemetry: telemetrySettings, - }); - +afterAll(async () => { await sdk.shutdown(); }); diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 4f2b6131b..156222273 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -2,9 +2,9 @@ import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; import type { ExportResult } from "@opentelemetry/core"; import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; import type { AISDKSpan } from "./exporter.types.js"; -import { Client, RunTree, RunTreeConfig } from "../../index.js"; -import { KVMap } from "../../schemas.js"; -import { AsyncLocalStorageProviderSingleton } from "../../singletons/traceable.js"; +import { Client } from "../../index.js"; +import { KVMap, RunCreate } from "../../schemas.js"; +import { v5 as uuid5 } from "uuid"; function assertNever(x: never): never { throw new Error("Unreachable state: " + x); @@ -174,163 +174,189 @@ const tryJson = ( } }; -const sortByHrTime = (a: ReadableSpan, b: ReadableSpan) => { +function stripNonAlphanumeric(input: string) { + return input.replace(/[-:.]/g, ""); +} + +function convertToDottedOrderFormat( + [seconds, nanoseconds]: [seconds: number, nanoseconds: number], + runId: string, + executionOrder: number +) { + // Date only has millisecond precision, so we use the microseconds to break + // possible ties, avoiding incorrect run order + const ms = Number(String(nanoseconds).slice(0, 3)); + const ns = String(Number(String(nanoseconds).slice(3, 6)) + executionOrder) + .padStart(3, "0") + .slice(0, 3); + return ( - Math.sign(a.startTime[0] - b.startTime[0]) || - Math.sign(a.startTime[1] - b.startTime[1]) + stripNonAlphanumeric( + `${new Date(seconds * 1000 + ms).toISOString().slice(0, -1)}${ns}Z` + ) + runId ); -}; +} + +function convertToTimestamp([seconds, nanoseconds]: [ + seconds: number, + nanoseconds: number +]) { + const ms = String(nanoseconds).slice(0, 3); + return Number(String(seconds) + ms); +} + +const RUN_ID_NS = "5c718b20-9078-11ef-9a3d-325096b39f47"; + +interface RunTask { + id: string; + parentId: string | undefined; + startTime: [seconds: number, nanoseconds: number]; + run: RunCreate; + sent: boolean; + executionOrder: number; +} export class LangSmithAISDKExporter implements SpanExporter { - private client: Client | undefined; + private client: Client; + private traceByMap: Record< + string, + { + childMap: Record; + nodeMap: Record; + relativeExecutionOrder: Record; + } + > = {}; constructor(args?: { client?: Client }) { - this.client = args?.client; + this.client = args?.client ?? new Client(); } - export( - spans: ReadableSpan[], - resultCallback: (result: ExportResult) => void - ): void { - const runTreeMap: Record = {}; - const sortedSpans = [...spans].sort(sortByHrTime) as AISDKSpan[]; - - for (const span of sortedSpans) { - const spanId = span.spanContext().spanId; - const parentSpanId = span.parentSpanId; - let parentRunTree = parentSpanId ? runTreeMap[parentSpanId] : null; - - if (parentRunTree == null) { - try { - parentRunTree = - AsyncLocalStorageProviderSingleton.getInstance().getStore() ?? null; - } catch { - // pass - } + protected getRunCreate(span: AISDKSpan): RunCreate { + const runId = uuid5(span.spanContext().spanId, RUN_ID_NS); + const parentRunId = span.parentSpanId + ? uuid5(span.parentSpanId, RUN_ID_NS) + : undefined; + + const asRunCreate = (rawConfig: RunCreate) => { + const aiMetadata = Object.keys(span.attributes) + .filter((key) => key.startsWith("ai.telemetry.metadata.")) + .reduce((acc, key) => { + acc[key.slice("ai.telemetry.metadata.".length)] = + span.attributes[key as keyof typeof span.attributes]; + + return acc; + }, {} as Record); + + if ( + ("ai.telemetry.functionId" in span.attributes && + span.attributes["ai.telemetry.functionId"]) || + ("resource.name" in span.attributes && span.attributes["resource.name"]) + ) { + aiMetadata["functionId"] = + span.attributes["ai.telemetry.functionId"] || + span.attributes["resource.name"]; } - const toRunTree = (rawConfig: RunTreeConfig) => { - const aiMetadata = Object.keys(span.attributes) - .filter((key) => key.startsWith("ai.telemetry.metadata.")) - .reduce((acc, key) => { - acc[key.slice("ai.telemetry.metadata.".length)] = - span.attributes[key as keyof typeof span.attributes]; - - return acc; - }, {} as Record); - - if ( - ("ai.telemetry.functionId" in span.attributes && - span.attributes["ai.telemetry.functionId"]) || - ("resource.name" in span.attributes && - span.attributes["resource.name"]) - ) { - aiMetadata["functionId"] = - span.attributes["ai.telemetry.functionId"] || - span.attributes["resource.name"]; - } - - const config: RunTreeConfig = { - ...rawConfig, + const config: RunCreate = { + ...rawConfig, + id: runId, + parent_run_id: parentRunId, + extra: { + ...rawConfig.extra, metadata: { - ...rawConfig.metadata, + ...rawConfig.extra?.metadata, ...aiMetadata, "ai.operationId": span.attributes["ai.operationId"], }, - start_time: +( - String(span.startTime[0]) + String(span.startTime[1]).slice(0, 3) - ), - end_time: +( - String(span.endTime[0]) + String(span.endTime[1]).slice(0, 3) - ), - client: this.client, - }; - const runTree = - parentRunTree?.createChild(config) ?? new RunTree(config); - this.client ??= runTree.client; - - return runTree; + }, + start_time: convertToTimestamp(span.startTime), + end_time: convertToTimestamp(span.endTime), }; - switch (span.name) { - case "ai.generateText.doGenerate": - case "ai.generateText": - case "ai.streamText.doStream": - case "ai.streamText": { - const inputs = ((): KVMap | undefined => { - if ("ai.prompt.messages" in span.attributes) { - return { - messages: tryJson( - span.attributes["ai.prompt.messages"] - ).flatMap((i: CoreMessage) => convertCoreToSmith(i)), - }; - } - - if ("ai.prompt" in span.attributes) { - const input = tryJson(span.attributes["ai.prompt"]); - - if ( - typeof input === "object" && - input != null && - "messages" in input && - Array.isArray(input.messages) - ) { - return { - messages: input.messages.flatMap((i: CoreMessage) => - convertCoreToSmith(i) - ), - }; - } + return config; + }; + + switch (span.name) { + case "ai.generateText.doGenerate": + case "ai.generateText": + case "ai.streamText.doStream": + case "ai.streamText": { + const inputs = ((): KVMap => { + if ("ai.prompt.messages" in span.attributes) { + return { + messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap( + (i: CoreMessage) => convertCoreToSmith(i) + ), + }; + } - return { input }; - } + if ("ai.prompt" in span.attributes) { + const input = tryJson(span.attributes["ai.prompt"]); - return undefined; - })(); - - const outputs = ((): KVMap | undefined => { - let result: KVMap | undefined = undefined; - if (span.attributes["ai.response.toolCalls"]) { - result = { - llm_output: convertCoreToSmith({ - role: "assistant", - content: tryJson(span.attributes["ai.response.toolCalls"]), - } satisfies CoreAssistantMessage), - }; - } else if (span.attributes["ai.response.text"]) { - result = { - llm_output: convertCoreToSmith({ - role: "assistant", - content: span.attributes["ai.response.text"], - }), + if ( + typeof input === "object" && + input != null && + "messages" in input && + Array.isArray(input.messages) + ) { + return { + messages: input.messages.flatMap((i: CoreMessage) => + convertCoreToSmith(i) + ), }; } - if (span.attributes["ai.usage.completionTokens"]) { - result ??= {}; - result.llm_output ??= {}; - result.llm_output.token_usage ??= {}; - result.llm_output.token_usage["completion_tokens"] = - span.attributes["ai.usage.completionTokens"]; - } + return { input }; + } - if (span.attributes["ai.usage.promptTokens"]) { - result ??= {}; - result.llm_output ??= {}; - result.llm_output.token_usage ??= {}; - result.llm_output.token_usage["prompt_tokens"] = - span.attributes["ai.usage.promptTokens"]; - } + return {}; + })(); + + const outputs = ((): KVMap | undefined => { + let result: KVMap | undefined = undefined; + if (span.attributes["ai.response.toolCalls"]) { + result = { + llm_output: convertCoreToSmith({ + role: "assistant", + content: tryJson(span.attributes["ai.response.toolCalls"]), + } satisfies CoreAssistantMessage), + }; + } else if (span.attributes["ai.response.text"]) { + result = { + llm_output: convertCoreToSmith({ + role: "assistant", + content: span.attributes["ai.response.text"], + }), + }; + } + + if (span.attributes["ai.usage.completionTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["completion_tokens"] = + span.attributes["ai.usage.completionTokens"]; + } - return result; - })(); + if (span.attributes["ai.usage.promptTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["prompt_tokens"] = + span.attributes["ai.usage.promptTokens"]; + } - // TODO: add first_token_time - runTreeMap[spanId] = toRunTree({ - run_type: "llm", - name: span.attributes["ai.model.provider"], - inputs, - outputs, + return result; + })(); + + // TODO: add first_token_time + return asRunCreate({ + run_type: "llm", + name: span.attributes["ai.model.provider"], + inputs, + outputs, + extra: { + batch_size: 1, metadata: { ls_provider: span.attributes["ai.model.provider"] .split(".") @@ -340,88 +366,89 @@ export class LangSmithAISDKExporter implements SpanExporter { .at(1), ls_model_name: span.attributes["ai.model.id"], }, - extra: { batch_size: 1 }, - }); - break; - } - - case "ai.toolCall": { - const args = tryJson(span.attributes["ai.toolCall.args"]); - let inputs: KVMap | undefined = { args }; + }, + }); + break; + } - if (typeof args === "object" && args != null) { - inputs = args; - } + case "ai.toolCall": { + const args = tryJson(span.attributes["ai.toolCall.args"]); + let inputs: KVMap = { args }; - const output = tryJson(span.attributes["ai.toolCall.result"]); - let outputs: KVMap | undefined = { output }; + if (typeof args === "object" && args != null) { + inputs = args; + } - if (typeof output === "object" && output != null) { - outputs = output; - } + const output = tryJson(span.attributes["ai.toolCall.result"]); + let outputs: KVMap = { output }; - runTreeMap[spanId] = toRunTree({ - run_type: "tool", - name: span.attributes["ai.toolCall.name"], - inputs, - outputs, - }); - break; + if (typeof output === "object" && output != null) { + outputs = output; } - case "ai.streamObject": - case "ai.streamObject.doStream": - case "ai.generateObject": - case "ai.generateObject.doGenerate": { - const inputs = ((): KVMap | undefined => { - if ("ai.prompt.messages" in span.attributes) { - return { - messages: tryJson( - span.attributes["ai.prompt.messages"] - ).flatMap((i: CoreMessage) => convertCoreToSmith(i)), - }; - } + return asRunCreate({ + run_type: "tool", + name: span.attributes["ai.toolCall.name"], + inputs, + outputs, + }); + } - if ("ai.prompt" in span.attributes) { - return { input: tryJson(span.attributes["ai.prompt"]) }; - } + case "ai.streamObject": + case "ai.streamObject.doStream": + case "ai.generateObject": + case "ai.generateObject.doGenerate": { + const inputs = ((): KVMap => { + if ("ai.prompt.messages" in span.attributes) { + return { + messages: tryJson(span.attributes["ai.prompt.messages"]).flatMap( + (i: CoreMessage) => convertCoreToSmith(i) + ), + }; + } - return undefined; - })(); + if ("ai.prompt" in span.attributes) { + return { input: tryJson(span.attributes["ai.prompt"]) }; + } - const outputs = ((): KVMap | undefined => { - let result: KVMap | undefined = undefined; + return {}; + })(); - if (span.attributes["ai.response.object"]) { - result = { - output: tryJson(span.attributes["ai.response.object"]), - }; - } + const outputs = ((): KVMap | undefined => { + let result: KVMap | undefined = undefined; - if (span.attributes["ai.usage.completionTokens"]) { - result ??= {}; - result.llm_output ??= {}; - result.llm_output.token_usage ??= {}; - result.llm_output.token_usage["completion_tokens"] = - span.attributes["ai.usage.completionTokens"]; - } + if (span.attributes["ai.response.object"]) { + result = { + output: tryJson(span.attributes["ai.response.object"]), + }; + } - if (span.attributes["ai.usage.promptTokens"]) { - result ??= {}; - result.llm_output ??= {}; - result.llm_output.token_usage ??= {}; - result.llm_output.token_usage["prompt_tokens"] = - +span.attributes["ai.usage.promptTokens"]; - } + if (span.attributes["ai.usage.completionTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["completion_tokens"] = + span.attributes["ai.usage.completionTokens"]; + } - return result; - })(); + if (span.attributes["ai.usage.promptTokens"]) { + result ??= {}; + result.llm_output ??= {}; + result.llm_output.token_usage ??= {}; + result.llm_output.token_usage["prompt_tokens"] = + +span.attributes["ai.usage.promptTokens"]; + } + + return result; + })(); - runTreeMap[spanId] = toRunTree({ - run_type: "llm", - name: span.attributes["ai.model.provider"], - inputs, - outputs, + return asRunCreate({ + run_type: "llm", + name: span.attributes["ai.model.provider"], + inputs, + outputs, + extra: { + batch_size: 1, metadata: { ls_provider: span.attributes["ai.model.provider"] .split(".") @@ -431,41 +458,137 @@ export class LangSmithAISDKExporter implements SpanExporter { .at(1), ls_model_name: span.attributes["ai.model.id"], }, - extra: { batch_size: 1 }, - }); - break; - } + }, + }); + } - case "ai.embed": { - runTreeMap[spanId] = toRunTree({ - run_type: "chain", - name: span.attributes["ai.model.provider"], - inputs: { value: tryJson(span.attributes["ai.value"]) }, - outputs: { embedding: tryJson(span.attributes["ai.embedding"]) }, - }); - break; - } - case "ai.embed.doEmbed": - case "ai.embedMany": - case "ai.embedMany.doEmbed": { - runTreeMap[spanId] = toRunTree({ - run_type: "chain", - name: span.attributes["ai.model.provider"], - inputs: { values: span.attributes["ai.values"].map(tryJson) }, - outputs: { - embeddings: span.attributes["ai.embeddings"].map(tryJson), + case "ai.embed": { + return asRunCreate({ + run_type: "chain", + name: span.attributes["ai.model.provider"], + inputs: { value: tryJson(span.attributes["ai.value"]) }, + outputs: { embedding: tryJson(span.attributes["ai.embedding"]) }, + }); + } + case "ai.embed.doEmbed": + case "ai.embedMany": + case "ai.embedMany.doEmbed": { + return asRunCreate({ + run_type: "chain", + name: span.attributes["ai.model.provider"], + inputs: { values: span.attributes["ai.values"].map(tryJson) }, + outputs: { + embeddings: span.attributes["ai.embeddings"].map(tryJson), + }, + }); + } + + default: + assertNever(span); + } + } + + export( + spans: ReadableSpan[], + resultCallback: (result: ExportResult) => void + ): void { + for (const span of spans) { + const { traceId, spanId } = span.spanContext(); + const parentId = span.parentSpanId ?? undefined; + this.traceByMap[traceId] ??= { + childMap: {}, + nodeMap: {}, + relativeExecutionOrder: {}, + }; + + const runId = uuid5(spanId, RUN_ID_NS); + const parentRunId = parentId ? uuid5(parentId, RUN_ID_NS) : undefined; + + const traceMap = this.traceByMap[traceId]; + + traceMap.relativeExecutionOrder[parentRunId ?? "$"] ??= -1; + traceMap.relativeExecutionOrder[parentRunId ?? "$"] += 1; + + traceMap.nodeMap[runId] ??= { + id: runId, + parentId: parentRunId, + startTime: span.startTime, + run: this.getRunCreate(span as AISDKSpan), + sent: false, + executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? "$"], + }; + + traceMap.childMap[parentRunId ?? "$"] ??= []; + traceMap.childMap[parentRunId ?? "$"].push(traceMap.nodeMap[runId]); + } + + // collect all subgraphs + const sampled: [ + { + dotted_order: string; + id: string; + trace_id: string; + parent_run_id: string | undefined; + }, + RunCreate + ][] = []; + + for (const traceId of Object.keys(this.traceByMap)) { + type QueueItem = { item: RunTask; dottedOrder: string; traceId: string }; + + const queue: QueueItem[] = + this.traceByMap[traceId].childMap["$"]?.map((item) => ({ + item, + dottedOrder: convertToDottedOrderFormat( + item.startTime, + item.id, + item.executionOrder + ), + traceId: item.id, + })) ?? []; + + const seen = new Set(); + while (queue.length) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const task = queue.shift()!; + if (seen.has(task.item.id)) continue; + + if (!task.item.sent) { + sampled.push([ + { + id: task.item.id, + parent_run_id: task.item.parentId, + dotted_order: task.dottedOrder, + trace_id: task.traceId, }, - }); - break; + task.item.run, + ]); + task.item.sent = true; } - default: - assertNever(span); + const children = this.traceByMap[traceId].childMap[task.item.id] ?? []; + queue.push( + ...children.map((child) => ({ + item: child, + dottedOrder: [ + task.dottedOrder, + convertToDottedOrderFormat( + child.startTime, + child.id, + child.executionOrder + ), + ].join("."), + traceId: task.traceId, + })) + ); } } Promise.all( - Object.values(runTreeMap).map((runTree) => runTree.postRun()) + sampled.map(([required, value]) => { + const payload = { ...value, ...required }; + return this.client.createRun(payload); + }) ).then( () => resultCallback({ code: 0 }), (error) => resultCallback({ code: 1, error }) @@ -473,6 +596,17 @@ export class LangSmithAISDKExporter implements SpanExporter { } async shutdown(): Promise { + // find nodes which are incomplete + const incompleteNodes = Object.values(this.traceByMap).flatMap((trace) => + Object.values(trace.nodeMap).filter((i) => !i.sent) + ); + + if (incompleteNodes.length > 0) { + console.warn( + "Some incomplete nodes were found before shutdown and not sent to LangSmith." + ); + } + await this.client?.awaitPendingTraceBatches(); } async forceFlush?(): Promise { From 963179cfb62477c7bde559d8c9efcd71a1b6da8f Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Tue, 22 Oct 2024 17:10:27 +0200 Subject: [PATCH 10/36] Verify presence of runs --- js/package.json | 3 +- js/src/tests/vercel_exporter.int.test.ts | 100 +++++-- js/src/wrappers/vercel/exporter.ts | 58 +++- js/yarn.lock | 366 +---------------------- 4 files changed, 127 insertions(+), 400 deletions(-) diff --git a/js/package.json b/js/package.json index d1822eb5f..0e64b4e93 100644 --- a/js/package.json +++ b/js/package.json @@ -112,8 +112,9 @@ "@langchain/core": "^0.3.1", "@langchain/langgraph": "^0.2.3", "@langchain/openai": "^0.3.0", - "@opentelemetry/sdk-node": "^0.53.0", + "@opentelemetry/api": "^1.9.0", "@opentelemetry/sdk-trace-base": "^1.26.0", + "@opentelemetry/sdk-trace-node": "^1.26.0", "@tsconfig/recommended": "^1.0.2", "@types/jest": "^29.5.1", "@typescript-eslint/eslint-plugin": "^5.59.8", diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel_exporter.int.test.ts index 7662e5434..72d4e6dba 100644 --- a/js/src/tests/vercel_exporter.int.test.ts +++ b/js/src/tests/vercel_exporter.int.test.ts @@ -1,8 +1,10 @@ import { openai } from "@ai-sdk/openai"; -import { NodeSDK } from "@opentelemetry/sdk-node"; + +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; +import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; import { z } from "zod"; import { LangSmithAISDKExporter } from "../wrappers/vercel.js"; - +import { v4 as uuid } from "uuid"; import { generateText, streamText, @@ -13,21 +15,35 @@ import { } from "ai"; import { tool } from "ai"; import { gatherIterator } from "./utils/iterator.js"; +import { Client } from "../index.js"; +import { waitUntilRunFound } from "./utils.js"; -const telemetrySettings = { - isEnabled: true, - functionId: "functionId", - metadata: { +const getTelemetrySettings = (langsmithRunId?: string) => { + const metadata: Record = { userId: "123", language: "english", - }, + }; + + if (langsmithRunId) metadata.langsmithRunId = langsmithRunId; + return { + isEnabled: true, + functionId: "functionId", + metadata, + }; }; -const traceExporter = new LangSmithAISDKExporter(); -const sdk = new NodeSDK({ traceExporter }); -sdk.start(); +const client = new Client(); +// Not using @opentelemetry/sdk-node because we need to force flush +// the spans to ensure they are sent to LangSmith between tests +const provider = new NodeTracerProvider(); +provider.addSpanProcessor( + new BatchSpanProcessor(new LangSmithAISDKExporter({ client })) +); +provider.register(); test("generateText", async () => { + const traceId = uuid(); + await generateText({ model: openai("gpt-4o-mini"), messages: [ @@ -50,14 +66,19 @@ test("generateText", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: telemetrySettings, + experimental_telemetry: getTelemetrySettings(traceId), maxSteps: 10, }); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); test("generateText with image", async () => { + const traceId = uuid(); await generateText({ model: openai("gpt-4o-mini"), messages: [ @@ -75,13 +96,18 @@ test("generateText with image", async () => { ], }, ], - experimental_telemetry: telemetrySettings, + experimental_telemetry: getTelemetrySettings(traceId), }); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); test("streamText", async () => { + const traceId = uuid(); const result = await streamText({ model: openai("gpt-4o-mini"), messages: [ @@ -104,15 +130,20 @@ test("streamText", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: { isEnabled: true }, + experimental_telemetry: getTelemetrySettings(traceId), maxSteps: 10, }); await gatherIterator(result.fullStream); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); test("generateObject", async () => { + const traceId = uuid(); await generateObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -122,13 +153,18 @@ test("generateObject", async () => { }), }), prompt: "What's the weather in Prague?", - experimental_telemetry: telemetrySettings, + experimental_telemetry: getTelemetrySettings(traceId), }); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); test("streamObject", async () => { + const traceId = uuid(); const result = await streamObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -138,24 +174,34 @@ test("streamObject", async () => { }), }), prompt: "What's the weather in Prague?", - experimental_telemetry: telemetrySettings, + experimental_telemetry: getTelemetrySettings(traceId), }); await gatherIterator(result.partialObjectStream); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); test("embed", async () => { + const traceId = uuid(); await embed({ model: openai.embedding("text-embedding-3-small"), value: "prague castle at sunset", - experimental_telemetry: telemetrySettings, + experimental_telemetry: getTelemetrySettings(traceId), }); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); test("embedMany", async () => { + const traceId = uuid(); await embedMany({ model: openai.embedding("text-embedding-3-small"), values: [ @@ -163,12 +209,16 @@ test("embedMany", async () => { "bustling city street at rush hour", "prague castle at sunset", ], - experimental_telemetry: telemetrySettings, + experimental_telemetry: getTelemetrySettings(traceId), }); - await traceExporter.forceFlush?.(); + await provider.forceFlush(); + await waitUntilRunFound(client, traceId, true); + + const storedRun = await client.readRun(traceId); + expect(storedRun.id).toEqual(traceId); }); afterAll(async () => { - await sdk.shutdown(); + await provider.shutdown(); }); diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 156222273..88fb0326a 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -224,6 +224,7 @@ export class LangSmithAISDKExporter implements SpanExporter { childMap: Record; nodeMap: Record; relativeExecutionOrder: Record; + userTraceId?: string; } > = {}; @@ -239,7 +240,11 @@ export class LangSmithAISDKExporter implements SpanExporter { const asRunCreate = (rawConfig: RunCreate) => { const aiMetadata = Object.keys(span.attributes) - .filter((key) => key.startsWith("ai.telemetry.metadata.")) + .filter( + (key) => + key.startsWith("ai.telemetry.metadata.") && + key !== "ai.telemetry.metadata.langsmithRunId" + ) .reduce((acc, key) => { acc[key.slice("ai.telemetry.metadata.".length)] = span.attributes[key as keyof typeof span.attributes]; @@ -509,17 +514,29 @@ export class LangSmithAISDKExporter implements SpanExporter { traceMap.relativeExecutionOrder[parentRunId ?? "$"] ??= -1; traceMap.relativeExecutionOrder[parentRunId ?? "$"] += 1; + const aiSpan = span as AISDKSpan; + traceMap.nodeMap[runId] ??= { id: runId, parentId: parentRunId, startTime: span.startTime, - run: this.getRunCreate(span as AISDKSpan), + run: this.getRunCreate(aiSpan), sent: false, executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? "$"], }; traceMap.childMap[parentRunId ?? "$"] ??= []; traceMap.childMap[parentRunId ?? "$"].push(traceMap.nodeMap[runId]); + + if ( + "ai.telemetry.metadata.langsmithRunId" in aiSpan.attributes && + typeof aiSpan.attributes["ai.telemetry.metadata.langsmithRunId"] === + "string" && + aiSpan.attributes["ai.telemetry.metadata.langsmithRunId"] + ) { + traceMap.userTraceId = + aiSpan.attributes["ai.telemetry.metadata.langsmithRunId"]; + } } // collect all subgraphs @@ -535,9 +552,10 @@ export class LangSmithAISDKExporter implements SpanExporter { for (const traceId of Object.keys(this.traceByMap)) { type QueueItem = { item: RunTask; dottedOrder: string; traceId: string }; + const traceMap = this.traceByMap[traceId]; const queue: QueueItem[] = - this.traceByMap[traceId].childMap["$"]?.map((item) => ({ + traceMap.childMap["$"]?.map((item) => ({ item, dottedOrder: convertToDottedOrderFormat( item.startTime, @@ -554,19 +572,33 @@ export class LangSmithAISDKExporter implements SpanExporter { if (seen.has(task.item.id)) continue; if (!task.item.sent) { - sampled.push([ - { - id: task.item.id, - parent_run_id: task.item.parentId, - dotted_order: task.dottedOrder, - trace_id: task.traceId, - }, - task.item.run, - ]); + let ident = { + id: task.item.id, + parent_run_id: task.item.parentId, + dotted_order: task.dottedOrder, + trace_id: task.traceId, + }; + + if (traceMap.userTraceId) { + ident = { + id: ident.id === ident.trace_id ? traceMap.userTraceId : ident.id, + parent_run_id: + ident.parent_run_id === ident.trace_id + ? traceMap.userTraceId + : ident.parent_run_id, + dotted_order: ident.dotted_order.replace( + ident.trace_id, + traceMap.userTraceId + ), + trace_id: traceMap.userTraceId, + }; + } + + sampled.push([ident, task.item.run]); task.item.sent = true; } - const children = this.traceByMap[traceId].childMap[task.item.id] ?? []; + const children = traceMap.childMap[task.item.id] ?? []; queue.push( ...children.map((child) => ({ item: child, diff --git a/js/yarn.lock b/js/yarn.lock index 480daf0d0..31411bc6d 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1099,24 +1099,6 @@ resolved "https://registry.yarnpkg.com/@faker-js/faker/-/faker-8.4.1.tgz#5d5e8aee8fce48f5e189bf730ebd1f758f491451" integrity sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg== -"@grpc/grpc-js@^1.7.1": - version "1.12.2" - resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.12.2.tgz#97eda82dd49bb9c24eaf6434ea8d7de446e95aac" - integrity sha512-bgxdZmgTrJZX50OjyVwz3+mNEnCTNkh3cIqGPWVNeW9jX6bn1ZkU80uPd+67/ZpIJIjRQ9qaHCjhavyoWYxumg== - dependencies: - "@grpc/proto-loader" "^0.7.13" - "@js-sdsl/ordered-map" "^4.4.2" - -"@grpc/proto-loader@^0.7.13": - version "0.7.13" - resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.13.tgz#f6a44b2b7c9f7b609f5748c6eac2d420e37670cf" - integrity sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw== - dependencies: - lodash.camelcase "^4.3.0" - long "^5.0.0" - protobufjs "^7.2.5" - yargs "^17.7.2" - "@humanwhocodes/config-array@^0.11.8": version "0.11.8" resolved "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz" @@ -1389,11 +1371,6 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" -"@js-sdsl/ordered-map@^4.4.2": - version "4.4.2" - resolved "https://registry.yarnpkg.com/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz#9299f82874bab9e4c7f9c48d865becbfe8d6907c" - integrity sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw== - "@langchain/core@^0.3.1": version "0.3.1" resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.3.1.tgz#f06206809575b2a95eaef609b3273842223c0786" @@ -1466,14 +1443,7 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@opentelemetry/api-logs@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.53.0.tgz#c478cbd8120ec2547b64edfa03a552cfe42170be" - integrity sha512-8HArjKx+RaAI8uEIgcORbZIPklyh1YLjPSBus8hjRmvLi6DeFzgOcdZ7KwPabKj8mXF8dX0hyfAyGfycz0DbFw== - dependencies: - "@opentelemetry/api" "^1.0.0" - -"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.0.0": +"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== @@ -1490,128 +1460,6 @@ dependencies: "@opentelemetry/semantic-conventions" "1.27.0" -"@opentelemetry/exporter-logs-otlp-grpc@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.53.0.tgz#cc2514acbff2a41fa428c8b8e81ca386027890e2" - integrity sha512-x5ygAQgWAQOI+UOhyV3z9eW7QU2dCfnfOuIBiyYmC2AWr74f6x/3JBnP27IAcEx6aihpqBYWKnpoUTztkVPAZw== - dependencies: - "@grpc/grpc-js" "^1.7.1" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-grpc-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - "@opentelemetry/sdk-logs" "0.53.0" - -"@opentelemetry/exporter-logs-otlp-http@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.53.0.tgz#1b4a152ea427ec4581532880fd0d620cc559cb11" - integrity sha512-cSRKgD/n8rb+Yd+Cif6EnHEL/VZg1o8lEcEwFji1lwene6BdH51Zh3feAD9p2TyVoBKrl6Q9Zm2WltSp2k9gWQ== - dependencies: - "@opentelemetry/api-logs" "0.53.0" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - "@opentelemetry/sdk-logs" "0.53.0" - -"@opentelemetry/exporter-logs-otlp-proto@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.53.0.tgz#5227efbd9ced9f8f5878dc3e60fc86707f42f5f5" - integrity sha512-jhEcVL1deeWNmTUP05UZMriZPSWUBcfg94ng7JuBb1q2NExgnADQFl1VQQ+xo62/JepK+MxQe4xAwlsDQFbISA== - dependencies: - "@opentelemetry/api-logs" "0.53.0" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-logs" "0.53.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - -"@opentelemetry/exporter-trace-otlp-grpc@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.53.0.tgz#716429f58e71e101fc1fa79b3634083faf7f76da" - integrity sha512-m6KSh6OBDwfDjpzPVbuJbMgMbkoZfpxYH2r262KckgX9cMYvooWXEKzlJYsNDC6ADr28A1rtRoUVRwNfIN4tUg== - dependencies: - "@grpc/grpc-js" "^1.7.1" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-grpc-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - -"@opentelemetry/exporter-trace-otlp-http@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.53.0.tgz#48e46c4573a35d31c14e6bc44635923e32970b9a" - integrity sha512-m7F5ZTq+V9mKGWYpX8EnZ7NjoqAU7VemQ1E2HAG+W/u0wpY1x0OmbxAXfGKFHCspdJk8UKlwPGrpcB8nay3P8A== - dependencies: - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - -"@opentelemetry/exporter-trace-otlp-proto@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.53.0.tgz#a5cf9ddd02f71c1cff7f425f2c138f056cfb3683" - integrity sha512-T/bdXslwRKj23S96qbvGtaYOdfyew3TjPEKOk5mHjkCmkVl1O9C/YMdejwSsdLdOq2YW30KjR9kVi0YMxZushQ== - dependencies: - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - -"@opentelemetry/exporter-zipkin@1.26.0": - version "1.26.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.26.0.tgz#7f7aa5f72f2048ff1316e006b14cce4182b408c3" - integrity sha512-PW5R34n3SJHO4t0UetyHKiXL6LixIqWN6lWncg3eRXhKuT30x+b7m5sDJS0kEWRfHeS+kG7uCw2vBzmB2lk3Dw== - dependencies: - "@opentelemetry/core" "1.26.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - "@opentelemetry/semantic-conventions" "1.27.0" - -"@opentelemetry/instrumentation@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.53.0.tgz#e6369e4015eb5112468a4d45d38dcada7dad892d" - integrity sha512-DMwg0hy4wzf7K73JJtl95m/e0boSoWhH07rfvHvYzQtBD3Bmv0Wc1x733vyZBqmFm8OjJD0/pfiUg1W3JjFX0A== - dependencies: - "@opentelemetry/api-logs" "0.53.0" - "@types/shimmer" "^1.2.0" - import-in-the-middle "^1.8.1" - require-in-the-middle "^7.1.1" - semver "^7.5.2" - shimmer "^1.2.1" - -"@opentelemetry/otlp-exporter-base@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.53.0.tgz#dfe51874b869c687c3cb463b70cddda7de282762" - integrity sha512-UCWPreGQEhD6FjBaeDuXhiMf6kkBODF0ZQzrk/tuQcaVDJ+dDQ/xhJp192H9yWnKxVpEjFrSSLnpqmX4VwX+eA== - dependencies: - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-transformer" "0.53.0" - -"@opentelemetry/otlp-grpc-exporter-base@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.53.0.tgz#6c5ba207352e23d45dc75473e157ef27a1a7f4c8" - integrity sha512-F7RCN8VN+lzSa4fGjewit8Z5fEUpY/lmMVy5EWn2ZpbAabg3EE3sCLuTNfOiooNGnmvzimUPruoeqeko/5/TzQ== - dependencies: - "@grpc/grpc-js" "^1.7.1" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/otlp-exporter-base" "0.53.0" - "@opentelemetry/otlp-transformer" "0.53.0" - -"@opentelemetry/otlp-transformer@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-transformer/-/otlp-transformer-0.53.0.tgz#55d435db5ed5cf56b99c010827294dd4921c45c2" - integrity sha512-rM0sDA9HD8dluwuBxLetUmoqGJKSAbWenwD65KY9iZhUxdBHRLrIdrABfNDP7aiTjcgK8XFyTn5fhDz7N+W6DA== - dependencies: - "@opentelemetry/api-logs" "0.53.0" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-logs" "0.53.0" - "@opentelemetry/sdk-metrics" "1.26.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - protobufjs "^7.3.0" - "@opentelemetry/propagator-b3@1.26.0": version "1.26.0" resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-b3/-/propagator-b3-1.26.0.tgz#3ebbeff26a3fb81e8be011666ea6d07ff3e4fba7" @@ -1634,45 +1482,6 @@ "@opentelemetry/core" "1.26.0" "@opentelemetry/semantic-conventions" "1.27.0" -"@opentelemetry/sdk-logs@0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-logs/-/sdk-logs-0.53.0.tgz#ec8b69278c4e683c13c58ed4285a47c27f5799c6" - integrity sha512-dhSisnEgIj/vJZXZV6f6KcTnyLDx/VuQ6l3ejuZpMpPlh9S1qMHiZU9NMmOkVkwwHkMy3G6mEBwdP23vUZVr4g== - dependencies: - "@opentelemetry/api-logs" "0.53.0" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/resources" "1.26.0" - -"@opentelemetry/sdk-metrics@1.26.0": - version "1.26.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics/-/sdk-metrics-1.26.0.tgz#37bb0afb1d4447f50aab9cdd05db6f2d8b86103e" - integrity sha512-0SvDXmou/JjzSDOjUmetAAvcKQW6ZrvosU0rkbDGpXvvZN+pQF6JbK/Kd4hNdK4q/22yeruqvukXEJyySTzyTQ== - dependencies: - "@opentelemetry/core" "1.26.0" - "@opentelemetry/resources" "1.26.0" - -"@opentelemetry/sdk-node@^0.53.0": - version "0.53.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-node/-/sdk-node-0.53.0.tgz#0d25a142009792f9a4d7d69ab243a225c229643b" - integrity sha512-0hsxfq3BKy05xGktwG8YdGdxV978++x40EAKyKr1CaHZRh8uqVlXnclnl7OMi9xLMJEcXUw7lGhiRlArFcovyg== - dependencies: - "@opentelemetry/api-logs" "0.53.0" - "@opentelemetry/core" "1.26.0" - "@opentelemetry/exporter-logs-otlp-grpc" "0.53.0" - "@opentelemetry/exporter-logs-otlp-http" "0.53.0" - "@opentelemetry/exporter-logs-otlp-proto" "0.53.0" - "@opentelemetry/exporter-trace-otlp-grpc" "0.53.0" - "@opentelemetry/exporter-trace-otlp-http" "0.53.0" - "@opentelemetry/exporter-trace-otlp-proto" "0.53.0" - "@opentelemetry/exporter-zipkin" "1.26.0" - "@opentelemetry/instrumentation" "0.53.0" - "@opentelemetry/resources" "1.26.0" - "@opentelemetry/sdk-logs" "0.53.0" - "@opentelemetry/sdk-metrics" "1.26.0" - "@opentelemetry/sdk-trace-base" "1.26.0" - "@opentelemetry/sdk-trace-node" "1.26.0" - "@opentelemetry/semantic-conventions" "1.27.0" - "@opentelemetry/sdk-trace-base@1.26.0", "@opentelemetry/sdk-trace-base@^1.26.0": version "1.26.0" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.26.0.tgz#0c913bc6d2cfafd901de330e4540952269ae579c" @@ -1682,7 +1491,7 @@ "@opentelemetry/resources" "1.26.0" "@opentelemetry/semantic-conventions" "1.27.0" -"@opentelemetry/sdk-trace-node@1.26.0": +"@opentelemetry/sdk-trace-node@^1.26.0": version "1.26.0" resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.26.0.tgz#169ef4fc058e82a12460da18cedaf6e4615fc617" integrity sha512-Fj5IVKrj0yeUwlewCRwzOVcr5avTuNnMHWf7GPc1t6WaT78J6CJyF3saZ/0RkZfdeNO8IcBl/bNcWMVZBMRW8Q== @@ -1699,59 +1508,6 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz#1a857dcc95a5ab30122e04417148211e6f945e6c" integrity sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg== -"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" - integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== - -"@protobufjs/base64@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" - integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== - -"@protobufjs/codegen@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" - integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== - -"@protobufjs/eventemitter@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" - integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== - -"@protobufjs/fetch@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" - integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== - dependencies: - "@protobufjs/aspromise" "^1.1.1" - "@protobufjs/inquire" "^1.1.0" - -"@protobufjs/float@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" - integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== - -"@protobufjs/inquire@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" - integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== - -"@protobufjs/path@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" - integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== - -"@protobufjs/pool@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" - integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== - -"@protobufjs/utf8@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" - integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== - "@sinclair/typebox@^0.25.16": version "0.25.24" resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz" @@ -1891,13 +1647,6 @@ resolved "https://registry.npmjs.org/@types/node/-/node-20.2.5.tgz" integrity sha512-JJulVEQXmiY9Px5axXHeYGLSjhkZEnD+MDPDGbCbIAbMslkKwmygtZFy1X6s/075Yo94sf8GuSlFfPzysQrWZQ== -"@types/node@>=13.7.0": - version "22.7.7" - resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.7.tgz#6cd9541c3dccb4f7e8b141b491443f4a1570e307" - integrity sha512-SRxCrrg9CL/y54aiMCG3edPKdprgMVGDXjA3gB8UmmBW5TcXzRUYAh8EWzTnSJFAd1rgImPELza+A3bJ+qxz8Q== - dependencies: - undici-types "~6.19.2" - "@types/node@^18.11.18": version "18.19.17" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.17.tgz#a581a9fb4b2cfdbc61f008804f4436b2d5c40354" @@ -1925,11 +1674,6 @@ resolved "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz" integrity sha512-G8hZ6XJiHnuhQKR7ZmysCeJWE08o8T0AXtk5darsCaTVsYZhhgUrq53jizaR2FvsoeCwJhlmwTjkXBY5Pn/ZHw== -"@types/shimmer@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@types/shimmer/-/shimmer-1.2.0.tgz#9b706af96fa06416828842397a70dfbbf1c14ded" - integrity sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg== - "@types/stack-utils@^2.0.0": version "2.0.1" resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.1.tgz" @@ -2043,11 +1787,6 @@ abort-controller@^3.0.0: dependencies: event-target-shim "^5.0.0" -acorn-import-attributes@^1.9.5: - version "1.9.5" - resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" - integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== - acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" @@ -2063,11 +1802,6 @@ acorn@^8.4.1, acorn@^8.8.0: resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== -acorn@^8.8.2: - version "8.13.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.13.0.tgz#2a30d670818ad16ddd6a35d3842dacec9e5d7ca3" - integrity sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w== - agentkeepalive@^4.2.1: version "4.5.0" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" @@ -2429,11 +2163,6 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz" integrity sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA== -cjs-module-lexer@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz#707413784dbb3a72aa11c2f2b042a0bef4004170" - integrity sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA== - client-only@^0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/client-only/-/client-only-0.0.1.tgz#38bba5d403c41ab150bff64a95c85013cf73bca1" @@ -2551,13 +2280,6 @@ debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: dependencies: ms "2.1.2" -debug@^4.3.5: - version "4.3.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52" - integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ== - dependencies: - ms "^2.1.3" - decamelize@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -3312,7 +3034,7 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" -hasown@^2.0.0, hasown@^2.0.2: +hasown@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== @@ -3349,16 +3071,6 @@ import-fresh@^3.0.0, import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" -import-in-the-middle@^1.8.1: - version "1.11.2" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.11.2.tgz#dd848e72b63ca6cd7c34df8b8d97fc9baee6174f" - integrity sha512-gK6Rr6EykBcc6cVWRSBR5TWf8nn6hZMYSRYqCcHa0l0d1fPK7JSYo6+Mlmck76jIX9aL/IZ71c06U2VpFwl1zA== - dependencies: - acorn "^8.8.2" - acorn-import-attributes "^1.9.5" - cjs-module-lexer "^1.2.2" - module-details-from-path "^1.0.3" - import-local@^3.0.2: version "3.1.0" resolved "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz" @@ -3435,13 +3147,6 @@ is-core-module@^2.11.0: dependencies: has "^1.0.3" -is-core-module@^2.13.0: - version "2.15.1" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" - integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== - dependencies: - hasown "^2.0.2" - is-date-object@^1.0.1: version "1.0.5" resolved "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz" @@ -4103,11 +3808,6 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" -lodash.camelcase@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" - integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== - lodash.debounce@^4.0.8: version "4.0.8" resolved "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz" @@ -4123,11 +3823,6 @@ lodash.merge@^4.6.2: resolved "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -long@^5.0.0: - version "5.2.3" - resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1" - integrity sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q== - lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" @@ -4208,17 +3903,12 @@ minimist@^1.2.0, minimist@^1.2.6: resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== -module-details-from-path@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" - integrity sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A== - ms@2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -4529,24 +4219,6 @@ prompts@^2.0.1: kleur "^3.0.3" sisteransi "^1.0.5" -protobufjs@^7.2.5, protobufjs@^7.3.0: - version "7.4.0" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.4.0.tgz#7efe324ce9b3b61c82aae5de810d287bc08a248a" - integrity sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw== - dependencies: - "@protobufjs/aspromise" "^1.1.2" - "@protobufjs/base64" "^1.1.2" - "@protobufjs/codegen" "^2.0.4" - "@protobufjs/eventemitter" "^1.1.0" - "@protobufjs/fetch" "^1.1.0" - "@protobufjs/float" "^1.0.2" - "@protobufjs/inquire" "^1.1.0" - "@protobufjs/path" "^1.1.2" - "@protobufjs/pool" "^1.1.0" - "@protobufjs/utf8" "^1.1.0" - "@types/node" ">=13.7.0" - long "^5.0.0" - punycode@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz" @@ -4631,15 +4303,6 @@ require-directory@^2.1.1: resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== -require-in-the-middle@^7.1.1: - version "7.4.0" - resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-7.4.0.tgz#606977820d4b5f9be75e5a108ce34cfed25b3bb4" - integrity sha512-X34iHADNbNDfr6OTStIAHWSAvvKQRYgLO6duASaVf7J2VA3lvmNYboAHOuLC2huav1IwgZJtyEcJCKVzFxOSMQ== - dependencies: - debug "^4.3.5" - module-details-from-path "^1.0.3" - resolve "^1.22.8" - resolve-cwd@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz" @@ -4671,15 +4334,6 @@ resolve@^1.14.2, resolve@^1.20.0, resolve@^1.22.1: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -resolve@^1.22.8: - version "1.22.8" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" - integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== - dependencies: - is-core-module "^2.13.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - retry@^0.13.1: version "0.13.1" resolved "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz" @@ -4759,11 +4413,6 @@ shebang-regex@^3.0.0: resolved "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -shimmer@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337" - integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw== - side-channel@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz" @@ -5087,11 +4736,6 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== -undici-types@~6.19.2: - version "6.19.8" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" - integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== - unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz" @@ -5261,7 +4905,7 @@ yargs-parser@^21.0.1, yargs-parser@^21.1.1: resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== -yargs@^17.3.1, yargs@^17.7.2: +yargs@^17.3.1: version "17.7.2" resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== From b0cb0524508856ae1fe315f958319058d98f750e Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Tue, 22 Oct 2024 19:42:27 +0200 Subject: [PATCH 11/36] Add warning --- js/src/wrappers/vercel/exporter.ts | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 88fb0326a..8820c143f 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -467,29 +467,12 @@ export class LangSmithAISDKExporter implements SpanExporter { }); } - case "ai.embed": { - return asRunCreate({ - run_type: "chain", - name: span.attributes["ai.model.provider"], - inputs: { value: tryJson(span.attributes["ai.value"]) }, - outputs: { embedding: tryJson(span.attributes["ai.embedding"]) }, - }); - } + case "ai.embed": case "ai.embed.doEmbed": case "ai.embedMany": - case "ai.embedMany.doEmbed": { - return asRunCreate({ - run_type: "chain", - name: span.attributes["ai.model.provider"], - inputs: { values: span.attributes["ai.values"].map(tryJson) }, - outputs: { - embeddings: span.attributes["ai.embeddings"].map(tryJson), - }, - }); - } - + case "ai.embedMany.doEmbed": default: - assertNever(span); + console.warn(`Span "${span.name}" is currently unsupported.`); } } From 19bdf65432cdc93626ed523e765e2b9be65bd2c3 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 00:49:01 +0200 Subject: [PATCH 12/36] Remove embed --- js/src/wrappers/vercel/exporter.ts | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 8820c143f..69afcccf7 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -6,10 +6,6 @@ import { Client } from "../../index.js"; import { KVMap, RunCreate } from "../../schemas.js"; import { v5 as uuid5 } from "uuid"; -function assertNever(x: never): never { - throw new Error("Unreachable state: " + x); -} - // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; @@ -205,7 +201,7 @@ function convertToTimestamp([seconds, nanoseconds]: [ return Number(String(seconds) + ms); } -const RUN_ID_NS = "5c718b20-9078-11ef-9a3d-325096b39f47"; +const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47"; interface RunTask { id: string; @@ -232,10 +228,10 @@ export class LangSmithAISDKExporter implements SpanExporter { this.client = args?.client ?? new Client(); } - protected getRunCreate(span: AISDKSpan): RunCreate { - const runId = uuid5(span.spanContext().spanId, RUN_ID_NS); + protected getRunCreate(span: AISDKSpan): RunCreate | undefined { + const runId = uuid5(span.spanContext().spanId, RUN_ID_NAMESPACE); const parentRunId = span.parentSpanId - ? uuid5(span.parentSpanId, RUN_ID_NS) + ? uuid5(span.parentSpanId, RUN_ID_NAMESPACE) : undefined; const asRunCreate = (rawConfig: RunCreate) => { @@ -473,6 +469,7 @@ export class LangSmithAISDKExporter implements SpanExporter { case "ai.embedMany.doEmbed": default: console.warn(`Span "${span.name}" is currently unsupported.`); + return undefined; } } @@ -489,21 +486,25 @@ export class LangSmithAISDKExporter implements SpanExporter { relativeExecutionOrder: {}, }; - const runId = uuid5(spanId, RUN_ID_NS); - const parentRunId = parentId ? uuid5(parentId, RUN_ID_NS) : undefined; + const runId = uuid5(spanId, RUN_ID_NAMESPACE); + const parentRunId = parentId + ? uuid5(parentId, RUN_ID_NAMESPACE) + : undefined; const traceMap = this.traceByMap[traceId]; + const aiSpan = span as AISDKSpan; + const run = this.getRunCreate(aiSpan); + if (!run) continue; + traceMap.relativeExecutionOrder[parentRunId ?? "$"] ??= -1; traceMap.relativeExecutionOrder[parentRunId ?? "$"] += 1; - const aiSpan = span as AISDKSpan; - traceMap.nodeMap[runId] ??= { id: runId, parentId: parentRunId, startTime: span.startTime, - run: this.getRunCreate(aiSpan), + run, sent: false, executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? "$"], }; From aab80c7387c18115f20bd4e472195a8e235c4622 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 00:59:02 +0200 Subject: [PATCH 13/36] Strip internal methods and types --- js/src/wrappers/vercel/exporter.ts | 1 + js/src/wrappers/vercel/exporter.types.ts | 27 ++++++++++++------------ js/tsconfig.json | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 69afcccf7..7a7741557 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -228,6 +228,7 @@ export class LangSmithAISDKExporter implements SpanExporter { this.client = args?.client ?? new Client(); } + /** @internal */ protected getRunCreate(span: AISDKSpan): RunCreate | undefined { const runId = uuid5(span.spanContext().spanId, RUN_ID_NAMESPACE); const parentRunId = span.parentSpanId diff --git a/js/src/wrappers/vercel/exporter.types.ts b/js/src/wrappers/vercel/exporter.types.ts index 95315ae09..165611454 100644 --- a/js/src/wrappers/vercel/exporter.types.ts +++ b/js/src/wrappers/vercel/exporter.types.ts @@ -35,7 +35,7 @@ interface BaseEmbedSpanAttributes { "resource.name"?: string; } -export type ToolCallSpan = TypedReadableSpan< +type ToolCallSpan = TypedReadableSpan< "ai.toolCall", { "operation.name": "ai.toolCall"; @@ -47,7 +47,7 @@ export type ToolCallSpan = TypedReadableSpan< } >; -export type GenerateTextSpan = TypedReadableSpan< +type GenerateTextSpan = TypedReadableSpan< "ai.generateText", BaseLLMSpanAttributes & { "operation.name": "ai.generateText"; @@ -60,7 +60,7 @@ export type GenerateTextSpan = TypedReadableSpan< } >; -export type DoGenerateTextSpan = TypedReadableSpan< +type DoGenerateTextSpan = TypedReadableSpan< "ai.generateText.doGenerate", CallLLMSpanAttributes & { "operation.name": "ai.generateText.doGenerate"; @@ -73,7 +73,7 @@ export type DoGenerateTextSpan = TypedReadableSpan< } >; -export type StreamTextSpan = TypedReadableSpan< +type StreamTextSpan = TypedReadableSpan< "ai.streamText", BaseLLMSpanAttributes & { "operation.name": "ai.streamText"; @@ -86,7 +86,7 @@ export type StreamTextSpan = TypedReadableSpan< } >; -export type DoStreamTextSpan = TypedReadableSpan< +type DoStreamTextSpan = TypedReadableSpan< "ai.streamText.doStream", CallLLMSpanAttributes & { "operation.name": "ai.streamText.doStream"; @@ -102,7 +102,7 @@ export type DoStreamTextSpan = TypedReadableSpan< } >; -export type GenerateObjectSpan = TypedReadableSpan< +type GenerateObjectSpan = TypedReadableSpan< "ai.generateObject", BaseLLMSpanAttributes & { "operation.name": "ai.generateObject"; @@ -119,7 +119,7 @@ export type GenerateObjectSpan = TypedReadableSpan< "ai.settings.output": "object" | "no-schema" | AnyString; } >; -export type DoGenerateObjectSpan = TypedReadableSpan< +type DoGenerateObjectSpan = TypedReadableSpan< "ai.generateObject.doGenerate", CallLLMSpanAttributes & { "operation.name": "ai.generateObject.doGenerate"; @@ -136,7 +136,7 @@ export type DoGenerateObjectSpan = TypedReadableSpan< } >; -export type StreamObjectSpan = TypedReadableSpan< +type StreamObjectSpan = TypedReadableSpan< "ai.streamObject", BaseLLMSpanAttributes & { "operation.name": "ai.streamObject"; @@ -153,7 +153,7 @@ export type StreamObjectSpan = TypedReadableSpan< "ai.settings.output": "object" | "no-schema" | AnyString; } >; -export type DoStreamObjectSpan = TypedReadableSpan< +type DoStreamObjectSpan = TypedReadableSpan< "ai.streamObject.doStream", CallLLMSpanAttributes & { "operation.name": "ai.streamObject.doStream"; @@ -170,7 +170,7 @@ export type DoStreamObjectSpan = TypedReadableSpan< } >; -export type EmbedSpan = TypedReadableSpan< +type EmbedSpan = TypedReadableSpan< "ai.embed", BaseEmbedSpanAttributes & { "operation.name": "ai.embed"; @@ -181,7 +181,7 @@ export type EmbedSpan = TypedReadableSpan< } >; -export type DoEmbedSpan = TypedReadableSpan< +type DoEmbedSpan = TypedReadableSpan< "ai.embed.doEmbed", BaseEmbedSpanAttributes & { "operation.name": "ai.embed.doEmbed"; @@ -192,7 +192,7 @@ export type DoEmbedSpan = TypedReadableSpan< } >; -export type EmbedManySpan = TypedReadableSpan< +type EmbedManySpan = TypedReadableSpan< "ai.embedMany", BaseEmbedSpanAttributes & { "operation.name": "ai.embedMany"; @@ -203,7 +203,7 @@ export type EmbedManySpan = TypedReadableSpan< } >; -export type DoEmbedManySpan = TypedReadableSpan< +type DoEmbedManySpan = TypedReadableSpan< "ai.embedMany.doEmbed", BaseEmbedSpanAttributes & { "operation.name": "ai.embedMany.doEmbed"; @@ -214,6 +214,7 @@ export type DoEmbedManySpan = TypedReadableSpan< } >; +/** @internal */ export type AISDKSpan = | ToolCallSpan | GenerateTextSpan diff --git a/js/tsconfig.json b/js/tsconfig.json index ab24d6247..a58d12a7b 100644 --- a/js/tsconfig.json +++ b/js/tsconfig.json @@ -17,6 +17,7 @@ "noUnusedParameters": true, "useDefineForClassFields": true, "strictPropertyInitialization": false, + "stripInternal": true, "allowJs": true, "strict": true, "outDir": "dist" From 3a37732617528070bfa1df7b15b3fbc7c8143e41 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 01:09:15 +0200 Subject: [PATCH 14/36] Remove the type dependency --- js/package.json | 1 - js/src/wrappers/vercel/exporter.ts | 12 ++++++------ js/yarn.lock | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/js/package.json b/js/package.json index 0e64b4e93..488285d6a 100644 --- a/js/package.json +++ b/js/package.json @@ -112,7 +112,6 @@ "@langchain/core": "^0.3.1", "@langchain/langgraph": "^0.2.3", "@langchain/openai": "^0.3.0", - "@opentelemetry/api": "^1.9.0", "@opentelemetry/sdk-trace-base": "^1.26.0", "@opentelemetry/sdk-trace-node": "^1.26.0", "@tsconfig/recommended": "^1.0.2", diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 7a7741557..6cd46b2a6 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -1,5 +1,4 @@ -import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; -import type { ExportResult } from "@opentelemetry/core"; +import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; import type { AISDKSpan } from "./exporter.types.js"; import { Client } from "../../index.js"; @@ -212,7 +211,7 @@ interface RunTask { executionOrder: number; } -export class LangSmithAISDKExporter implements SpanExporter { +export class LangSmithAISDKExporter { private client: Client; private traceByMap: Record< string, @@ -475,10 +474,11 @@ export class LangSmithAISDKExporter implements SpanExporter { } export( - spans: ReadableSpan[], - resultCallback: (result: ExportResult) => void + spans: unknown[], + resultCallback: (result: { code: 0 | 1; error?: Error }) => void ): void { - for (const span of spans) { + const typedSpans = spans as ReadableSpan[]; + for (const span of typedSpans) { const { traceId, spanId } = span.spanContext(); const parentId = span.parentSpanId ?? undefined; this.traceByMap[traceId] ??= { diff --git a/js/yarn.lock b/js/yarn.lock index 31411bc6d..bf35ed96d 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1443,7 +1443,7 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" -"@opentelemetry/api@1.9.0", "@opentelemetry/api@^1.9.0": +"@opentelemetry/api@1.9.0": version "1.9.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== From d4a2af4f6d9170158e91540dba5570c25fa5289b Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 01:16:14 +0200 Subject: [PATCH 15/36] remove embed from tests --- js/src/tests/vercel_exporter.int.test.ts | 43 +----------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel_exporter.int.test.ts index 72d4e6dba..1de9d963e 100644 --- a/js/src/tests/vercel_exporter.int.test.ts +++ b/js/src/tests/vercel_exporter.int.test.ts @@ -5,14 +5,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; import { z } from "zod"; import { LangSmithAISDKExporter } from "../wrappers/vercel.js"; import { v4 as uuid } from "uuid"; -import { - generateText, - streamText, - generateObject, - streamObject, - embed, - embedMany, -} from "ai"; +import { generateText, streamText, generateObject, streamObject } from "ai"; import { tool } from "ai"; import { gatherIterator } from "./utils/iterator.js"; import { Client } from "../index.js"; @@ -185,40 +178,6 @@ test("streamObject", async () => { expect(storedRun.id).toEqual(traceId); }); -test("embed", async () => { - const traceId = uuid(); - await embed({ - model: openai.embedding("text-embedding-3-small"), - value: "prague castle at sunset", - experimental_telemetry: getTelemetrySettings(traceId), - }); - - await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); - - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); -}); - -test("embedMany", async () => { - const traceId = uuid(); - await embedMany({ - model: openai.embedding("text-embedding-3-small"), - values: [ - "a peaceful meadow with wildflowers", - "bustling city street at rush hour", - "prague castle at sunset", - ], - experimental_telemetry: getTelemetrySettings(traceId), - }); - - await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); - - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); -}); - afterAll(async () => { await provider.shutdown(); }); From 0fd9611c28e4c8257fe600a5e2af7dd5301e022a Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 02:09:13 +0200 Subject: [PATCH 16/36] Added handoff between traceable and telemetry --- js/src/tests/vercel_exporter.int.test.ts | 56 +++++++++- js/src/wrappers/vercel/exporter.ts | 125 +++++++++++++++++------ 2 files changed, 149 insertions(+), 32 deletions(-) diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel_exporter.int.test.ts index 1de9d963e..f3f6616ca 100644 --- a/js/src/tests/vercel_exporter.int.test.ts +++ b/js/src/tests/vercel_exporter.int.test.ts @@ -10,14 +10,15 @@ import { tool } from "ai"; import { gatherIterator } from "./utils/iterator.js"; import { Client } from "../index.js"; import { waitUntilRunFound } from "./utils.js"; +import { getCurrentRunTree, traceable } from "../traceable.js"; -const getTelemetrySettings = (langsmithRunId?: string) => { +const getTelemetrySettings = (runId?: string) => { const metadata: Record = { userId: "123", language: "english", }; - if (langsmithRunId) metadata.langsmithRunId = langsmithRunId; + if (runId) metadata["langsmith:runId"] = runId; return { isEnabled: true, functionId: "functionId", @@ -178,6 +179,57 @@ test("streamObject", async () => { expect(storedRun.id).toEqual(traceId); }); +test("traceable", async () => { + const runId = uuid(); + + const wrappedText = traceable( + async (content: string) => { + const runTree = getCurrentRunTree(); + const headers = runTree.toHeaders(); + + const telemetry = getTelemetrySettings(); + + const { text } = await generateText({ + model: openai("gpt-4o-mini"), + messages: [{ role: "user", content }], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => + `User ${userId} has the following orders: 1`, + }), + viewTrackingInformation: tool({ + description: "view tracking information for a specific order", + parameters: z.object({ orderId: z.string() }), + execute: async ({ orderId }) => + `Here is the tracking information for ${orderId}`, + }), + }, + experimental_telemetry: { + ...telemetry, + metadata: { + ...telemetry.metadata, + "langsmith:trace": headers["langsmith-trace"], + "langsmith:baggage": headers["baggage"], + }, + }, + maxSteps: 10, + }); + + return { text }; + }, + { name: "wrappedText", id: runId } + ); + + const result = await wrappedText( + "What are my orders and where are they? My user ID is 123" + ); + await waitUntilRunFound(client, runId, true); + const storedRun = await client.readRun(runId); + expect(storedRun.outputs).toEqual(result); +}); + afterAll(async () => { await provider.shutdown(); }); diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/wrappers/vercel/exporter.ts index 6cd46b2a6..bb75fb6d6 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/wrappers/vercel/exporter.ts @@ -1,7 +1,6 @@ -import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; import type { AISDKSpan } from "./exporter.types.js"; -import { Client } from "../../index.js"; +import { Client, RunTree } from "../../index.js"; import { KVMap, RunCreate } from "../../schemas.js"; import { v5 as uuid5 } from "uuid"; @@ -202,6 +201,16 @@ function convertToTimestamp([seconds, nanoseconds]: [ const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47"; +const RUN_ID_METADATA_KEY = "ai.telemetry.metadata.langsmith:runId"; +const TRACE_METADATA_KEY = "ai.telemetry.metadata.langsmith:trace"; +const BAGGAGE_METADATA_KEY = "ai.telemetry.metadata.langsmith:baggage"; + +const RESERVED_METADATA_KEYS = [ + RUN_ID_METADATA_KEY, + TRACE_METADATA_KEY, + BAGGAGE_METADATA_KEY, +]; + interface RunTask { id: string; parentId: string | undefined; @@ -211,6 +220,11 @@ interface RunTask { executionOrder: number; } +type InteropType = + | { type: "traceable"; parentRunTree: RunTree } + | { type: "manual"; userTraceId: string } + | undefined; + export class LangSmithAISDKExporter { private client: Client; private traceByMap: Record< @@ -219,7 +233,7 @@ export class LangSmithAISDKExporter { childMap: Record; nodeMap: Record; relativeExecutionOrder: Record; - userTraceId?: string; + interop?: InteropType; } > = {}; @@ -227,6 +241,49 @@ export class LangSmithAISDKExporter { this.client = args?.client ?? new Client(); } + /** @internal */ + protected parseInteropFromMetadata(span: AISDKSpan): InteropType { + let userTraceId: string | undefined = undefined; + + if ( + RUN_ID_METADATA_KEY in span.attributes && + typeof span.attributes[RUN_ID_METADATA_KEY] === "string" && + span.attributes[RUN_ID_METADATA_KEY] + ) { + userTraceId = span.attributes[RUN_ID_METADATA_KEY]; + } + + if ( + TRACE_METADATA_KEY in span.attributes && + typeof span.attributes[TRACE_METADATA_KEY] === "string" && + span.attributes[TRACE_METADATA_KEY] + ) { + if (userTraceId) { + throw new Error( + "Cannot provide both `langsmith:runId` and `langsmith:trace` metadata keys." + ); + } + + const baggage = + BAGGAGE_METADATA_KEY in span.attributes && + typeof span.attributes[BAGGAGE_METADATA_KEY] === "string" + ? span.attributes[BAGGAGE_METADATA_KEY] + : ""; + + const parentRunTree = RunTree.fromHeaders({ + "langsmith-trace": span.attributes[TRACE_METADATA_KEY], + baggage, + }); + + if (!parentRunTree) + throw new Error("Unreachable code: empty parent run tree"); + return { type: "traceable", parentRunTree }; + } + + if (userTraceId) return { type: "manual", userTraceId }; + return undefined; + } + /** @internal */ protected getRunCreate(span: AISDKSpan): RunCreate | undefined { const runId = uuid5(span.spanContext().spanId, RUN_ID_NAMESPACE); @@ -239,7 +296,7 @@ export class LangSmithAISDKExporter { .filter( (key) => key.startsWith("ai.telemetry.metadata.") && - key !== "ai.telemetry.metadata.langsmithRunId" + !RESERVED_METADATA_KEYS.includes(key) ) .reduce((acc, key) => { acc[key.slice("ai.telemetry.metadata.".length)] = @@ -477,7 +534,7 @@ export class LangSmithAISDKExporter { spans: unknown[], resultCallback: (result: { code: 0 | 1; error?: Error }) => void ): void { - const typedSpans = spans as ReadableSpan[]; + const typedSpans = spans as AISDKSpan[]; for (const span of typedSpans) { const { traceId, spanId } = span.spanContext(); const parentId = span.parentSpanId ?? undefined; @@ -494,8 +551,7 @@ export class LangSmithAISDKExporter { const traceMap = this.traceByMap[traceId]; - const aiSpan = span as AISDKSpan; - const run = this.getRunCreate(aiSpan); + const run = this.getRunCreate(span); if (!run) continue; traceMap.relativeExecutionOrder[parentRunId ?? "$"] ??= -1; @@ -512,16 +568,7 @@ export class LangSmithAISDKExporter { traceMap.childMap[parentRunId ?? "$"] ??= []; traceMap.childMap[parentRunId ?? "$"].push(traceMap.nodeMap[runId]); - - if ( - "ai.telemetry.metadata.langsmithRunId" in aiSpan.attributes && - typeof aiSpan.attributes["ai.telemetry.metadata.langsmithRunId"] === - "string" && - aiSpan.attributes["ai.telemetry.metadata.langsmithRunId"] - ) { - traceMap.userTraceId = - aiSpan.attributes["ai.telemetry.metadata.langsmithRunId"]; - } + traceMap.interop = this.parseInteropFromMetadata(span); } // collect all subgraphs @@ -564,19 +611,37 @@ export class LangSmithAISDKExporter { trace_id: task.traceId, }; - if (traceMap.userTraceId) { - ident = { - id: ident.id === ident.trace_id ? traceMap.userTraceId : ident.id, - parent_run_id: - ident.parent_run_id === ident.trace_id - ? traceMap.userTraceId - : ident.parent_run_id, - dotted_order: ident.dotted_order.replace( - ident.trace_id, - traceMap.userTraceId - ), - trace_id: traceMap.userTraceId, - }; + if (traceMap.interop) { + if (traceMap.interop.type === "traceable") { + ident = { + id: ident.id, + parent_run_id: + ident.parent_run_id ?? traceMap.interop.parentRunTree.id, + dotted_order: [ + traceMap.interop.parentRunTree.dotted_order, + ident.dotted_order, + ] + .filter(Boolean) + .join("."), + trace_id: traceMap.interop.parentRunTree.trace_id, + }; + } else if (traceMap.interop.type === "manual") { + ident = { + id: + ident.id === ident.trace_id + ? traceMap.interop.userTraceId + : ident.id, + parent_run_id: + ident.parent_run_id === ident.trace_id + ? traceMap.interop.userTraceId + : ident.parent_run_id, + dotted_order: ident.dotted_order.replace( + ident.trace_id, + traceMap.interop.userTraceId + ), + trace_id: traceMap.interop.userTraceId, + }; + } } sampled.push([ident, task.item.run]); From 7a3424dd6cb1d70287c8bc75006c8f0d10a8a087 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 03:11:32 +0200 Subject: [PATCH 17/36] Add as new entrypoint --- js/.gitignore | 4 + js/package.json | 13 ++ js/scripts/create-entrypoints.js | 1 + js/src/tests/utils/iterator.ts | 7 - ...xporter.int.test.ts => vercel.int.test.ts} | 130 +++++++++--------- .../vercel/exporter.ts => vercel.ts} | 121 +++++++++++----- .../exporter.types.ts => vercel.types.ts} | 0 js/src/wrappers/vercel.ts | 2 - js/tsconfig.json | 1 + 9 files changed, 171 insertions(+), 108 deletions(-) delete mode 100644 js/src/tests/utils/iterator.ts rename js/src/tests/{vercel_exporter.int.test.ts => vercel.int.test.ts} (67%) rename js/src/{wrappers/vercel/exporter.ts => vercel.ts} (87%) rename js/src/{wrappers/vercel/exporter.types.ts => vercel.types.ts} (100%) diff --git a/js/.gitignore b/js/.gitignore index e758389d2..4b11d6959 100644 --- a/js/.gitignore +++ b/js/.gitignore @@ -59,6 +59,10 @@ Chinook_Sqlite.sql /langchain.js /langchain.d.ts /langchain.d.cts +/vercel.cjs +/vercel.js +/vercel.d.ts +/vercel.d.cts /wrappers.cjs /wrappers.js /wrappers.d.ts diff --git a/js/package.json b/js/package.json index 488285d6a..53d29968c 100644 --- a/js/package.json +++ b/js/package.json @@ -33,6 +33,10 @@ "langchain.js", "langchain.d.ts", "langchain.d.cts", + "vercel.cjs", + "vercel.js", + "vercel.d.ts", + "vercel.d.cts", "wrappers.cjs", "wrappers.js", "wrappers.d.ts", @@ -223,6 +227,15 @@ "import": "./langchain.js", "require": "./langchain.cjs" }, + "./vercel": { + "types": { + "import": "./vercel.d.ts", + "require": "./vercel.d.cts", + "default": "./vercel.d.ts" + }, + "import": "./vercel.js", + "require": "./vercel.cjs" + }, "./wrappers": { "types": { "import": "./wrappers.d.ts", diff --git a/js/scripts/create-entrypoints.js b/js/scripts/create-entrypoints.js index a3487f756..9cce2ab22 100644 --- a/js/scripts/create-entrypoints.js +++ b/js/scripts/create-entrypoints.js @@ -14,6 +14,7 @@ const entrypoints = { "evaluation/langchain": "evaluation/langchain", schemas: "schemas", langchain: "langchain", + vercel: "vercel", wrappers: "wrappers/index", anonymizer: "anonymizer/index", "wrappers/openai": "wrappers/openai", diff --git a/js/src/tests/utils/iterator.ts b/js/src/tests/utils/iterator.ts deleted file mode 100644 index 4734369ba..000000000 --- a/js/src/tests/utils/iterator.ts +++ /dev/null @@ -1,7 +0,0 @@ -export async function gatherIterator( - i: AsyncIterable | Promise> -): Promise> { - const out: T[] = []; - for await (const item of await i) out.push(item); - return out; -} diff --git a/js/src/tests/vercel_exporter.int.test.ts b/js/src/tests/vercel.int.test.ts similarity index 67% rename from js/src/tests/vercel_exporter.int.test.ts rename to js/src/tests/vercel.int.test.ts index f3f6616ca..8741ad4b4 100644 --- a/js/src/tests/vercel_exporter.int.test.ts +++ b/js/src/tests/vercel.int.test.ts @@ -1,42 +1,33 @@ -import { openai } from "@ai-sdk/openai"; - import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; -import { z } from "zod"; -import { LangSmithAISDKExporter } from "../wrappers/vercel.js"; + +import { + generateText, + streamText, + generateObject, + streamObject, + tool, +} from "ai"; +import { openai } from "@ai-sdk/openai"; + import { v4 as uuid } from "uuid"; -import { generateText, streamText, generateObject, streamObject } from "ai"; -import { tool } from "ai"; -import { gatherIterator } from "./utils/iterator.js"; +import { z } from "zod"; +import { AISDKExporter } from "../vercel.js"; import { Client } from "../index.js"; -import { waitUntilRunFound } from "./utils.js"; -import { getCurrentRunTree, traceable } from "../traceable.js"; - -const getTelemetrySettings = (runId?: string) => { - const metadata: Record = { - userId: "123", - language: "english", - }; - - if (runId) metadata["langsmith:runId"] = runId; - return { - isEnabled: true, - functionId: "functionId", - metadata, - }; -}; +import { traceable } from "../traceable.js"; +import { waitUntilRunFound, toArray } from "./utils.js"; const client = new Client(); // Not using @opentelemetry/sdk-node because we need to force flush // the spans to ensure they are sent to LangSmith between tests const provider = new NodeTracerProvider(); provider.addSpanProcessor( - new BatchSpanProcessor(new LangSmithAISDKExporter({ client })) + new BatchSpanProcessor(new AISDKExporter({ client })) ); provider.register(); test("generateText", async () => { - const traceId = uuid(); + const runId = uuid(); await generateText({ model: openai("gpt-4o-mini"), @@ -60,19 +51,23 @@ test("generateText", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), maxSteps: 10, }); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("generateText with image", async () => { - const traceId = uuid(); + const runId = uuid(); await generateText({ model: openai("gpt-4o-mini"), messages: [ @@ -90,18 +85,22 @@ test("generateText with image", async () => { ], }, ], - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), }); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("streamText", async () => { - const traceId = uuid(); + const runId = uuid(); const result = await streamText({ model: openai("gpt-4o-mini"), messages: [ @@ -124,20 +123,24 @@ test("streamText", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), maxSteps: 10, }); - await gatherIterator(result.fullStream); + await toArray(result.fullStream); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("generateObject", async () => { - const traceId = uuid(); + const runId = uuid(); await generateObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -147,18 +150,22 @@ test("generateObject", async () => { }), }), prompt: "What's the weather in Prague?", - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), }); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("streamObject", async () => { - const traceId = uuid(); + const runId = uuid(); const result = await streamObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ @@ -168,15 +175,19 @@ test("streamObject", async () => { }), }), prompt: "What's the weather in Prague?", - experimental_telemetry: getTelemetrySettings(traceId), + experimental_telemetry: AISDKExporter.getSettings({ + runId, + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), }); - await gatherIterator(result.partialObjectStream); + await toArray(result.partialObjectStream); await provider.forceFlush(); - await waitUntilRunFound(client, traceId, true); + await waitUntilRunFound(client, runId, true); - const storedRun = await client.readRun(traceId); - expect(storedRun.id).toEqual(traceId); + const storedRun = await client.readRun(runId); + expect(storedRun.id).toEqual(runId); }); test("traceable", async () => { @@ -184,11 +195,6 @@ test("traceable", async () => { const wrappedText = traceable( async (content: string) => { - const runTree = getCurrentRunTree(); - const headers = runTree.toHeaders(); - - const telemetry = getTelemetrySettings(); - const { text } = await generateText({ model: openai("gpt-4o-mini"), messages: [{ role: "user", content }], @@ -206,14 +212,10 @@ test("traceable", async () => { `Here is the tracking information for ${orderId}`, }), }, - experimental_telemetry: { - ...telemetry, - metadata: { - ...telemetry.metadata, - "langsmith:trace": headers["langsmith-trace"], - "langsmith:baggage": headers["baggage"], - }, - }, + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), maxSteps: 10, }); diff --git a/js/src/wrappers/vercel/exporter.ts b/js/src/vercel.ts similarity index 87% rename from js/src/wrappers/vercel/exporter.ts rename to js/src/vercel.ts index bb75fb6d6..e85db5ee7 100644 --- a/js/src/wrappers/vercel/exporter.ts +++ b/js/src/vercel.ts @@ -1,12 +1,28 @@ -import type { CoreAssistantMessage, CoreMessage, ToolCallPart } from "ai"; -import type { AISDKSpan } from "./exporter.types.js"; -import { Client, RunTree } from "../../index.js"; -import { KVMap, RunCreate } from "../../schemas.js"; +import type { + CoreAssistantMessage, + CoreMessage, + ToolCallPart, + generateText, +} from "ai"; +import type { AISDKSpan } from "./vercel.types.js"; +import { Client, RunTree } from "./index.js"; +import { KVMap, RunCreate } from "./schemas.js"; import { v5 as uuid5 } from "uuid"; +import { getCurrentRunTree } from "./singletons/traceable.js"; // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; +type AITelemetrySettings = Exclude< + Parameters[0]["experimental_telemetry"], + undefined +>; + +interface TelemetrySettings extends AITelemetrySettings { + /** ID of the run sent to LangSmith */ + runId?: string; +} + type LangChainMessageFields = { content: | string @@ -201,14 +217,25 @@ function convertToTimestamp([seconds, nanoseconds]: [ const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47"; -const RUN_ID_METADATA_KEY = "ai.telemetry.metadata.langsmith:runId"; -const TRACE_METADATA_KEY = "ai.telemetry.metadata.langsmith:trace"; -const BAGGAGE_METADATA_KEY = "ai.telemetry.metadata.langsmith:baggage"; +const RUN_ID_METADATA_KEY = { + input: "langsmith:runId", + output: "ai.telemetry.metadata.langsmith:runId", +}; + +const TRACE_METADATA_KEY = { + input: "langsmith:trace", + output: "ai.telemetry.metadata.langsmith:trace", +}; + +const BAGGAGE_METADATA_KEY = { + input: "langsmith:baggage", + output: "ai.telemetry.metadata.langsmith:baggage", +}; const RESERVED_METADATA_KEYS = [ - RUN_ID_METADATA_KEY, - TRACE_METADATA_KEY, - BAGGAGE_METADATA_KEY, + RUN_ID_METADATA_KEY.output, + TRACE_METADATA_KEY.output, + BAGGAGE_METADATA_KEY.output, ]; interface RunTask { @@ -225,7 +252,7 @@ type InteropType = | { type: "manual"; userTraceId: string } | undefined; -export class LangSmithAISDKExporter { +export class AISDKExporter { private client: Client; private traceByMap: Record< string, @@ -241,38 +268,62 @@ export class LangSmithAISDKExporter { this.client = args?.client ?? new Client(); } - /** @internal */ - protected parseInteropFromMetadata(span: AISDKSpan): InteropType { - let userTraceId: string | undefined = undefined; + static getSettings(settings: TelemetrySettings) { + const { runId, ...rest } = settings; + const metadata = { ...rest?.metadata }; + if (runId != null) metadata[RUN_ID_METADATA_KEY.input] = runId; + + // attempt to obtain the run tree if used within a traceable function + let defaultEnabled = true; + try { + const runTree = getCurrentRunTree(); + const headers = runTree.toHeaders(); + metadata[TRACE_METADATA_KEY.input] = headers["langsmith-trace"]; + metadata[BAGGAGE_METADATA_KEY.input] = headers["baggage"]; + + // honor the tracingEnabled flag if coming from traceable + if (runTree.tracingEnabled != null) { + defaultEnabled = runTree.tracingEnabled; + } + } catch { + // pass + } if ( - RUN_ID_METADATA_KEY in span.attributes && - typeof span.attributes[RUN_ID_METADATA_KEY] === "string" && - span.attributes[RUN_ID_METADATA_KEY] + metadata[RUN_ID_METADATA_KEY.input] && + metadata[TRACE_METADATA_KEY.input] ) { - userTraceId = span.attributes[RUN_ID_METADATA_KEY]; + throw new Error( + "Cannot provide `runId` when used within traceable function." + ); } - if ( - TRACE_METADATA_KEY in span.attributes && - typeof span.attributes[TRACE_METADATA_KEY] === "string" && - span.attributes[TRACE_METADATA_KEY] - ) { - if (userTraceId) { - throw new Error( - "Cannot provide both `langsmith:runId` and `langsmith:trace` metadata keys." - ); - } + return { ...rest, isEnabled: rest.isEnabled ?? defaultEnabled, metadata }; + } + + /** @internal */ + protected parseInteropFromMetadata(span: AISDKSpan): InteropType { + const getKey = (key: string): string | undefined => { + const attributes = span.attributes as Record; - const baggage = - BAGGAGE_METADATA_KEY in span.attributes && - typeof span.attributes[BAGGAGE_METADATA_KEY] === "string" - ? span.attributes[BAGGAGE_METADATA_KEY] - : ""; + return key in attributes && typeof attributes[key] === "string" + ? (attributes[key] as string) + : undefined; + }; + + const userTraceId = getKey(RUN_ID_METADATA_KEY.output) || undefined; + const parentTrace = getKey(TRACE_METADATA_KEY.output) || undefined; + + if (parentTrace && userTraceId) { + throw new Error( + `Cannot provide both "${RUN_ID_METADATA_KEY.input}" and "${TRACE_METADATA_KEY.input}" metadata keys.` + ); + } + if (parentTrace) { const parentRunTree = RunTree.fromHeaders({ - "langsmith-trace": span.attributes[TRACE_METADATA_KEY], - baggage, + "langsmith-trace": parentTrace, + baggage: getKey(BAGGAGE_METADATA_KEY.output) || "", }); if (!parentRunTree) diff --git a/js/src/wrappers/vercel/exporter.types.ts b/js/src/vercel.types.ts similarity index 100% rename from js/src/wrappers/vercel/exporter.types.ts rename to js/src/vercel.types.ts diff --git a/js/src/wrappers/vercel.ts b/js/src/wrappers/vercel.ts index e143647a4..dc022d7c8 100644 --- a/js/src/wrappers/vercel.ts +++ b/js/src/wrappers/vercel.ts @@ -107,5 +107,3 @@ export const wrapAISDKModel = ( }, }); }; - -export { LangSmithAISDKExporter } from "./vercel/exporter.js"; diff --git a/js/tsconfig.json b/js/tsconfig.json index a58d12a7b..b778ed83f 100644 --- a/js/tsconfig.json +++ b/js/tsconfig.json @@ -40,6 +40,7 @@ "src/evaluation/langchain.ts", "src/schemas.ts", "src/langchain.ts", + "src/vercel.ts", "src/wrappers/index.ts", "src/anonymizer/index.ts", "src/wrappers/openai.ts", From fdca4d9b726040dac86c9771406251915914ffee Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 18:26:11 +0200 Subject: [PATCH 18/36] Code review --- js/src/vercel.ts | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index e85db5ee7..a6a2a6a9a 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -27,20 +27,8 @@ type LangChainMessageFields = { content: | string | Array< - | { type: "text"; text: string } - | { - type: "image_url"; - image_url: - | string - | { - url: string; - detail?: "auto" | "low" | "high" | AnyString; - }; - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - | (Record & { type?: "text" | "image_url" | string }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - | (Record & { type?: never }) + Record & { type?: "text" | "image_url" | AnyString } >; name?: string; id?: string; @@ -215,6 +203,7 @@ function convertToTimestamp([seconds, nanoseconds]: [ return Number(String(seconds) + ms); } +const ROOT = "$"; const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47"; const RUN_ID_METADATA_KEY = { @@ -605,8 +594,8 @@ export class AISDKExporter { const run = this.getRunCreate(span); if (!run) continue; - traceMap.relativeExecutionOrder[parentRunId ?? "$"] ??= -1; - traceMap.relativeExecutionOrder[parentRunId ?? "$"] += 1; + traceMap.relativeExecutionOrder[parentRunId ?? ROOT] ??= -1; + traceMap.relativeExecutionOrder[parentRunId ?? ROOT] += 1; traceMap.nodeMap[runId] ??= { id: runId, @@ -614,11 +603,11 @@ export class AISDKExporter { startTime: span.startTime, run, sent: false, - executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? "$"], + executionOrder: traceMap.relativeExecutionOrder[parentRunId ?? ROOT], }; - traceMap.childMap[parentRunId ?? "$"] ??= []; - traceMap.childMap[parentRunId ?? "$"].push(traceMap.nodeMap[runId]); + traceMap.childMap[parentRunId ?? ROOT] ??= []; + traceMap.childMap[parentRunId ?? ROOT].push(traceMap.nodeMap[runId]); traceMap.interop = this.parseInteropFromMetadata(span); } @@ -638,7 +627,7 @@ export class AISDKExporter { const traceMap = this.traceByMap[traceId]; const queue: QueueItem[] = - traceMap.childMap["$"]?.map((item) => ({ + traceMap.childMap[ROOT]?.map((item) => ({ item, dottedOrder: convertToDottedOrderFormat( item.startTime, From 5c2d75307dcbd646c9d3010fb0244feabb60c39c Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 18:27:54 +0200 Subject: [PATCH 19/36] Handle first chunk event --- js/src/vercel.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index a6a2a6a9a..e00d822ed 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -447,12 +447,21 @@ export class AISDKExporter { return result; })(); + const events: KVMap[] = []; + if ("ai.response.msToFirstChunk" in span.attributes) { + events.push({ + type: "first_chunk", + time: span.attributes["ai.response.msToFirstChunk"], + }); + } + // TODO: add first_token_time return asRunCreate({ run_type: "llm", name: span.attributes["ai.model.provider"], inputs, outputs, + events, extra: { batch_size: 1, metadata: { @@ -466,7 +475,6 @@ export class AISDKExporter { }, }, }); - break; } case "ai.toolCall": { @@ -540,11 +548,20 @@ export class AISDKExporter { return result; })(); + const events: KVMap[] = []; + if ("ai.response.msToFirstChunk" in span.attributes) { + events.push({ + type: "first_chunk", + time: span.attributes["ai.response.msToFirstChunk"], + }); + } + return asRunCreate({ run_type: "llm", name: span.attributes["ai.model.provider"], inputs, outputs, + events, extra: { batch_size: 1, metadata: { From 73abeaead67074b77f8089691bfc643225d4addd Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 18:37:03 +0200 Subject: [PATCH 20/36] Fix formatting of first token events, avoid negative time stamp --- js/src/vercel.ts | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index e00d822ed..6158dce02 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -355,6 +355,9 @@ export class AISDKExporter { span.attributes["resource.name"]; } + const parsedStart = convertToTimestamp(span.startTime); + const parsedEnd = convertToTimestamp(span.endTime); + const config: RunCreate = { ...rawConfig, id: runId, @@ -367,8 +370,8 @@ export class AISDKExporter { "ai.operationId": span.attributes["ai.operationId"], }, }, - start_time: convertToTimestamp(span.startTime), - end_time: convertToTimestamp(span.endTime), + start_time: Math.min(parsedStart, parsedEnd), + end_time: Math.max(parsedStart, parsedEnd), }; return config; @@ -448,10 +451,13 @@ export class AISDKExporter { })(); const events: KVMap[] = []; - if ("ai.response.msToFirstChunk" in span.attributes) { + const firstChunkEvent = span.events.find( + (i) => i.name === "ai.stream.firstChunk" + ); + if (firstChunkEvent) { events.push({ - type: "first_chunk", - time: span.attributes["ai.response.msToFirstChunk"], + name: "new_token", + time: convertToTimestamp(firstChunkEvent.time), }); } @@ -549,10 +555,13 @@ export class AISDKExporter { })(); const events: KVMap[] = []; - if ("ai.response.msToFirstChunk" in span.attributes) { + const firstChunkEvent = span.events.find( + (i) => i.name === "ai.stream.firstChunk" + ); + if (firstChunkEvent) { events.push({ - type: "first_chunk", - time: span.attributes["ai.response.msToFirstChunk"], + name: "new_token", + time: convertToTimestamp(firstChunkEvent.time), }); } From 2c895493e65a18007de8f207cc66672b79267ded Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Wed, 23 Oct 2024 18:45:10 +0200 Subject: [PATCH 21/36] Improve rendering of intermediate tool calls --- js/src/vercel.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 6158dce02..9ab1e127f 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -415,11 +415,22 @@ export class AISDKExporter { const outputs = ((): KVMap | undefined => { let result: KVMap | undefined = undefined; + if (span.attributes["ai.response.toolCalls"]) { + let content = tryJson(span.attributes["ai.response.toolCalls"]); + + if (Array.isArray(content)) { + content = content.map((i) => ({ + type: "tool-call", + ...i, + args: tryJson(i.args), + })); + } + result = { llm_output: convertCoreToSmith({ role: "assistant", - content: tryJson(span.attributes["ai.response.toolCalls"]), + content, } satisfies CoreAssistantMessage), }; } else if (span.attributes["ai.response.text"]) { From c487e8559b7128a38a92a88aae1b79303ca9fa18 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Thu, 24 Oct 2024 15:04:14 +0200 Subject: [PATCH 22/36] Add unit tests with input/output fixtures --- js/src/tests/vercel.test.ts | 898 ++++++++++++++++++++++++++++++++++++ js/src/vercel.ts | 13 +- 2 files changed, 910 insertions(+), 1 deletion(-) create mode 100644 js/src/tests/vercel.test.ts diff --git a/js/src/tests/vercel.test.ts b/js/src/tests/vercel.test.ts new file mode 100644 index 000000000..15ddb2f3f --- /dev/null +++ b/js/src/tests/vercel.test.ts @@ -0,0 +1,898 @@ +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; +import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base"; + +import { + generateText, + streamText, + generateObject, + streamObject, + tool, + LanguageModelV1StreamPart, +} from "ai"; + +import { z } from "zod"; +import { AISDKExporter } from "../vercel.js"; +import { traceable } from "../traceable.js"; +import { toArray } from "./utils.js"; +import { mockClient } from "./utils/mock_client.js"; +import { convertArrayToReadableStream, MockLanguageModelV1 } from "ai/test"; +import { getAssumedTreeFromCalls } from "./utils/tree.js"; + +const { client, callSpy } = mockClient(); +const provider = new NodeTracerProvider(); +provider.addSpanProcessor( + new BatchSpanProcessor(new AISDKExporter({ client })) +); +provider.register(); + +class ExecutionOrderSame { + $$typeof = Symbol.for("jest.asymmetricMatcher"); + + private expectedNs: string; + private expectedDepth: number; + + constructor(depth: number, ns: string) { + this.expectedDepth = depth; + this.expectedNs = ns; + } + + asymmetricMatch(other: unknown) { + // eslint-disable-next-line no-instanceof/no-instanceof + if (!(typeof other === "string" || other instanceof String)) { + return false; + } + + const segments = other.split("."); + if (segments.length !== this.expectedDepth) return false; + + const last = segments.at(-1); + if (!last) return false; + + const nanoseconds = last.split("Z").at(0)?.slice(-3); + return nanoseconds === this.expectedNs; + } + + toString() { + return "ExecutionOrderSame"; + } + + getExpectedType() { + return "string"; + } + + toAsymmetricMatcher() { + return `ExecutionOrderSame<${this.expectedDepth}, ${this.expectedNs}>`; + } +} + +class MockMultiStepLanguageModelV1 extends MockLanguageModelV1 { + generateStep = -1; + streamStep = -1; + + constructor(...args: ConstructorParameters) { + super(...args); + + const oldDoGenerate = this.doGenerate; + this.doGenerate = async (...args) => { + this.generateStep += 1; + return await oldDoGenerate(...args); + }; + + const oldDoStream = this.doStream; + this.doStream = async (...args) => { + this.streamStep += 1; + return await oldDoStream(...args); + }; + } +} + +beforeEach(() => callSpy.mockClear()); +afterAll(async () => await provider.shutdown()); + +test("generateText", async () => { + const model = new MockMultiStepLanguageModelV1({ + doGenerate: async () => { + if (model.generateStep === 0) { + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + toolCalls: [ + { + toolCallType: "function", + toolName: "listOrders", + toolCallId: "tool-id", + args: JSON.stringify({ userId: "123" }), + }, + ], + }; + } + + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + text: `Hello, world!`, + }; + }, + }); + + await generateText({ + model, + messages: [ + { + role: "user", + content: "What are my orders? My user ID is 123", + }, + ], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => + `User ${userId} has the following orders: 1`, + }), + }, + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), + maxSteps: 10, + }); + + await provider.forceFlush(); + expect(getAssumedTreeFromCalls(callSpy.mock.calls)).toMatchObject({ + nodes: [ + "mock-provider:0", + "mock-provider:1", + "listOrders:2", + "mock-provider:3", + ], + edges: [ + ["mock-provider:0", "mock-provider:1"], + ["mock-provider:0", "listOrders:2"], + ["mock-provider:0", "mock-provider:3"], + ], + data: { + "mock-provider:0": { + inputs: { + messages: [ + { + type: "human", + data: { content: "What are my orders? My user ID is 123" }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { content: "Hello, world!" }, + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(1, "000"), + }, + "mock-provider:1": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { + type: "text", + text: "What are my orders? My user ID is 123", + }, + ], + }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { + content: [ + { + type: "tool_use", + name: "listOrders", + id: "tool-id", + input: { userId: "123" }, + }, + ], + additional_kwargs: { + tool_calls: [ + { + id: "tool-id", + type: "function", + function: { + name: "listOrders", + id: "tool-id", + arguments: '{"userId":"123"}', + }, + }, + ], + }, + }, + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "000"), + }, + "listOrders:2": { + inputs: { userId: "123" }, + outputs: { output: "User 123 has the following orders: 1" }, + dotted_order: new ExecutionOrderSame(2, "001"), + }, + "mock-provider:3": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { + type: "text", + text: "What are my orders? My user ID is 123", + }, + ], + }, + }, + { + type: "ai", + data: { + content: [ + { + type: "tool_use", + name: "listOrders", + id: "tool-id", + input: { userId: "123" }, + }, + ], + additional_kwargs: { + tool_calls: [ + { + id: "tool-id", + type: "function", + function: { + name: "listOrders", + id: "tool-id", + arguments: '{"userId":"123"}', + }, + }, + ], + }, + }, + }, + { + type: "tool", + data: { + content: '"User 123 has the following orders: 1"', + name: "listOrders", + tool_call_id: "tool-id", + }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { content: "Hello, world!" }, + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "002"), + }, + }, + }); +}); + +test("streamText", async () => { + const model = new MockMultiStepLanguageModelV1({ + doStream: async () => { + if (model.streamStep === 0) { + return { + stream: convertArrayToReadableStream([ + { + type: "tool-call", + toolCallType: "function", + toolName: "listOrders", + toolCallId: "tool-id", + args: JSON.stringify({ userId: "123" }), + }, + { + type: "finish", + finishReason: "stop", + logprobs: undefined, + usage: { completionTokens: 10, promptTokens: 3 }, + }, + ] satisfies LanguageModelV1StreamPart[]), + rawCall: { rawPrompt: null, rawSettings: {} }, + }; + } + + return { + stream: convertArrayToReadableStream([ + { type: "text-delta", textDelta: "Hello" }, + { type: "text-delta", textDelta: ", " }, + { type: "text-delta", textDelta: `world!` }, + { + type: "finish", + finishReason: "stop", + logprobs: undefined, + usage: { completionTokens: 10, promptTokens: 3 }, + }, + ]), + rawCall: { rawPrompt: null, rawSettings: {} }, + }; + }, + }); + + const result = await streamText({ + model, + messages: [ + { + role: "user", + content: "What are my orders? My user ID is 123", + }, + ], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => + `User ${userId} has the following orders: 1`, + }), + }, + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), + maxSteps: 10, + }); + + await toArray(result.fullStream); + await provider.forceFlush(); + + const actual = getAssumedTreeFromCalls(callSpy.mock.calls); + expect(actual).toMatchObject({ + nodes: [ + "mock-provider:0", + "mock-provider:1", + "listOrders:2", + "mock-provider:3", + ], + edges: [ + ["mock-provider:0", "mock-provider:1"], + ["mock-provider:0", "listOrders:2"], + ["mock-provider:0", "mock-provider:3"], + ], + data: { + "mock-provider:0": { + inputs: { + messages: [ + { + type: "human", + data: { content: "What are my orders? My user ID is 123" }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { content: "Hello, world!" }, + token_usage: { completion_tokens: 20, prompt_tokens: 6 }, + }, + }, + dotted_order: new ExecutionOrderSame(1, "000"), + }, + "mock-provider:1": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { + type: "text", + text: "What are my orders? My user ID is 123", + }, + ], + }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { + content: [ + { + type: "tool_use", + name: "listOrders", + id: "tool-id", + input: { userId: "123" }, + }, + ], + additional_kwargs: { + tool_calls: [ + { + id: "tool-id", + type: "function", + function: { + name: "listOrders", + id: "tool-id", + arguments: '{"userId":"123"}', + }, + }, + ], + }, + }, + token_usage: { completion_tokens: 10, prompt_tokens: 3 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "000"), + }, + "listOrders:2": { + inputs: { userId: "123" }, + outputs: { output: "User 123 has the following orders: 1" }, + dotted_order: new ExecutionOrderSame(2, "001"), + }, + "mock-provider:3": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { + type: "text", + text: "What are my orders? My user ID is 123", + }, + ], + }, + }, + { + type: "ai", + data: { + content: [ + { + type: "tool_use", + name: "listOrders", + id: "tool-id", + input: { userId: "123" }, + }, + ], + additional_kwargs: { + tool_calls: [ + { + id: "tool-id", + type: "function", + function: { + name: "listOrders", + id: "tool-id", + arguments: '{"userId":"123"}', + }, + }, + ], + }, + }, + }, + { + type: "tool", + data: { + content: '"User 123 has the following orders: 1"', + name: "listOrders", + tool_call_id: "tool-id", + }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { content: "Hello, world!" }, + token_usage: { completion_tokens: 10, prompt_tokens: 3 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "002"), + }, + }, + }); +}); + +test("generateObject", async () => { + const model = new MockMultiStepLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + toolCalls: [ + { + toolCallType: "function", + toolName: "json", + toolCallId: "tool-id", + args: JSON.stringify({ + weather: { city: "Prague", unit: "celsius" }, + }), + }, + ], + }), + defaultObjectGenerationMode: "tool", + }); + + await generateObject({ + model, + schema: z.object({ + weather: z.object({ + city: z.string(), + unit: z.union([z.literal("celsius"), z.literal("fahrenheit")]), + }), + }), + prompt: "What's the weather in Prague?", + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), + }); + + await provider.forceFlush(); + const actual = getAssumedTreeFromCalls(callSpy.mock.calls); + + expect(actual).toMatchObject({ + nodes: ["mock-provider:0", "mock-provider:1"], + edges: [["mock-provider:0", "mock-provider:1"]], + data: { + "mock-provider:0": { + inputs: { + input: { prompt: "What's the weather in Prague?" }, + }, + outputs: { + output: { weather: { city: "Prague", unit: "celsius" } }, + llm_output: { + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(1, "000"), + }, + "mock-provider:1": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { type: "text", text: "What's the weather in Prague?" }, + ], + }, + }, + ], + }, + outputs: { + output: { weather: { city: "Prague", unit: "celsius" } }, + llm_output: { + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "000"), + }, + }, + }); +}); + +test("streamObject", async () => { + const model = new MockMultiStepLanguageModelV1({ + doGenerate: async () => ({ + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + toolCalls: [ + { + toolCallType: "function", + toolName: "json", + toolCallId: "tool-id", + args: JSON.stringify({ + weather: { city: "Prague", unit: "celsius" }, + }), + }, + ], + }), + + doStream: async () => { + return { + stream: convertArrayToReadableStream([ + { + type: "tool-call-delta", + toolCallType: "function", + toolName: "json", + toolCallId: "tool-id", + argsTextDelta: JSON.stringify({ + weather: { city: "Prague", unit: "celsius" }, + }), + }, + { + type: "finish", + finishReason: "stop", + logprobs: undefined, + usage: { completionTokens: 10, promptTokens: 3 }, + }, + ] satisfies LanguageModelV1StreamPart[]), + rawCall: { rawPrompt: null, rawSettings: {} }, + }; + }, + defaultObjectGenerationMode: "tool", + }); + + const result = await streamObject({ + model, + schema: z.object({ + weather: z.object({ + city: z.string(), + unit: z.union([z.literal("celsius"), z.literal("fahrenheit")]), + }), + }), + prompt: "What's the weather in Prague?", + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), + }); + + await toArray(result.partialObjectStream); + await provider.forceFlush(); + + const actual = getAssumedTreeFromCalls(callSpy.mock.calls); + expect(actual).toMatchObject({ + nodes: ["mock-provider:0", "mock-provider:1"], + edges: [["mock-provider:0", "mock-provider:1"]], + data: { + "mock-provider:0": { + inputs: { + input: { prompt: "What's the weather in Prague?" }, + }, + outputs: { + output: { weather: { city: "Prague", unit: "celsius" } }, + llm_output: { + token_usage: { completion_tokens: 10, prompt_tokens: 3 }, + }, + }, + dotted_order: new ExecutionOrderSame(1, "000"), + }, + "mock-provider:1": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { type: "text", text: "What's the weather in Prague?" }, + ], + }, + }, + ], + }, + outputs: { + output: { weather: { city: "Prague", unit: "celsius" } }, + llm_output: { + token_usage: { completion_tokens: 10, prompt_tokens: 3 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "000"), + }, + }, + }); +}); + +test("traceable", async () => { + const model = new MockMultiStepLanguageModelV1({ + doGenerate: async () => { + if (model.generateStep === 0) { + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + toolCalls: [ + { + toolCallType: "function", + toolName: "listOrders", + toolCallId: "tool-id", + args: JSON.stringify({ userId: "123" }), + }, + ], + }; + } + + return { + rawCall: { rawPrompt: null, rawSettings: {} }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 20 }, + text: `Hello, world!`, + }; + }, + }); + + const wrappedText = traceable( + async (content: string) => { + const { text } = await generateText({ + model, + messages: [{ role: "user", content }], + tools: { + listOrders: tool({ + description: "list all orders", + parameters: z.object({ userId: z.string() }), + execute: async ({ userId }) => + `User ${userId} has the following orders: 1`, + }), + }, + experimental_telemetry: AISDKExporter.getSettings({ + functionId: "functionId", + metadata: { userId: "123", language: "english" }, + }), + maxSteps: 10, + }); + + return { text }; + }, + { name: "wrappedText", client, tracingEnabled: true } + ); + + await wrappedText("What are my orders? My user ID is 123"); + await provider.forceFlush(); + + const actual = getAssumedTreeFromCalls(callSpy.mock.calls); + expect(actual).toMatchObject({ + nodes: [ + "wrappedText:0", + "mock-provider:1", + "mock-provider:2", + "listOrders:3", + "mock-provider:4", + ], + edges: [ + ["wrappedText:0", "mock-provider:1"], + ["mock-provider:1", "mock-provider:2"], + ["mock-provider:1", "listOrders:3"], + ["mock-provider:1", "mock-provider:4"], + ], + data: { + "wrappedText:0": { + inputs: { + input: "What are my orders? My user ID is 123", + }, + outputs: { + text: "Hello, world!", + }, + dotted_order: new ExecutionOrderSame(1, "001"), + }, + "mock-provider:1": { + inputs: { + messages: [ + { + type: "human", + data: { content: "What are my orders? My user ID is 123" }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { content: "Hello, world!" }, + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(2, "000"), + }, + "mock-provider:2": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { + type: "text", + text: "What are my orders? My user ID is 123", + }, + ], + }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { + content: [ + { + type: "tool_use", + name: "listOrders", + id: "tool-id", + input: { userId: "123" }, + }, + ], + additional_kwargs: { + tool_calls: [ + { + id: "tool-id", + type: "function", + function: { + name: "listOrders", + id: "tool-id", + arguments: '{"userId":"123"}', + }, + }, + ], + }, + }, + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(3, "000"), + }, + "listOrders:3": { + inputs: { userId: "123" }, + outputs: { output: "User 123 has the following orders: 1" }, + dotted_order: new ExecutionOrderSame(3, "001"), + }, + "mock-provider:4": { + inputs: { + messages: [ + { + type: "human", + data: { + content: [ + { + type: "text", + text: "What are my orders? My user ID is 123", + }, + ], + }, + }, + { + type: "ai", + data: { + content: [ + { + type: "tool_use", + name: "listOrders", + id: "tool-id", + input: { userId: "123" }, + }, + ], + additional_kwargs: { + tool_calls: [ + { + id: "tool-id", + type: "function", + function: { + name: "listOrders", + id: "tool-id", + arguments: '{"userId":"123"}', + }, + }, + ], + }, + }, + }, + { + type: "tool", + data: { + content: '"User 123 has the following orders: 1"', + name: "listOrders", + tool_call_id: "tool-id", + }, + }, + ], + }, + outputs: { + llm_output: { + type: "ai", + data: { content: "Hello, world!" }, + token_usage: { completion_tokens: 20, prompt_tokens: 10 }, + }, + }, + dotted_order: new ExecutionOrderSame(3, "002"), + }, + }, + }); +}); diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 9ab1e127f..fe1250381 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -203,6 +203,14 @@ function convertToTimestamp([seconds, nanoseconds]: [ return Number(String(seconds) + ms); } +function sortByHr( + a: [seconds: number, nanoseconds: number], + b: [seconds: number, nanoseconds: number] +): number { + if (a[0] !== b[0]) return Math.sign(a[0] - b[0]); + return Math.sign(a[1] - b[1]); +} + const ROOT = "$"; const RUN_ID_NAMESPACE = "5c718b20-9078-11ef-9a3d-325096b39f47"; @@ -611,7 +619,10 @@ export class AISDKExporter { spans: unknown[], resultCallback: (result: { code: 0 | 1; error?: Error }) => void ): void { - const typedSpans = spans as AISDKSpan[]; + const typedSpans = (spans as AISDKSpan[]) + .slice() + .sort((a, b) => sortByHr(a.startTime, b.startTime)); + for (const span of typedSpans) { const { traceId, spanId } = span.spanContext(); const parentId = span.parentSpanId ?? undefined; From 44466ff1ab47d1ad14ff55d3f8f1af66889ac287 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Thu, 24 Oct 2024 15:49:16 +0200 Subject: [PATCH 23/36] Fix typo --- js/src/tests/vercel.int.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/src/tests/vercel.int.test.ts b/js/src/tests/vercel.int.test.ts index 8741ad4b4..1abe12e66 100644 --- a/js/src/tests/vercel.int.test.ts +++ b/js/src/tests/vercel.int.test.ts @@ -144,7 +144,7 @@ test("generateObject", async () => { await generateObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ - recipe: z.object({ + weather: z.object({ city: z.string(), unit: z.union([z.literal("celsius"), z.literal("fahrenheit")]), }), @@ -169,7 +169,7 @@ test("streamObject", async () => { const result = await streamObject({ model: openai("gpt-4o-mini", { structuredOutputs: true }), schema: z.object({ - recipe: z.object({ + weather: z.object({ city: z.string(), unit: z.union([z.literal("celsius"), z.literal("fahrenheit")]), }), From c95430ce682f258fa6c1b4e90a0811ae1a4e9a95 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 19:13:26 +0200 Subject: [PATCH 24/36] Add session name --- js/src/vercel.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index fe1250381..e44844c64 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -9,6 +9,7 @@ import { Client, RunTree } from "./index.js"; import { KVMap, RunCreate } from "./schemas.js"; import { v5 as uuid5 } from "uuid"; import { getCurrentRunTree } from "./singletons/traceable.js"; +import { getEnvironmentVariable } from "./utils/env.js"; // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; @@ -378,6 +379,9 @@ export class AISDKExporter { "ai.operationId": span.attributes["ai.operationId"], }, }, + session_name: + getEnvironmentVariable("LANGCHAIN_PROJECT") ?? + getEnvironmentVariable("LANGCHAIN_SESSION"), start_time: Math.min(parsedStart, parsedEnd), end_time: Math.max(parsedStart, parsedEnd), }; From a541c7c057d8767bb9496ee92b353744209023f8 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 19:14:01 +0200 Subject: [PATCH 25/36] Use langsmith prefix --- js/src/vercel.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index e44844c64..7c7d6b5e8 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -9,7 +9,10 @@ import { Client, RunTree } from "./index.js"; import { KVMap, RunCreate } from "./schemas.js"; import { v5 as uuid5 } from "uuid"; import { getCurrentRunTree } from "./singletons/traceable.js"; -import { getEnvironmentVariable } from "./utils/env.js"; +import { + getEnvironmentVariable, + getLangSmithEnvironmentVariable, +} from "./utils/env.js"; // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; @@ -380,8 +383,8 @@ export class AISDKExporter { }, }, session_name: - getEnvironmentVariable("LANGCHAIN_PROJECT") ?? - getEnvironmentVariable("LANGCHAIN_SESSION"), + getLangSmithEnvironmentVariable("PROJECT") ?? + getLangSmithEnvironmentVariable("SESSION"), start_time: Math.min(parsedStart, parsedEnd), end_time: Math.max(parsedStart, parsedEnd), }; From 48e1a559a1ab160784121f691b239ece3813db0f Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 19:14:10 +0200 Subject: [PATCH 26/36] Remove unused --- js/src/vercel.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 7c7d6b5e8..53bee9533 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -9,10 +9,7 @@ import { Client, RunTree } from "./index.js"; import { KVMap, RunCreate } from "./schemas.js"; import { v5 as uuid5 } from "uuid"; import { getCurrentRunTree } from "./singletons/traceable.js"; -import { - getEnvironmentVariable, - getLangSmithEnvironmentVariable, -} from "./utils/env.js"; +import { getLangSmithEnvironmentVariable } from "./utils/env.js"; // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; From 6f8964e9e74b9b69e7aca4343d204e4b7a6133d8 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Fri, 25 Oct 2024 11:21:24 -0700 Subject: [PATCH 27/36] Misguided support for run name --- js/package.json | 2 +- js/src/index.ts | 2 +- js/src/tests/vercel.int.test.ts | 24 ++++++-- js/src/vercel.ts | 105 ++++++++++++++++++++------------ 4 files changed, 88 insertions(+), 45 deletions(-) diff --git a/js/package.json b/js/package.json index ac504e0ba..7e897da13 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.2.0", + "version": "0.2.1", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ diff --git a/js/src/index.ts b/js/src/index.ts index 77f0939f1..78f45d4d6 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js"; export { overrideFetchImplementation } from "./singletons/fetch.js"; // Update using yarn bump-version -export const __version__ = "0.2.0"; +export const __version__ = "0.2.1"; diff --git a/js/src/tests/vercel.int.test.ts b/js/src/tests/vercel.int.test.ts index 1abe12e66..a9892fc59 100644 --- a/js/src/tests/vercel.int.test.ts +++ b/js/src/tests/vercel.int.test.ts @@ -87,6 +87,7 @@ test("generateText with image", async () => { ], experimental_telemetry: AISDKExporter.getSettings({ runId, + runName: "vercelImageTest", functionId: "functionId", metadata: { userId: "123", language: "english" }, }), @@ -178,7 +179,10 @@ test("streamObject", async () => { experimental_telemetry: AISDKExporter.getSettings({ runId, functionId: "functionId", - metadata: { userId: "123", language: "english" }, + metadata: { + userId: "123", + language: "english", + }, }), }); @@ -190,7 +194,7 @@ test("streamObject", async () => { expect(storedRun.id).toEqual(runId); }); -test("traceable", async () => { +test.only("traceable", async () => { const runId = uuid(); const wrappedText = traceable( @@ -214,18 +218,30 @@ test("traceable", async () => { }, experimental_telemetry: AISDKExporter.getSettings({ functionId: "functionId", + // runName: "nestedVercelTrace", metadata: { userId: "123", language: "english" }, }), maxSteps: 10, }); + const foo = traceable( + async () => { + return "bar"; + }, + { + name: "foo", + } + ); + + await foo(); + return { text }; }, - { name: "wrappedText", id: runId } + { name: "parentTraceable", id: runId } ); const result = await wrappedText( - "What are my orders and where are they? My user ID is 123" + "What are my orders and where are they? My user ID is 123. Use available tools." ); await waitUntilRunFound(client, runId, true); const storedRun = await client.readRun(runId); diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 53bee9533..b42425860 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -7,21 +7,23 @@ import type { import type { AISDKSpan } from "./vercel.types.js"; import { Client, RunTree } from "./index.js"; import { KVMap, RunCreate } from "./schemas.js"; -import { v5 as uuid5 } from "uuid"; +import { v5 as uuid5, v4 as uuid4 } from "uuid"; import { getCurrentRunTree } from "./singletons/traceable.js"; import { getLangSmithEnvironmentVariable } from "./utils/env.js"; // eslint-disable-next-line @typescript-eslint/ban-types type AnyString = string & {}; -type AITelemetrySettings = Exclude< +export type AITelemetrySettings = Exclude< Parameters[0]["experimental_telemetry"], undefined >; -interface TelemetrySettings extends AITelemetrySettings { +export interface TelemetrySettings extends AITelemetrySettings { /** ID of the run sent to LangSmith */ runId?: string; + /** Name of the run sent to LangSmith */ + runName?: string; } type LangChainMessageFields = { @@ -220,6 +222,11 @@ const RUN_ID_METADATA_KEY = { output: "ai.telemetry.metadata.langsmith:runId", }; +const RUN_NAME_METADATA_KEY = { + input: "langsmith:runName", + output: "ai.telemetry.metadata.langsmith:runName", +}; + const TRACE_METADATA_KEY = { input: "langsmith:trace", output: "ai.telemetry.metadata.langsmith:trace", @@ -232,6 +239,7 @@ const BAGGAGE_METADATA_KEY = { const RESERVED_METADATA_KEYS = [ RUN_ID_METADATA_KEY.output, + RUN_NAME_METADATA_KEY.output, TRACE_METADATA_KEY.output, BAGGAGE_METADATA_KEY.output, ]; @@ -246,10 +254,13 @@ interface RunTask { } type InteropType = - | { type: "traceable"; parentRunTree: RunTree } - | { type: "manual"; userTraceId: string } + | { type: "traceable"; parentRunTree: RunTree; userRunName?: string } + | { type: "manual"; userTraceId?: string; userRunName?: string } | undefined; +/** + * + */ export class AISDKExporter { private client: Client; private traceByMap: Record< @@ -267,9 +278,10 @@ export class AISDKExporter { } static getSettings(settings: TelemetrySettings) { - const { runId, ...rest } = settings; + const { runId, runName, ...rest } = settings; const metadata = { ...rest?.metadata }; if (runId != null) metadata[RUN_ID_METADATA_KEY.input] = runId; + if (runName != null) metadata[RUN_NAME_METADATA_KEY.input] = runName; // attempt to obtain the run tree if used within a traceable function let defaultEnabled = true; @@ -309,8 +321,9 @@ export class AISDKExporter { : undefined; }; - const userTraceId = getKey(RUN_ID_METADATA_KEY.output) || undefined; - const parentTrace = getKey(TRACE_METADATA_KEY.output) || undefined; + const userTraceId = getKey(RUN_ID_METADATA_KEY.output); + const userRunName = getKey(RUN_NAME_METADATA_KEY.output); + const parentTrace = getKey(TRACE_METADATA_KEY.output); if (parentTrace && userTraceId) { throw new Error( @@ -326,10 +339,11 @@ export class AISDKExporter { if (!parentRunTree) throw new Error("Unreachable code: empty parent run tree"); - return { type: "traceable", parentRunTree }; + return { type: "traceable", parentRunTree, userRunName }; } - if (userTraceId) return { type: "manual", userTraceId }; + if (userTraceId || userRunName) + return { type: "manual", userTraceId, userRunName }; return undefined; } @@ -663,16 +677,16 @@ export class AISDKExporter { traceMap.interop = this.parseInteropFromMetadata(span); } + type TraceMapData = { + dotted_order: string; + id: string; + trace_id: string; + parent_run_id: string | undefined; + name?: string; + }; + // collect all subgraphs - const sampled: [ - { - dotted_order: string; - id: string; - trace_id: string; - parent_run_id: string | undefined; - }, - RunCreate - ][] = []; + const sampled: [TraceMapData, RunCreate][] = []; for (const traceId of Object.keys(this.traceByMap)) { type QueueItem = { item: RunTask; dottedOrder: string; traceId: string }; @@ -696,7 +710,8 @@ export class AISDKExporter { if (seen.has(task.item.id)) continue; if (!task.item.sent) { - let ident = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let ident: TraceMapData = { id: task.item.id, parent_run_id: task.item.parentId, dotted_order: task.dottedOrder, @@ -717,22 +732,32 @@ export class AISDKExporter { .join("."), trace_id: traceMap.interop.parentRunTree.trace_id, }; + if ( + traceMap.interop.userRunName && + ident.parent_run_id === ident.trace_id + ) { + ident.name = traceMap.interop.userRunName; + } } else if (traceMap.interop.type === "manual") { + const userTraceId = traceMap.interop.userTraceId ?? uuid4(); ident = { - id: - ident.id === ident.trace_id - ? traceMap.interop.userTraceId - : ident.id, + id: ident.id === ident.trace_id ? userTraceId : ident.id, parent_run_id: ident.parent_run_id === ident.trace_id - ? traceMap.interop.userTraceId + ? userTraceId : ident.parent_run_id, dotted_order: ident.dotted_order.replace( ident.trace_id, - traceMap.interop.userTraceId + userTraceId ), - trace_id: traceMap.interop.userTraceId, + trace_id: userTraceId, }; + if ( + traceMap.interop.userRunName && + ident.parent_run_id !== ident.trace_id + ) { + ident.name = traceMap.interop.userRunName; + } } } @@ -742,18 +767,20 @@ export class AISDKExporter { const children = traceMap.childMap[task.item.id] ?? []; queue.push( - ...children.map((child) => ({ - item: child, - dottedOrder: [ - task.dottedOrder, - convertToDottedOrderFormat( - child.startTime, - child.id, - child.executionOrder - ), - ].join("."), - traceId: task.traceId, - })) + ...children.map((child) => { + return { + item: child, + dottedOrder: [ + task.dottedOrder, + convertToDottedOrderFormat( + child.startTime, + child.id, + child.executionOrder + ), + ].join("."), + traceId: task.traceId, + }; + }) ); } } From e09e94b6e227c7585fee157d33474226b136e432 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 21:04:21 +0200 Subject: [PATCH 28/36] Rename to make more sense, add tests for runName --- js/src/tests/vercel.test.ts | 26 ++++---- js/src/vercel.ts | 119 ++++++++++++++++++++---------------- 2 files changed, 82 insertions(+), 63 deletions(-) diff --git a/js/src/tests/vercel.test.ts b/js/src/tests/vercel.test.ts index 15ddb2f3f..cd28d9054 100644 --- a/js/src/tests/vercel.test.ts +++ b/js/src/tests/vercel.test.ts @@ -134,6 +134,7 @@ test("generateText", async () => { }), }, experimental_telemetry: AISDKExporter.getSettings({ + runName: "generateText", functionId: "functionId", metadata: { userId: "123", language: "english" }, }), @@ -143,18 +144,19 @@ test("generateText", async () => { await provider.forceFlush(); expect(getAssumedTreeFromCalls(callSpy.mock.calls)).toMatchObject({ nodes: [ - "mock-provider:0", + "generateText:0", "mock-provider:1", "listOrders:2", "mock-provider:3", ], edges: [ - ["mock-provider:0", "mock-provider:1"], - ["mock-provider:0", "listOrders:2"], - ["mock-provider:0", "mock-provider:3"], + ["generateText:0", "mock-provider:1"], + ["generateText:0", "listOrders:2"], + ["generateText:0", "mock-provider:3"], ], data: { - "mock-provider:0": { + "generateText:0": { + name: "generateText", inputs: { messages: [ { @@ -725,6 +727,7 @@ test("traceable", async () => { }), }, experimental_telemetry: AISDKExporter.getSettings({ + runName: "generateText", functionId: "functionId", metadata: { userId: "123", language: "english" }, }), @@ -743,16 +746,16 @@ test("traceable", async () => { expect(actual).toMatchObject({ nodes: [ "wrappedText:0", - "mock-provider:1", + "generateText:1", "mock-provider:2", "listOrders:3", "mock-provider:4", ], edges: [ - ["wrappedText:0", "mock-provider:1"], - ["mock-provider:1", "mock-provider:2"], - ["mock-provider:1", "listOrders:3"], - ["mock-provider:1", "mock-provider:4"], + ["wrappedText:0", "generateText:1"], + ["generateText:1", "mock-provider:2"], + ["generateText:1", "listOrders:3"], + ["generateText:1", "mock-provider:4"], ], data: { "wrappedText:0": { @@ -764,7 +767,8 @@ test("traceable", async () => { }, dotted_order: new ExecutionOrderSame(1, "001"), }, - "mock-provider:1": { + "generateText:1": { + name: "generateText", inputs: { messages: [ { diff --git a/js/src/vercel.ts b/js/src/vercel.ts index b42425860..efe56829e 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -254,12 +254,12 @@ interface RunTask { } type InteropType = - | { type: "traceable"; parentRunTree: RunTree; userRunName?: string } - | { type: "manual"; userTraceId?: string; userRunName?: string } + | { type: "traceable"; parentRunTree: RunTree } + | { type: "user"; userTraceId?: string } | undefined; /** - * + * OpenTelemetry trace exporter for Vercel AI SDK */ export class AISDKExporter { private client: Client; @@ -312,18 +312,27 @@ export class AISDKExporter { } /** @internal */ - protected parseInteropFromMetadata(span: AISDKSpan): InteropType { - const getKey = (key: string): string | undefined => { - const attributes = span.attributes as Record; - - return key in attributes && typeof attributes[key] === "string" - ? (attributes[key] as string) - : undefined; - }; + protected getSpanAttributeKey = ( + span: AISDKSpan, + key: string + ): string | undefined => { + const attributes = span.attributes as Record; + + return key in attributes && typeof attributes[key] === "string" + ? (attributes[key] as string) + : undefined; + }; - const userTraceId = getKey(RUN_ID_METADATA_KEY.output); - const userRunName = getKey(RUN_NAME_METADATA_KEY.output); - const parentTrace = getKey(TRACE_METADATA_KEY.output); + /** @internal */ + protected parseInteropFromMetadata(span: AISDKSpan): InteropType { + const userTraceId = this.getSpanAttributeKey( + span, + RUN_ID_METADATA_KEY.output + ); + const parentTrace = this.getSpanAttributeKey( + span, + TRACE_METADATA_KEY.output + ); if (parentTrace && userTraceId) { throw new Error( @@ -334,16 +343,16 @@ export class AISDKExporter { if (parentTrace) { const parentRunTree = RunTree.fromHeaders({ "langsmith-trace": parentTrace, - baggage: getKey(BAGGAGE_METADATA_KEY.output) || "", + baggage: + this.getSpanAttributeKey(span, BAGGAGE_METADATA_KEY.output) || "", }); if (!parentRunTree) throw new Error("Unreachable code: empty parent run tree"); - return { type: "traceable", parentRunTree, userRunName }; + return { type: "traceable", parentRunTree }; } - if (userTraceId || userRunName) - return { type: "manual", userTraceId, userRunName }; + if (userTraceId) return { type: "user", userTraceId }; return undefined; } @@ -381,8 +390,17 @@ export class AISDKExporter { const parsedStart = convertToTimestamp(span.startTime); const parsedEnd = convertToTimestamp(span.endTime); + let name = rawConfig.name; + + // if user provided a custom name, only use it if it's the root + if (span.parentSpanId == null) { + name = + this.getSpanAttributeKey(span, RUN_NAME_METADATA_KEY.output) || name; + } + const config: RunCreate = { ...rawConfig, + name, id: runId, parent_run_id: parentRunId, extra: { @@ -677,16 +695,15 @@ export class AISDKExporter { traceMap.interop = this.parseInteropFromMetadata(span); } - type TraceMapData = { - dotted_order: string; + type OverrideRunCreate = { id: string; trace_id: string; + dotted_order: string; parent_run_id: string | undefined; - name?: string; }; - // collect all subgraphs - const sampled: [TraceMapData, RunCreate][] = []; + // We separate `id`, + const sampled: [OverrideRunCreate, RunCreate][] = []; for (const traceId of Object.keys(this.traceByMap)) { type QueueItem = { item: RunTask; dottedOrder: string; traceId: string }; @@ -711,7 +728,7 @@ export class AISDKExporter { if (!task.item.sent) { // eslint-disable-next-line @typescript-eslint/no-explicit-any - let ident: TraceMapData = { + let override: OverrideRunCreate = { id: task.item.id, parent_run_id: task.item.parentId, dotted_order: task.dottedOrder, @@ -719,49 +736,48 @@ export class AISDKExporter { }; if (traceMap.interop) { + // attach the run to a parent run tree + // - id: preserve + // - parent_run_id: use existing parent run id or hook to the provided run tree + // - dotted_order: append to the dotted_order of the parent run tree + // - trace_id: use from the existing run tree if (traceMap.interop.type === "traceable") { - ident = { - id: ident.id, + override = { + id: override.id, parent_run_id: - ident.parent_run_id ?? traceMap.interop.parentRunTree.id, + override.parent_run_id ?? traceMap.interop.parentRunTree.id, dotted_order: [ traceMap.interop.parentRunTree.dotted_order, - ident.dotted_order, + override.dotted_order, ] .filter(Boolean) .join("."), trace_id: traceMap.interop.parentRunTree.trace_id, }; - if ( - traceMap.interop.userRunName && - ident.parent_run_id === ident.trace_id - ) { - ident.name = traceMap.interop.userRunName; - } - } else if (traceMap.interop.type === "manual") { + } else if (traceMap.interop.type === "user") { + // Allow user to specify custom trace ID = run ID of the root run + // - id: use user provided run ID if root run, otherwise preserve + // - parent_run_id: use user provided run ID if root run, otherwise preserve + // - dotted_order: replace the trace_id with the user provided run ID + // - trace_id: use user provided run ID const userTraceId = traceMap.interop.userTraceId ?? uuid4(); - ident = { - id: ident.id === ident.trace_id ? userTraceId : ident.id, + override = { + id: + override.id === override.trace_id ? userTraceId : override.id, parent_run_id: - ident.parent_run_id === ident.trace_id + override.parent_run_id === override.trace_id ? userTraceId - : ident.parent_run_id, - dotted_order: ident.dotted_order.replace( - ident.trace_id, + : override.parent_run_id, + dotted_order: override.dotted_order.replace( + override.trace_id, userTraceId ), trace_id: userTraceId, }; - if ( - traceMap.interop.userRunName && - ident.parent_run_id !== ident.trace_id - ) { - ident.name = traceMap.interop.userRunName; - } } } - sampled.push([ident, task.item.run]); + sampled.push([override, task.item.run]); task.item.sent = true; } @@ -786,10 +802,9 @@ export class AISDKExporter { } Promise.all( - sampled.map(([required, value]) => { - const payload = { ...value, ...required }; - return this.client.createRun(payload); - }) + sampled.map(([override, value]) => + this.client.createRun({ ...value, ...override }) + ) ).then( () => resultCallback({ code: 0 }), (error) => resultCallback({ code: 1, error }) From 95ae69d148c770d777a0d3e30f5876c5dd7605bf Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 21:07:34 +0200 Subject: [PATCH 29/36] Add unit test asserts for metadata --- js/src/tests/vercel.test.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/js/src/tests/vercel.test.ts b/js/src/tests/vercel.test.ts index cd28d9054..bc104b9bd 100644 --- a/js/src/tests/vercel.test.ts +++ b/js/src/tests/vercel.test.ts @@ -172,6 +172,13 @@ test("generateText", async () => { token_usage: { completion_tokens: 20, prompt_tokens: 10 }, }, }, + extra: { + metadata: { + functionId: "functionId", + userId: "123", + language: "english", + }, + }, dotted_order: new ExecutionOrderSame(1, "000"), }, "mock-provider:1": { @@ -386,6 +393,13 @@ test("streamText", async () => { token_usage: { completion_tokens: 20, prompt_tokens: 6 }, }, }, + extra: { + metadata: { + functionId: "functionId", + userId: "123", + language: "english", + }, + }, dotted_order: new ExecutionOrderSame(1, "000"), }, "mock-provider:1": { @@ -576,6 +590,13 @@ test("generateObject", async () => { token_usage: { completion_tokens: 20, prompt_tokens: 10 }, }, }, + extra: { + metadata: { + functionId: "functionId", + userId: "123", + language: "english", + }, + }, dotted_order: new ExecutionOrderSame(2, "000"), }, }, @@ -658,6 +679,13 @@ test("streamObject", async () => { token_usage: { completion_tokens: 10, prompt_tokens: 3 }, }, }, + extra: { + metadata: { + functionId: "functionId", + userId: "123", + language: "english", + }, + }, dotted_order: new ExecutionOrderSame(1, "000"), }, "mock-provider:1": { @@ -769,6 +797,13 @@ test("traceable", async () => { }, "generateText:1": { name: "generateText", + extra: { + metadata: { + functionId: "functionId", + userId: "123", + language: "english", + }, + }, inputs: { messages: [ { From 2079b12563ddaabc2f7ce83e2e14adb5f0e92731 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 21:08:01 +0200 Subject: [PATCH 30/36] Remove unused eslint warning --- js/src/vercel.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index efe56829e..2c7a7f02c 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -727,7 +727,6 @@ export class AISDKExporter { if (seen.has(task.item.id)) continue; if (!task.item.sent) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any let override: OverrideRunCreate = { id: task.item.id, parent_run_id: task.item.parentId, From a5bf1d7d83b2594bc9f7208b5702eaaf148321ee Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Fri, 25 Oct 2024 12:11:42 -0700 Subject: [PATCH 31/36] Remove focused test --- js/src/tests/vercel.int.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/tests/vercel.int.test.ts b/js/src/tests/vercel.int.test.ts index a9892fc59..4817a4388 100644 --- a/js/src/tests/vercel.int.test.ts +++ b/js/src/tests/vercel.int.test.ts @@ -218,7 +218,7 @@ test.only("traceable", async () => { }, experimental_telemetry: AISDKExporter.getSettings({ functionId: "functionId", - // runName: "nestedVercelTrace", + runName: "nestedVercelTrace", metadata: { userId: "123", language: "english" }, }), maxSteps: 10, From 601c498fea798f709b42ae8bd134b53ae313b14c Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 21:12:30 +0200 Subject: [PATCH 32/36] remove focues test --- js/src/tests/vercel.int.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/tests/vercel.int.test.ts b/js/src/tests/vercel.int.test.ts index 4817a4388..968ec4bdd 100644 --- a/js/src/tests/vercel.int.test.ts +++ b/js/src/tests/vercel.int.test.ts @@ -194,7 +194,7 @@ test("streamObject", async () => { expect(storedRun.id).toEqual(runId); }); -test.only("traceable", async () => { +test("traceable", async () => { const runId = uuid(); const wrappedText = traceable( From 7fc05a344f0e21c99e376df0a76d783d7e96551b Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 21:49:47 +0200 Subject: [PATCH 33/36] Remove warning --- js/src/vercel.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 2c7a7f02c..226da4c7a 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -646,7 +646,6 @@ export class AISDKExporter { case "ai.embedMany": case "ai.embedMany.doEmbed": default: - console.warn(`Span "${span.name}" is currently unsupported.`); return undefined; } } From dd159b2458791ab4f563a9b4b0b84c3f0df69e68 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Fri, 25 Oct 2024 12:55:42 -0700 Subject: [PATCH 34/36] Update docstrings --- js/src/vercel.ts | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 2c7a7f02c..d8aa8236c 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -259,7 +259,44 @@ type InteropType = | undefined; /** - * OpenTelemetry trace exporter for Vercel AI SDK + * OpenTelemetry trace exporter for Vercel AI SDK. + * + * @example + * ```ts + * import { AISDKExporter } from "langsmith/vercel"; + * import { Client } from "langsmith"; + * + * import { generateText } from "ai"; + * import { openai } from "@ai-sdk/openai"; + * + * import { NodeSDK } from "@opentelemetry/sdk-node"; + * import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; + * + * const client = new Client(); + * + * const sdk = new NodeSDK({ + * traceExporter: new AISDKExporter({ client }), + * instrumentations: [getNodeAutoInstrumentations()], + * }); + * + * sdk.start(); + * + * const res = await generateText({ + * model: openai("gpt-4o-mini"), + * messages: [ + * { + * role: "user", + * content: "What color is the sky?", + * }, + * ], + * experimental_telemetry: AISDKExporter.getSettings({ + * runName: "langsmith_traced_call", + * metadata: { userId: "123", language: "english" }, + * }), + * }); + * + * await sdk.shutdown(); + * ``` */ export class AISDKExporter { private client: Client; @@ -277,6 +314,9 @@ export class AISDKExporter { this.client = args?.client ?? new Client(); } + /** + * Helper method for initializing OTEL settings. + */ static getSettings(settings: TelemetrySettings) { const { runId, runName, ...rest } = settings; const metadata = { ...rest?.metadata }; From 4a65bb9435f1ec352bdde263772dc8f5b2c1a59f Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 22:15:11 +0200 Subject: [PATCH 35/36] Make getSettings param optional --- js/src/vercel.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index bb6c3099e..840defaba 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -314,11 +314,8 @@ export class AISDKExporter { this.client = args?.client ?? new Client(); } - /** - * Helper method for initializing OTEL settings. - */ - static getSettings(settings: TelemetrySettings) { - const { runId, runName, ...rest } = settings; + static getSettings(settings?: TelemetrySettings) { + const { runId, runName, ...rest } = settings ?? {}; const metadata = { ...rest?.metadata }; if (runId != null) metadata[RUN_ID_METADATA_KEY.input] = runId; if (runName != null) metadata[RUN_NAME_METADATA_KEY.input] = runName; From 90c75f2be0f59d3b45be01276f9d277973d47814 Mon Sep 17 00:00:00 2001 From: Tat Dat Duong Date: Fri, 25 Oct 2024 22:15:52 +0200 Subject: [PATCH 36/36] Add root run check --- js/src/vercel.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/js/src/vercel.ts b/js/src/vercel.ts index 840defaba..2d703ba97 100644 --- a/js/src/vercel.ts +++ b/js/src/vercel.ts @@ -687,6 +687,21 @@ export class AISDKExporter { } } + /** @internal */ + protected isRootRun(span: AISDKSpan): boolean { + switch (span.name) { + case "ai.generateText": + case "ai.streamText": + case "ai.generateObject": + case "ai.streamObject": + case "ai.embed": + case "ai.embedMany": + return true; + default: + return false; + } + } + export( spans: unknown[], resultCallback: (result: { code: 0 | 1; error?: Error }) => void @@ -705,10 +720,13 @@ export class AISDKExporter { }; const runId = uuid5(spanId, RUN_ID_NAMESPACE); - const parentRunId = parentId + let parentRunId = parentId ? uuid5(parentId, RUN_ID_NAMESPACE) : undefined; + // in LangSmith we currently only support certain spans + // which may be deeply nested within other traces + if (this.isRootRun(span)) parentRunId = undefined; const traceMap = this.traceByMap[traceId]; const run = this.getRunCreate(span);