From 19c0f57b094e09c4b059377a978a8f9377084763 Mon Sep 17 00:00:00 2001 From: tofuliang Date: Thu, 25 Jul 2024 19:41:48 +0800 Subject: [PATCH 1/6] anthropic[patch]: add vertex and bedrock support, streamResponseChunks optimize --- libs/langchain-anthropic/package.json | 2 + libs/langchain-anthropic/src/chat_models.ts | 208 +++++++------------- 2 files changed, 76 insertions(+), 134 deletions(-) diff --git a/libs/langchain-anthropic/package.json b/libs/langchain-anthropic/package.json index a36254041664..e8b89117c833 100644 --- a/libs/langchain-anthropic/package.json +++ b/libs/langchain-anthropic/package.json @@ -35,7 +35,9 @@ "author": "LangChain", "license": "MIT", "dependencies": { + "@anthropic-ai/bedrock-sdk": "^0.10.0", "@anthropic-ai/sdk": "^0.22.0", + "@anthropic-ai/vertex-sdk": "^0.4.0", "@langchain/core": ">=0.2.16 <0.3.0", "fast-xml-parser": "^4.3.5", "zod": "^3.22.4", diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index a9fcbb118944..152f73aae67a 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -47,17 +47,21 @@ import type { MessageCreateParams, Tool as AnthropicTool, } from "@anthropic-ai/sdk/resources/index.mjs"; -import { concat } from "@langchain/core/utils/stream"; + +import { AnthropicVertex } from "@anthropic-ai/vertex-sdk"; +import { AnthropicBedrock } from "@anthropic-ai/bedrock-sdk"; +import type { ClientOptions as VertexClientOptions } from "@anthropic-ai/vertex-sdk/client"; +import type { ClientOptions as BedrockClientOptions } from "@anthropic-ai/bedrock-sdk/client"; import { AnthropicToolsOutputParser, extractToolCalls, } from "./output_parsers.js"; -import { AnthropicToolResponse } from "./types.js"; import { AnthropicToolChoice, AnthropicToolTypes, handleToolChoice, } from "./utils.js"; +import { AnthropicToolResponse } from "./types.js"; type AnthropicMessage = Anthropic.MessageParam; type AnthropicMessageCreateParams = Anthropic.MessageCreateParamsNonStreaming; @@ -160,18 +164,10 @@ function _makeMessageChunkFromAnthropicEvent( streamUsage: boolean; coerceContentToString: boolean; usageData: { input_tokens: number; output_tokens: number }; - toolUse?: { - id: string; - name: string; - }; } ): { chunk: AIMessageChunk; usageData: { input_tokens: number; output_tokens: number }; - toolUse?: { - id: string; - name: string; - }; } | null { let usageDataCopy = { ...fields.usageData }; @@ -241,10 +237,6 @@ function _makeMessageChunkFromAnthropicEvent( additional_kwargs: {}, }), usageData: usageDataCopy, - toolUse: { - id: data.content_block.id, - name: data.content_block.name, - }, }; } else if ( data.type === "content_block_delta" && @@ -286,25 +278,25 @@ function _makeMessageChunkFromAnthropicEvent( }), usageData: usageDataCopy, }; - } else if (data.type === "content_block_stop" && fields.toolUse) { - // Only yield the ID & name when the tool_use block is complete. - // This is so the names & IDs do not get concatenated. - return { - chunk: new AIMessageChunk({ - content: fields.coerceContentToString - ? "" - : [ + } else if (data.type === "content_block_start" && + data.content_block.type === "text") { + const content = data.content_block?.text; + if (content !== undefined) { + return { + chunk: new AIMessageChunk({ + content: fields.coerceContentToString + ? content + : [ { - id: fields.toolUse.id, - name: fields.toolUse.name, index: data.index, - type: "input_json_delta", + ...data.content_block, }, ], - additional_kwargs: {}, - }), - usageData: usageDataCopy, - }; + additional_kwargs: {}, + }), + usageData: usageDataCopy, + }; + } } return null; @@ -384,6 +376,11 @@ export interface AnthropicInput { streamUsage?: boolean; } +export interface AnthropicPlatform { + platform?: "vertex" | "bedrock"; + platformClientOptions?: VertexClientOptions & BedrockClientOptions; +} + /** * A type representing additional parameters that can be passed to the * Anthropic API. @@ -669,60 +666,14 @@ function extractToken(chunk: AIMessageChunk): string | undefined { return typeof chunk.content[0].input === "string" ? chunk.content[0].input : JSON.stringify(chunk.content[0].input); - } - return undefined; -} - -function extractToolUseContent( - chunk: AIMessageChunk, - concatenatedChunks: AIMessageChunk | undefined -) { - let newConcatenatedChunks = concatenatedChunks; - // Remove `tool_use` content types until the last chunk. - let toolUseContent: - | { - id: string; - type: "tool_use"; - name: string; - input: Record; - } - | undefined; - if (!newConcatenatedChunks) { - newConcatenatedChunks = chunk; - } else { - newConcatenatedChunks = concat(newConcatenatedChunks, chunk); - } - if ( - Array.isArray(newConcatenatedChunks.content) && - newConcatenatedChunks.content.find((c) => c.type === "tool_use") + } else if ( + Array.isArray(chunk.content) && + chunk.content.length >= 1 && + "text" in chunk.content[0] ) { - try { - const toolUseMsg = newConcatenatedChunks.content.find( - (c) => c.type === "tool_use" - ); - if ( - !toolUseMsg || - !("input" in toolUseMsg || "name" in toolUseMsg || "id" in toolUseMsg) - ) - return; - const parsedArgs = JSON.parse(toolUseMsg.input); - if (parsedArgs) { - toolUseContent = { - type: "tool_use", - id: toolUseMsg.id, - name: toolUseMsg.name, - input: parsedArgs, - }; - } - } catch (_) { - // no-op - } + return chunk.content[0].text; } - - return { - toolUseContent, - concatenatedChunks: newConcatenatedChunks, - }; + return undefined; } /** @@ -797,24 +748,43 @@ export class ChatAnthropicMessages< streaming = false; - clientOptions: ClientOptions; + clientOptions: ClientOptions | VertexClientOptions | BedrockClientOptions; + + platform?: "vertex" | "bedrock"; // Used for non-streaming requests - protected batchClient: Anthropic; + protected batchClient: Anthropic | AnthropicVertex | AnthropicBedrock; // Used for streaming requests - protected streamingClient: Anthropic; + protected streamingClient: Anthropic | AnthropicVertex | AnthropicBedrock; streamUsage = true; - constructor(fields?: Partial & BaseChatModelParams) { + constructor(fields?: Partial & BaseChatModelParams & AnthropicPlatform) { super(fields ?? {}); this.anthropicApiKey = fields?.apiKey ?? fields?.anthropicApiKey ?? getEnvironmentVariable("ANTHROPIC_API_KEY"); - if (!this.anthropicApiKey) { + this.platform = fields?.platform; + this.clientOptions = fields?.clientOptions ?? {}; + if (this.platform) { + if (this.platform === "vertex" + && ((!fields?.platformClientOptions?.accessToken && !fields?.platformClientOptions?.googleAuth) + || !fields?.platformClientOptions?.projectId || !fields?.platformClientOptions?.region)) { + throw new Error("Vertex authOption invalid"); + } + if (this.platform === "bedrock" + && (((!fields?.platformClientOptions?.awsSecretKey || !fields?.platformClientOptions?.awsAccessKey) + && !fields?.platformClientOptions?.awsSessionToken) || !fields?.platformClientOptions?.awsRegion)) { + throw new Error("Bedrock authOption invalid"); + } + this.clientOptions = { ...this.clientOptions, ...fields?.platformClientOptions }; + this.model = this.platform === "vertex" ? "claude-3-haiku@20240307" : "anthropic.claude-3-haiku-20240307-v1:0"; + this.modelName = this.model; + } + if (!this.anthropicApiKey && !this.platform) { throw new Error("Anthropic API key not found"); } /** Keep anthropicApiKey for backwards compatibility */ @@ -837,10 +807,20 @@ export class ChatAnthropicMessages< this.stopSequences = fields?.stopSequences ?? this.stopSequences; this.streaming = fields?.streaming ?? false; - this.clientOptions = fields?.clientOptions ?? {}; this.streamUsage = fields?.streamUsage ?? this.streamUsage; } + getClientClass() { + if (this.platform === "vertex") { + return AnthropicVertex; + } + if (this.platform === "bedrock") { + return AnthropicBedrock; + } + + return Anthropic; + } + getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { @@ -979,16 +959,6 @@ export class ChatAnthropicMessages< }); let usageData = { input_tokens: 0, output_tokens: 0 }; - let concatenatedChunks: AIMessageChunk | undefined; - // Anthropic only yields the tool name and id once, so we need to save those - // so we can yield them with the rest of the tool_use content. - let toolUse: - | { - id: string; - name: string; - } - | undefined; - for await (const data of stream) { if (options.signal?.aborted) { stream.controller.abort(); @@ -998,54 +968,25 @@ export class ChatAnthropicMessages< const result = _makeMessageChunkFromAnthropicEvent(data, { streamUsage: !!(this.streamUsage || options.streamUsage), coerceContentToString, - usageData, - toolUse: toolUse - ? { - id: toolUse.id, - name: toolUse.name, - } - : undefined, + usageData }); if (!result) continue; const { chunk, usageData: updatedUsageData, - toolUse: updatedToolUse, } = result; usageData = updatedUsageData; - if (updatedToolUse) { - toolUse = updatedToolUse; - } - const newToolCallChunk = extractToolCallChunk(chunk); - // Maintain concatenatedChunks for accessing the complete `tool_use` content block. - concatenatedChunks = concatenatedChunks - ? concat(concatenatedChunks, chunk) - : chunk; - - let toolUseContent; - const extractedContent = extractToolUseContent(chunk, concatenatedChunks); - if (extractedContent) { - toolUseContent = extractedContent.toolUseContent; - concatenatedChunks = extractedContent.concatenatedChunks; - } - - // Filter partial `tool_use` content, and only add `tool_use` chunks if complete JSON available. - const chunkContent = Array.isArray(chunk.content) - ? chunk.content.filter((c) => c.type !== "tool_use") - : chunk.content; - if (Array.isArray(chunkContent) && toolUseContent) { - chunkContent.push(toolUseContent); - } // Extract the text content token for text field and runManager. const token = extractToken(chunk); yield new ChatGenerationChunk({ message: new AIMessageChunk({ - content: chunkContent, + // Just yield chunk as it is and tool_use will be concat by BaseChatModel._generateUncached(). + content: chunk.content, additional_kwargs: chunk.additional_kwargs, tool_call_chunks: newToolCallChunk ? [newToolCallChunk] : undefined, usage_metadata: chunk.usage_metadata, @@ -1163,6 +1104,7 @@ export class ChatAnthropicMessages< /** * Creates a streaming request with retry. * @param request The parameters for creating a completion. + * @param options * @returns A streaming request. */ protected async createStreamWithRetry( @@ -1171,7 +1113,7 @@ export class ChatAnthropicMessages< ): Promise> { if (!this.streamingClient) { const options_ = this.apiUrl ? { baseURL: this.apiUrl } : undefined; - this.streamingClient = new Anthropic({ + this.streamingClient = new (this.getClientClass())({ ...this.clientOptions, ...options_, apiKey: this.apiKey, @@ -1196,12 +1138,10 @@ export class ChatAnthropicMessages< request: AnthropicMessageCreateParams & Kwargs, options: AnthropicRequestOptions ): Promise { - if (!this.apiKey) { - throw new Error("Missing Anthropic API key."); - } if (!this.batchClient) { const options = this.apiUrl ? { baseURL: this.apiUrl } : undefined; - this.batchClient = new Anthropic({ + + this.batchClient = new (this.getClientClass())({ ...this.clientOptions, ...options, apiKey: this.apiKey, From f0f0d8fb956cfce01ca38b7751d08592a3b05ad0 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 25 Jul 2024 09:52:16 -0700 Subject: [PATCH 2/6] yarn.lock and format/lint --- libs/langchain-anthropic/src/chat_models.ts | 57 +++-- yarn.lock | 237 ++++++++++++++++++++ 2 files changed, 273 insertions(+), 21 deletions(-) diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index 152f73aae67a..0b797451da06 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -278,8 +278,10 @@ function _makeMessageChunkFromAnthropicEvent( }), usageData: usageDataCopy, }; - } else if (data.type === "content_block_start" && - data.content_block.type === "text") { + } else if ( + data.type === "content_block_start" && + data.content_block.type === "text" + ) { const content = data.content_block?.text; if (content !== undefined) { return { @@ -287,11 +289,11 @@ function _makeMessageChunkFromAnthropicEvent( content: fields.coerceContentToString ? content : [ - { - index: data.index, - ...data.content_block, - }, - ], + { + index: data.index, + ...data.content_block, + }, + ], additional_kwargs: {}, }), usageData: usageDataCopy, @@ -760,7 +762,9 @@ export class ChatAnthropicMessages< streamUsage = true; - constructor(fields?: Partial & BaseChatModelParams & AnthropicPlatform) { + constructor( + fields?: Partial & BaseChatModelParams & AnthropicPlatform + ) { super(fields ?? {}); this.anthropicApiKey = @@ -770,18 +774,32 @@ export class ChatAnthropicMessages< this.platform = fields?.platform; this.clientOptions = fields?.clientOptions ?? {}; if (this.platform) { - if (this.platform === "vertex" - && ((!fields?.platformClientOptions?.accessToken && !fields?.platformClientOptions?.googleAuth) - || !fields?.platformClientOptions?.projectId || !fields?.platformClientOptions?.region)) { + if ( + this.platform === "vertex" && + ((!fields?.platformClientOptions?.accessToken && + !fields?.platformClientOptions?.googleAuth) || + !fields?.platformClientOptions?.projectId || + !fields?.platformClientOptions?.region) + ) { throw new Error("Vertex authOption invalid"); } - if (this.platform === "bedrock" - && (((!fields?.platformClientOptions?.awsSecretKey || !fields?.platformClientOptions?.awsAccessKey) - && !fields?.platformClientOptions?.awsSessionToken) || !fields?.platformClientOptions?.awsRegion)) { + if ( + this.platform === "bedrock" && + (((!fields?.platformClientOptions?.awsSecretKey || + !fields?.platformClientOptions?.awsAccessKey) && + !fields?.platformClientOptions?.awsSessionToken) || + !fields?.platformClientOptions?.awsRegion) + ) { throw new Error("Bedrock authOption invalid"); } - this.clientOptions = { ...this.clientOptions, ...fields?.platformClientOptions }; - this.model = this.platform === "vertex" ? "claude-3-haiku@20240307" : "anthropic.claude-3-haiku-20240307-v1:0"; + this.clientOptions = { + ...this.clientOptions, + ...fields?.platformClientOptions, + }; + this.model = + this.platform === "vertex" + ? "claude-3-haiku@20240307" + : "anthropic.claude-3-haiku-20240307-v1:0"; this.modelName = this.model; } if (!this.anthropicApiKey && !this.platform) { @@ -968,14 +986,11 @@ export class ChatAnthropicMessages< const result = _makeMessageChunkFromAnthropicEvent(data, { streamUsage: !!(this.streamUsage || options.streamUsage), coerceContentToString, - usageData + usageData, }); if (!result) continue; - const { - chunk, - usageData: updatedUsageData, - } = result; + const { chunk, usageData: updatedUsageData } = result; usageData = updatedUsageData; diff --git a/yarn.lock b/yarn.lock index c5dcede32a3e..b4672bba68b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -211,6 +211,41 @@ __metadata: languageName: node linkType: hard +"@anthropic-ai/bedrock-sdk@npm:^0.10.0": + version: 0.10.1 + resolution: "@anthropic-ai/bedrock-sdk@npm:0.10.1" + dependencies: + "@anthropic-ai/sdk": ^0 + "@aws-crypto/sha256-js": ^4.0.0 + "@aws-sdk/client-bedrock-runtime": ^3.423.0 + "@aws-sdk/credential-providers": ^3.341.0 + "@smithy/eventstream-serde-node": ^2.0.10 + "@smithy/fetch-http-handler": ^2.2.1 + "@smithy/protocol-http": ^3.0.6 + "@smithy/signature-v4": ^3.1.1 + "@smithy/smithy-client": ^2.1.9 + "@smithy/types": ^2.3.4 + "@smithy/util-base64": ^2.0.0 + checksum: db3c1d14ad4f5d800ef9fee80b401a02761633c25f28db3863be674908c4ad7dab3db8b37ce96785f81dd3278582c50cc7971e0be2c5bd89ca5cd9f9de89e33e + languageName: node + linkType: hard + +"@anthropic-ai/sdk@npm:>=0.14 <1, @anthropic-ai/sdk@npm:^0": + version: 0.24.3 + resolution: "@anthropic-ai/sdk@npm:0.24.3" + dependencies: + "@types/node": ^18.11.18 + "@types/node-fetch": ^2.6.4 + abort-controller: ^3.0.0 + agentkeepalive: ^4.2.1 + form-data-encoder: 1.7.2 + formdata-node: ^4.3.2 + node-fetch: ^2.6.7 + web-streams-polyfill: ^3.2.1 + checksum: 9be0e6207fd29aa97ccc23227c72a242a47545eccf3eb458293befe2cac10a5b62553af133cdc025fd34ad86559736a06578dee406493aef4b8ba0fb1aa483c2 + languageName: node + linkType: hard + "@anthropic-ai/sdk@npm:^0.22.0": version: 0.22.0 resolution: "@anthropic-ai/sdk@npm:0.22.0" @@ -227,6 +262,16 @@ __metadata: languageName: node linkType: hard +"@anthropic-ai/vertex-sdk@npm:^0.4.0": + version: 0.4.0 + resolution: "@anthropic-ai/vertex-sdk@npm:0.4.0" + dependencies: + "@anthropic-ai/sdk": ">=0.14 <1" + google-auth-library: ^9.4.2 + checksum: c225f3e7131ed148aad8df06ff0622abc705f29c7b506b93d70f91f459880c1b73e8d895a93ee6085525625d8bb86308980b7895cd08f7388e9684ff76b1c441 + languageName: node + linkType: hard + "@apache-arrow/ts@npm:^12.0.0": version: 12.0.0 resolution: "@apache-arrow/ts@npm:12.0.0" @@ -372,6 +417,17 @@ __metadata: languageName: node linkType: hard +"@aws-crypto/sha256-js@npm:^4.0.0": + version: 4.0.0 + resolution: "@aws-crypto/sha256-js@npm:4.0.0" + dependencies: + "@aws-crypto/util": ^4.0.0 + "@aws-sdk/types": ^3.222.0 + tslib: ^1.11.1 + checksum: 672d05ec0c15b525d47af9d3442e9141f02e62e40b35c4609bf466faabca3fe5704cc5b6a175dc815a78e32e11af68b7cbc5d838158d748486e745f369b29cfb + languageName: node + linkType: hard + "@aws-crypto/sha256-js@npm:^5.0.0": version: 5.0.0 resolution: "@aws-crypto/sha256-js@npm:5.0.0" @@ -412,6 +468,17 @@ __metadata: languageName: node linkType: hard +"@aws-crypto/util@npm:^4.0.0": + version: 4.0.0 + resolution: "@aws-crypto/util@npm:4.0.0" + dependencies: + "@aws-sdk/types": ^3.222.0 + "@aws-sdk/util-utf8-browser": ^3.0.0 + tslib: ^1.11.1 + checksum: 09c0c6d7791f33f0479e051b636be3fe6d67a8c24cce8419f0906f714457d5398b8071de0d447f61c7d5cf975bbc97ca0cd888551d40276878ae7d39fd298424 + languageName: node + linkType: hard + "@aws-crypto/util@npm:^5.0.0": version: 5.0.0 resolution: "@aws-crypto/util@npm:5.0.0" @@ -619,6 +686,59 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-bedrock-runtime@npm:^3.423.0": + version: 3.616.0 + resolution: "@aws-sdk/client-bedrock-runtime@npm:3.616.0" + dependencies: + "@aws-crypto/sha256-browser": 5.2.0 + "@aws-crypto/sha256-js": 5.2.0 + "@aws-sdk/client-sso-oidc": 3.616.0 + "@aws-sdk/client-sts": 3.616.0 + "@aws-sdk/core": 3.616.0 + "@aws-sdk/credential-provider-node": 3.616.0 + "@aws-sdk/middleware-host-header": 3.616.0 + "@aws-sdk/middleware-logger": 3.609.0 + "@aws-sdk/middleware-recursion-detection": 3.616.0 + "@aws-sdk/middleware-user-agent": 3.616.0 + "@aws-sdk/region-config-resolver": 3.614.0 + "@aws-sdk/types": 3.609.0 + "@aws-sdk/util-endpoints": 3.614.0 + "@aws-sdk/util-user-agent-browser": 3.609.0 + "@aws-sdk/util-user-agent-node": 3.614.0 + "@smithy/config-resolver": ^3.0.5 + "@smithy/core": ^2.2.7 + "@smithy/eventstream-serde-browser": ^3.0.4 + "@smithy/eventstream-serde-config-resolver": ^3.0.3 + "@smithy/eventstream-serde-node": ^3.0.4 + "@smithy/fetch-http-handler": ^3.2.2 + "@smithy/hash-node": ^3.0.3 + "@smithy/invalid-dependency": ^3.0.3 + "@smithy/middleware-content-length": ^3.0.4 + "@smithy/middleware-endpoint": ^3.0.5 + "@smithy/middleware-retry": ^3.0.10 + "@smithy/middleware-serde": ^3.0.3 + "@smithy/middleware-stack": ^3.0.3 + "@smithy/node-config-provider": ^3.1.4 + "@smithy/node-http-handler": ^3.1.3 + "@smithy/protocol-http": ^4.0.4 + "@smithy/smithy-client": ^3.1.8 + "@smithy/types": ^3.3.0 + "@smithy/url-parser": ^3.0.3 + "@smithy/util-base64": ^3.0.0 + "@smithy/util-body-length-browser": ^3.0.0 + "@smithy/util-body-length-node": ^3.0.0 + "@smithy/util-defaults-mode-browser": ^3.0.10 + "@smithy/util-defaults-mode-node": ^3.0.10 + "@smithy/util-endpoints": ^2.0.5 + "@smithy/util-middleware": ^3.0.3 + "@smithy/util-retry": ^3.0.3 + "@smithy/util-stream": ^3.1.0 + "@smithy/util-utf8": ^3.0.0 + tslib: ^2.6.2 + checksum: 06cd8dbcf37687bfe5b6f9d062766f3ff66425428ed6fc7cd7651206d99b9763a043f671cd1d0a93623a5d700bb99ed2179b2f90df3cb42a40c18f3d062ced37 + languageName: node + linkType: hard + "@aws-sdk/client-bedrock-runtime@npm:^3.602.0": version: 3.602.0 resolution: "@aws-sdk/client-bedrock-runtime@npm:3.602.0" @@ -721,6 +841,55 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-cognito-identity@npm:3.616.0": + version: 3.616.0 + resolution: "@aws-sdk/client-cognito-identity@npm:3.616.0" + dependencies: + "@aws-crypto/sha256-browser": 5.2.0 + "@aws-crypto/sha256-js": 5.2.0 + "@aws-sdk/client-sso-oidc": 3.616.0 + "@aws-sdk/client-sts": 3.616.0 + "@aws-sdk/core": 3.616.0 + "@aws-sdk/credential-provider-node": 3.616.0 + "@aws-sdk/middleware-host-header": 3.616.0 + "@aws-sdk/middleware-logger": 3.609.0 + "@aws-sdk/middleware-recursion-detection": 3.616.0 + "@aws-sdk/middleware-user-agent": 3.616.0 + "@aws-sdk/region-config-resolver": 3.614.0 + "@aws-sdk/types": 3.609.0 + "@aws-sdk/util-endpoints": 3.614.0 + "@aws-sdk/util-user-agent-browser": 3.609.0 + "@aws-sdk/util-user-agent-node": 3.614.0 + "@smithy/config-resolver": ^3.0.5 + "@smithy/core": ^2.2.7 + "@smithy/fetch-http-handler": ^3.2.2 + "@smithy/hash-node": ^3.0.3 + "@smithy/invalid-dependency": ^3.0.3 + "@smithy/middleware-content-length": ^3.0.4 + "@smithy/middleware-endpoint": ^3.0.5 + "@smithy/middleware-retry": ^3.0.10 + "@smithy/middleware-serde": ^3.0.3 + "@smithy/middleware-stack": ^3.0.3 + "@smithy/node-config-provider": ^3.1.4 + "@smithy/node-http-handler": ^3.1.3 + "@smithy/protocol-http": ^4.0.4 + "@smithy/smithy-client": ^3.1.8 + "@smithy/types": ^3.3.0 + "@smithy/url-parser": ^3.0.3 + "@smithy/util-base64": ^3.0.0 + "@smithy/util-body-length-browser": ^3.0.0 + "@smithy/util-body-length-node": ^3.0.0 + "@smithy/util-defaults-mode-browser": ^3.0.10 + "@smithy/util-defaults-mode-node": ^3.0.10 + "@smithy/util-endpoints": ^2.0.5 + "@smithy/util-middleware": ^3.0.3 + "@smithy/util-retry": ^3.0.3 + "@smithy/util-utf8": ^3.0.0 + tslib: ^2.6.2 + checksum: 8ff406d4ac7a47315e3261e90358fb9ebaacc9f3f327c81e984c11a09f77f83a089f0df5534fd6721c8df587f9af1a71baf96640407665571738a0b149d20d23 + languageName: node + linkType: hard + "@aws-sdk/client-dynamodb@npm:^3.310.0": version: 3.327.0 resolution: "@aws-sdk/client-dynamodb@npm:3.327.0" @@ -2469,6 +2638,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-cognito-identity@npm:3.616.0": + version: 3.616.0 + resolution: "@aws-sdk/credential-provider-cognito-identity@npm:3.616.0" + dependencies: + "@aws-sdk/client-cognito-identity": 3.616.0 + "@aws-sdk/types": 3.609.0 + "@smithy/property-provider": ^3.1.3 + "@smithy/types": ^3.3.0 + tslib: ^2.6.2 + checksum: 3aada71f75b633badfaf84c7cb8f4e6899f19d49e99a18cd1d3e591acf3e5aa495293e25ead945e2833630bae07a7bceca885fbf122ce8a4e6b4778251c29c46 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-env@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/credential-provider-env@npm:3.310.0" @@ -3473,6 +3655,30 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-providers@npm:^3.341.0": + version: 3.617.0 + resolution: "@aws-sdk/credential-providers@npm:3.617.0" + dependencies: + "@aws-sdk/client-cognito-identity": 3.616.0 + "@aws-sdk/client-sso": 3.616.0 + "@aws-sdk/client-sts": 3.616.0 + "@aws-sdk/credential-provider-cognito-identity": 3.616.0 + "@aws-sdk/credential-provider-env": 3.609.0 + "@aws-sdk/credential-provider-http": 3.616.0 + "@aws-sdk/credential-provider-ini": 3.616.0 + "@aws-sdk/credential-provider-node": 3.616.0 + "@aws-sdk/credential-provider-process": 3.614.0 + "@aws-sdk/credential-provider-sso": 3.616.0 + "@aws-sdk/credential-provider-web-identity": 3.609.0 + "@aws-sdk/types": 3.609.0 + "@smithy/credential-provider-imds": ^3.1.4 + "@smithy/property-provider": ^3.1.3 + "@smithy/types": ^3.3.0 + tslib: ^2.6.2 + checksum: 47b55918526f466a96a16eae73563406be305a859ac19b824f829be89513313066d3018223d1210038c3d492298cf01ea50cc62c05b0cbd1d3aed52ecefdc3af + languageName: node + linkType: hard + "@aws-sdk/credential-providers@npm:^3.583.0": version: 3.592.0 resolution: "@aws-sdk/credential-providers@npm:3.592.0" @@ -10661,7 +10867,9 @@ __metadata: version: 0.0.0-use.local resolution: "@langchain/anthropic@workspace:libs/langchain-anthropic" dependencies: + "@anthropic-ai/bedrock-sdk": ^0.10.0 "@anthropic-ai/sdk": ^0.22.0 + "@anthropic-ai/vertex-sdk": ^0.4.0 "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.16 <0.3.0" "@langchain/scripts": ~0.0.20 @@ -15874,6 +16082,21 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^3.1.1": + version: 3.1.2 + resolution: "@smithy/signature-v4@npm:3.1.2" + dependencies: + "@smithy/is-array-buffer": ^3.0.0 + "@smithy/types": ^3.3.0 + "@smithy/util-hex-encoding": ^3.0.0 + "@smithy/util-middleware": ^3.0.3 + "@smithy/util-uri-escape": ^3.0.0 + "@smithy/util-utf8": ^3.0.0 + tslib: ^2.6.2 + checksum: 5d10bfe89116a79ea13fe159b05d7fc10f7f67f11333cad6b96990b862cae9ed6c7c7a466d0bf296368610a43e34730feab0ec62f214019f18c61115d4dc8923 + languageName: node + linkType: hard + "@smithy/signature-v4@npm:^4.0.0": version: 4.0.0 resolution: "@smithy/signature-v4@npm:4.0.0" @@ -27577,6 +27800,20 @@ __metadata: languageName: node linkType: hard +"google-auth-library@npm:^9.4.2": + version: 9.11.0 + resolution: "google-auth-library@npm:9.11.0" + dependencies: + base64-js: ^1.3.0 + ecdsa-sig-formatter: ^1.0.11 + gaxios: ^6.1.1 + gcp-metadata: ^6.1.0 + gtoken: ^7.0.0 + jws: ^4.0.0 + checksum: 984d344b5e0a21ea1e097d06e27173035619c0e8f89a363e538b445adb1414b79e938b56b4432aa36fda074c5922fa6a34f9b64734765c01dff73c45c8568554 + languageName: node + linkType: hard + "google-gax@npm:^4.0.3": version: 4.3.3 resolution: "google-gax@npm:4.3.3" From 7a3e2dd36434601fd268326ea5d195bd51558645 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 26 Jul 2024 15:28:00 -0700 Subject: [PATCH 3/6] drop bedrock/vertex --- libs/langchain-anthropic/package.json | 2 - libs/langchain-anthropic/src/chat_models.ts | 75 +------ yarn.lock | 237 -------------------- 3 files changed, 12 insertions(+), 302 deletions(-) diff --git a/libs/langchain-anthropic/package.json b/libs/langchain-anthropic/package.json index e8b89117c833..a36254041664 100644 --- a/libs/langchain-anthropic/package.json +++ b/libs/langchain-anthropic/package.json @@ -35,9 +35,7 @@ "author": "LangChain", "license": "MIT", "dependencies": { - "@anthropic-ai/bedrock-sdk": "^0.10.0", "@anthropic-ai/sdk": "^0.22.0", - "@anthropic-ai/vertex-sdk": "^0.4.0", "@langchain/core": ">=0.2.16 <0.3.0", "fast-xml-parser": "^4.3.5", "zod": "^3.22.4", diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index 0b797451da06..394e0f62b6cf 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -48,10 +48,6 @@ import type { Tool as AnthropicTool, } from "@anthropic-ai/sdk/resources/index.mjs"; -import { AnthropicVertex } from "@anthropic-ai/vertex-sdk"; -import { AnthropicBedrock } from "@anthropic-ai/bedrock-sdk"; -import type { ClientOptions as VertexClientOptions } from "@anthropic-ai/vertex-sdk/client"; -import type { ClientOptions as BedrockClientOptions } from "@anthropic-ai/bedrock-sdk/client"; import { AnthropicToolsOutputParser, extractToolCalls, @@ -378,11 +374,6 @@ export interface AnthropicInput { streamUsage?: boolean; } -export interface AnthropicPlatform { - platform?: "vertex" | "bedrock"; - platformClientOptions?: VertexClientOptions & BedrockClientOptions; -} - /** * A type representing additional parameters that can be passed to the * Anthropic API. @@ -750,61 +741,28 @@ export class ChatAnthropicMessages< streaming = false; - clientOptions: ClientOptions | VertexClientOptions | BedrockClientOptions; - - platform?: "vertex" | "bedrock"; + clientOptions: ClientOptions; // Used for non-streaming requests - protected batchClient: Anthropic | AnthropicVertex | AnthropicBedrock; + protected batchClient: Anthropic; // Used for streaming requests - protected streamingClient: Anthropic | AnthropicVertex | AnthropicBedrock; + protected streamingClient: Anthropic; streamUsage = true; - constructor( - fields?: Partial & BaseChatModelParams & AnthropicPlatform - ) { + constructor(fields?: Partial & BaseChatModelParams) { super(fields ?? {}); this.anthropicApiKey = fields?.apiKey ?? fields?.anthropicApiKey ?? getEnvironmentVariable("ANTHROPIC_API_KEY"); - this.platform = fields?.platform; - this.clientOptions = fields?.clientOptions ?? {}; - if (this.platform) { - if ( - this.platform === "vertex" && - ((!fields?.platformClientOptions?.accessToken && - !fields?.platformClientOptions?.googleAuth) || - !fields?.platformClientOptions?.projectId || - !fields?.platformClientOptions?.region) - ) { - throw new Error("Vertex authOption invalid"); - } - if ( - this.platform === "bedrock" && - (((!fields?.platformClientOptions?.awsSecretKey || - !fields?.platformClientOptions?.awsAccessKey) && - !fields?.platformClientOptions?.awsSessionToken) || - !fields?.platformClientOptions?.awsRegion) - ) { - throw new Error("Bedrock authOption invalid"); - } - this.clientOptions = { - ...this.clientOptions, - ...fields?.platformClientOptions, - }; - this.model = - this.platform === "vertex" - ? "claude-3-haiku@20240307" - : "anthropic.claude-3-haiku-20240307-v1:0"; - this.modelName = this.model; - } - if (!this.anthropicApiKey && !this.platform) { + + if (!this.anthropicApiKey) { throw new Error("Anthropic API key not found"); } + this.clientOptions = fields?.clientOptions ?? {}; /** Keep anthropicApiKey for backwards compatibility */ this.apiKey = this.anthropicApiKey; @@ -828,17 +786,6 @@ export class ChatAnthropicMessages< this.streamUsage = fields?.streamUsage ?? this.streamUsage; } - getClientClass() { - if (this.platform === "vertex") { - return AnthropicVertex; - } - if (this.platform === "bedrock") { - return AnthropicBedrock; - } - - return Anthropic; - } - getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { const params = this.invocationParams(options); return { @@ -1128,7 +1075,7 @@ export class ChatAnthropicMessages< ): Promise> { if (!this.streamingClient) { const options_ = this.apiUrl ? { baseURL: this.apiUrl } : undefined; - this.streamingClient = new (this.getClientClass())({ + this.streamingClient = new Anthropic({ ...this.clientOptions, ...options_, apiKey: this.apiKey, @@ -1155,8 +1102,10 @@ export class ChatAnthropicMessages< ): Promise { if (!this.batchClient) { const options = this.apiUrl ? { baseURL: this.apiUrl } : undefined; - - this.batchClient = new (this.getClientClass())({ + if (!this.apiKey) { + throw new Error("Missing Anthropic API key."); + } + this.batchClient = new Anthropic({ ...this.clientOptions, ...options, apiKey: this.apiKey, diff --git a/yarn.lock b/yarn.lock index 886452659b52..57851482b66f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -211,41 +211,6 @@ __metadata: languageName: node linkType: hard -"@anthropic-ai/bedrock-sdk@npm:^0.10.0": - version: 0.10.1 - resolution: "@anthropic-ai/bedrock-sdk@npm:0.10.1" - dependencies: - "@anthropic-ai/sdk": ^0 - "@aws-crypto/sha256-js": ^4.0.0 - "@aws-sdk/client-bedrock-runtime": ^3.423.0 - "@aws-sdk/credential-providers": ^3.341.0 - "@smithy/eventstream-serde-node": ^2.0.10 - "@smithy/fetch-http-handler": ^2.2.1 - "@smithy/protocol-http": ^3.0.6 - "@smithy/signature-v4": ^3.1.1 - "@smithy/smithy-client": ^2.1.9 - "@smithy/types": ^2.3.4 - "@smithy/util-base64": ^2.0.0 - checksum: db3c1d14ad4f5d800ef9fee80b401a02761633c25f28db3863be674908c4ad7dab3db8b37ce96785f81dd3278582c50cc7971e0be2c5bd89ca5cd9f9de89e33e - languageName: node - linkType: hard - -"@anthropic-ai/sdk@npm:>=0.14 <1, @anthropic-ai/sdk@npm:^0": - version: 0.24.3 - resolution: "@anthropic-ai/sdk@npm:0.24.3" - dependencies: - "@types/node": ^18.11.18 - "@types/node-fetch": ^2.6.4 - abort-controller: ^3.0.0 - agentkeepalive: ^4.2.1 - form-data-encoder: 1.7.2 - formdata-node: ^4.3.2 - node-fetch: ^2.6.7 - web-streams-polyfill: ^3.2.1 - checksum: 9be0e6207fd29aa97ccc23227c72a242a47545eccf3eb458293befe2cac10a5b62553af133cdc025fd34ad86559736a06578dee406493aef4b8ba0fb1aa483c2 - languageName: node - linkType: hard - "@anthropic-ai/sdk@npm:^0.22.0": version: 0.22.0 resolution: "@anthropic-ai/sdk@npm:0.22.0" @@ -262,16 +227,6 @@ __metadata: languageName: node linkType: hard -"@anthropic-ai/vertex-sdk@npm:^0.4.0": - version: 0.4.0 - resolution: "@anthropic-ai/vertex-sdk@npm:0.4.0" - dependencies: - "@anthropic-ai/sdk": ">=0.14 <1" - google-auth-library: ^9.4.2 - checksum: c225f3e7131ed148aad8df06ff0622abc705f29c7b506b93d70f91f459880c1b73e8d895a93ee6085525625d8bb86308980b7895cd08f7388e9684ff76b1c441 - languageName: node - linkType: hard - "@apache-arrow/ts@npm:^12.0.0": version: 12.0.0 resolution: "@apache-arrow/ts@npm:12.0.0" @@ -417,17 +372,6 @@ __metadata: languageName: node linkType: hard -"@aws-crypto/sha256-js@npm:^4.0.0": - version: 4.0.0 - resolution: "@aws-crypto/sha256-js@npm:4.0.0" - dependencies: - "@aws-crypto/util": ^4.0.0 - "@aws-sdk/types": ^3.222.0 - tslib: ^1.11.1 - checksum: 672d05ec0c15b525d47af9d3442e9141f02e62e40b35c4609bf466faabca3fe5704cc5b6a175dc815a78e32e11af68b7cbc5d838158d748486e745f369b29cfb - languageName: node - linkType: hard - "@aws-crypto/sha256-js@npm:^5.0.0": version: 5.0.0 resolution: "@aws-crypto/sha256-js@npm:5.0.0" @@ -468,17 +412,6 @@ __metadata: languageName: node linkType: hard -"@aws-crypto/util@npm:^4.0.0": - version: 4.0.0 - resolution: "@aws-crypto/util@npm:4.0.0" - dependencies: - "@aws-sdk/types": ^3.222.0 - "@aws-sdk/util-utf8-browser": ^3.0.0 - tslib: ^1.11.1 - checksum: 09c0c6d7791f33f0479e051b636be3fe6d67a8c24cce8419f0906f714457d5398b8071de0d447f61c7d5cf975bbc97ca0cd888551d40276878ae7d39fd298424 - languageName: node - linkType: hard - "@aws-crypto/util@npm:^5.0.0": version: 5.0.0 resolution: "@aws-crypto/util@npm:5.0.0" @@ -686,59 +619,6 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-bedrock-runtime@npm:^3.423.0": - version: 3.616.0 - resolution: "@aws-sdk/client-bedrock-runtime@npm:3.616.0" - dependencies: - "@aws-crypto/sha256-browser": 5.2.0 - "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.616.0 - "@aws-sdk/client-sts": 3.616.0 - "@aws-sdk/core": 3.616.0 - "@aws-sdk/credential-provider-node": 3.616.0 - "@aws-sdk/middleware-host-header": 3.616.0 - "@aws-sdk/middleware-logger": 3.609.0 - "@aws-sdk/middleware-recursion-detection": 3.616.0 - "@aws-sdk/middleware-user-agent": 3.616.0 - "@aws-sdk/region-config-resolver": 3.614.0 - "@aws-sdk/types": 3.609.0 - "@aws-sdk/util-endpoints": 3.614.0 - "@aws-sdk/util-user-agent-browser": 3.609.0 - "@aws-sdk/util-user-agent-node": 3.614.0 - "@smithy/config-resolver": ^3.0.5 - "@smithy/core": ^2.2.7 - "@smithy/eventstream-serde-browser": ^3.0.4 - "@smithy/eventstream-serde-config-resolver": ^3.0.3 - "@smithy/eventstream-serde-node": ^3.0.4 - "@smithy/fetch-http-handler": ^3.2.2 - "@smithy/hash-node": ^3.0.3 - "@smithy/invalid-dependency": ^3.0.3 - "@smithy/middleware-content-length": ^3.0.4 - "@smithy/middleware-endpoint": ^3.0.5 - "@smithy/middleware-retry": ^3.0.10 - "@smithy/middleware-serde": ^3.0.3 - "@smithy/middleware-stack": ^3.0.3 - "@smithy/node-config-provider": ^3.1.4 - "@smithy/node-http-handler": ^3.1.3 - "@smithy/protocol-http": ^4.0.4 - "@smithy/smithy-client": ^3.1.8 - "@smithy/types": ^3.3.0 - "@smithy/url-parser": ^3.0.3 - "@smithy/util-base64": ^3.0.0 - "@smithy/util-body-length-browser": ^3.0.0 - "@smithy/util-body-length-node": ^3.0.0 - "@smithy/util-defaults-mode-browser": ^3.0.10 - "@smithy/util-defaults-mode-node": ^3.0.10 - "@smithy/util-endpoints": ^2.0.5 - "@smithy/util-middleware": ^3.0.3 - "@smithy/util-retry": ^3.0.3 - "@smithy/util-stream": ^3.1.0 - "@smithy/util-utf8": ^3.0.0 - tslib: ^2.6.2 - checksum: 06cd8dbcf37687bfe5b6f9d062766f3ff66425428ed6fc7cd7651206d99b9763a043f671cd1d0a93623a5d700bb99ed2179b2f90df3cb42a40c18f3d062ced37 - languageName: node - linkType: hard - "@aws-sdk/client-bedrock-runtime@npm:^3.602.0": version: 3.602.0 resolution: "@aws-sdk/client-bedrock-runtime@npm:3.602.0" @@ -841,55 +721,6 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/client-cognito-identity@npm:3.616.0": - version: 3.616.0 - resolution: "@aws-sdk/client-cognito-identity@npm:3.616.0" - dependencies: - "@aws-crypto/sha256-browser": 5.2.0 - "@aws-crypto/sha256-js": 5.2.0 - "@aws-sdk/client-sso-oidc": 3.616.0 - "@aws-sdk/client-sts": 3.616.0 - "@aws-sdk/core": 3.616.0 - "@aws-sdk/credential-provider-node": 3.616.0 - "@aws-sdk/middleware-host-header": 3.616.0 - "@aws-sdk/middleware-logger": 3.609.0 - "@aws-sdk/middleware-recursion-detection": 3.616.0 - "@aws-sdk/middleware-user-agent": 3.616.0 - "@aws-sdk/region-config-resolver": 3.614.0 - "@aws-sdk/types": 3.609.0 - "@aws-sdk/util-endpoints": 3.614.0 - "@aws-sdk/util-user-agent-browser": 3.609.0 - "@aws-sdk/util-user-agent-node": 3.614.0 - "@smithy/config-resolver": ^3.0.5 - "@smithy/core": ^2.2.7 - "@smithy/fetch-http-handler": ^3.2.2 - "@smithy/hash-node": ^3.0.3 - "@smithy/invalid-dependency": ^3.0.3 - "@smithy/middleware-content-length": ^3.0.4 - "@smithy/middleware-endpoint": ^3.0.5 - "@smithy/middleware-retry": ^3.0.10 - "@smithy/middleware-serde": ^3.0.3 - "@smithy/middleware-stack": ^3.0.3 - "@smithy/node-config-provider": ^3.1.4 - "@smithy/node-http-handler": ^3.1.3 - "@smithy/protocol-http": ^4.0.4 - "@smithy/smithy-client": ^3.1.8 - "@smithy/types": ^3.3.0 - "@smithy/url-parser": ^3.0.3 - "@smithy/util-base64": ^3.0.0 - "@smithy/util-body-length-browser": ^3.0.0 - "@smithy/util-body-length-node": ^3.0.0 - "@smithy/util-defaults-mode-browser": ^3.0.10 - "@smithy/util-defaults-mode-node": ^3.0.10 - "@smithy/util-endpoints": ^2.0.5 - "@smithy/util-middleware": ^3.0.3 - "@smithy/util-retry": ^3.0.3 - "@smithy/util-utf8": ^3.0.0 - tslib: ^2.6.2 - checksum: 8ff406d4ac7a47315e3261e90358fb9ebaacc9f3f327c81e984c11a09f77f83a089f0df5534fd6721c8df587f9af1a71baf96640407665571738a0b149d20d23 - languageName: node - linkType: hard - "@aws-sdk/client-dynamodb@npm:^3.310.0": version: 3.327.0 resolution: "@aws-sdk/client-dynamodb@npm:3.327.0" @@ -2638,19 +2469,6 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-provider-cognito-identity@npm:3.616.0": - version: 3.616.0 - resolution: "@aws-sdk/credential-provider-cognito-identity@npm:3.616.0" - dependencies: - "@aws-sdk/client-cognito-identity": 3.616.0 - "@aws-sdk/types": 3.609.0 - "@smithy/property-provider": ^3.1.3 - "@smithy/types": ^3.3.0 - tslib: ^2.6.2 - checksum: 3aada71f75b633badfaf84c7cb8f4e6899f19d49e99a18cd1d3e591acf3e5aa495293e25ead945e2833630bae07a7bceca885fbf122ce8a4e6b4778251c29c46 - languageName: node - linkType: hard - "@aws-sdk/credential-provider-env@npm:3.310.0": version: 3.310.0 resolution: "@aws-sdk/credential-provider-env@npm:3.310.0" @@ -3655,30 +3473,6 @@ __metadata: languageName: node linkType: hard -"@aws-sdk/credential-providers@npm:^3.341.0": - version: 3.617.0 - resolution: "@aws-sdk/credential-providers@npm:3.617.0" - dependencies: - "@aws-sdk/client-cognito-identity": 3.616.0 - "@aws-sdk/client-sso": 3.616.0 - "@aws-sdk/client-sts": 3.616.0 - "@aws-sdk/credential-provider-cognito-identity": 3.616.0 - "@aws-sdk/credential-provider-env": 3.609.0 - "@aws-sdk/credential-provider-http": 3.616.0 - "@aws-sdk/credential-provider-ini": 3.616.0 - "@aws-sdk/credential-provider-node": 3.616.0 - "@aws-sdk/credential-provider-process": 3.614.0 - "@aws-sdk/credential-provider-sso": 3.616.0 - "@aws-sdk/credential-provider-web-identity": 3.609.0 - "@aws-sdk/types": 3.609.0 - "@smithy/credential-provider-imds": ^3.1.4 - "@smithy/property-provider": ^3.1.3 - "@smithy/types": ^3.3.0 - tslib: ^2.6.2 - checksum: 47b55918526f466a96a16eae73563406be305a859ac19b824f829be89513313066d3018223d1210038c3d492298cf01ea50cc62c05b0cbd1d3aed52ecefdc3af - languageName: node - linkType: hard - "@aws-sdk/credential-providers@npm:^3.583.0": version: 3.592.0 resolution: "@aws-sdk/credential-providers@npm:3.592.0" @@ -11028,9 +10822,7 @@ __metadata: version: 0.0.0-use.local resolution: "@langchain/anthropic@workspace:libs/langchain-anthropic" dependencies: - "@anthropic-ai/bedrock-sdk": ^0.10.0 "@anthropic-ai/sdk": ^0.22.0 - "@anthropic-ai/vertex-sdk": ^0.4.0 "@jest/globals": ^29.5.0 "@langchain/core": ">=0.2.16 <0.3.0" "@langchain/scripts": ~0.0.20 @@ -16280,21 +16072,6 @@ __metadata: languageName: node linkType: hard -"@smithy/signature-v4@npm:^3.1.1": - version: 3.1.2 - resolution: "@smithy/signature-v4@npm:3.1.2" - dependencies: - "@smithy/is-array-buffer": ^3.0.0 - "@smithy/types": ^3.3.0 - "@smithy/util-hex-encoding": ^3.0.0 - "@smithy/util-middleware": ^3.0.3 - "@smithy/util-uri-escape": ^3.0.0 - "@smithy/util-utf8": ^3.0.0 - tslib: ^2.6.2 - checksum: 5d10bfe89116a79ea13fe159b05d7fc10f7f67f11333cad6b96990b862cae9ed6c7c7a466d0bf296368610a43e34730feab0ec62f214019f18c61115d4dc8923 - languageName: node - linkType: hard - "@smithy/signature-v4@npm:^4.0.0": version: 4.0.0 resolution: "@smithy/signature-v4@npm:4.0.0" @@ -28075,20 +27852,6 @@ __metadata: languageName: node linkType: hard -"google-auth-library@npm:^9.4.2": - version: 9.11.0 - resolution: "google-auth-library@npm:9.11.0" - dependencies: - base64-js: ^1.3.0 - ecdsa-sig-formatter: ^1.0.11 - gaxios: ^6.1.1 - gcp-metadata: ^6.1.0 - gtoken: ^7.0.0 - jws: ^4.0.0 - checksum: 984d344b5e0a21ea1e097d06e27173035619c0e8f89a363e538b445adb1414b79e938b56b4432aa36fda074c5922fa6a34f9b64734765c01dff73c45c8568554 - languageName: node - linkType: hard - "google-gax@npm:^4.0.3": version: 4.3.3 resolution: "google-gax@npm:4.3.3" From a74504f69eaf09be962a28f3d8952a8ee9766244 Mon Sep 17 00:00:00 2001 From: tofuliang Date: Mon, 29 Jul 2024 10:32:43 +0800 Subject: [PATCH 4/6] anthropic[patch]: fix stream usage calculation --- libs/langchain-anthropic/src/chat_models.ts | 87 ++++----------------- 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index 394e0f62b6cf..d5aad1d5c47c 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -67,8 +67,7 @@ type AnthropicMessageStreamEvent = Anthropic.MessageStreamEvent; type AnthropicRequestOptions = Anthropic.RequestOptions; export interface ChatAnthropicCallOptions - extends BaseChatModelCallOptions, - Pick { + extends BaseChatModelCallOptions { tools?: AnthropicToolTypes[]; /** * Whether or not to specify what tool the model should use @@ -157,15 +156,11 @@ function isAnthropicTool(tool: any): tool is AnthropicTool { function _makeMessageChunkFromAnthropicEvent( data: Anthropic.Messages.RawMessageStreamEvent, fields: { - streamUsage: boolean; coerceContentToString: boolean; - usageData: { input_tokens: number; output_tokens: number }; } ): { chunk: AIMessageChunk; - usageData: { input_tokens: number; output_tokens: number }; } | null { - let usageDataCopy = { ...fields.usageData }; if (data.type === "message_start") { // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -177,15 +172,11 @@ function _makeMessageChunkFromAnthropicEvent( filteredAdditionalKwargs[key] = value; } } - usageDataCopy = usage; - let usageMetadata: UsageMetadata | undefined; - if (fields.streamUsage) { - usageMetadata = { - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - total_tokens: usage.input_tokens + usage.output_tokens, - }; - } + const usageMetadata: UsageMetadata = { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + total_tokens: usage.input_tokens + usage.output_tokens, + }; return { chunk: new AIMessageChunk({ content: fields.coerceContentToString ? "" : [], @@ -193,27 +184,19 @@ function _makeMessageChunkFromAnthropicEvent( usage_metadata: usageMetadata, id: data.message.id, }), - usageData: usageDataCopy, }; } else if (data.type === "message_delta") { - let usageMetadata: UsageMetadata | undefined; - if (fields.streamUsage) { - usageMetadata = { - input_tokens: data.usage.output_tokens, - output_tokens: 0, + const usageMetadata: UsageMetadata = { + input_tokens: 0, + output_tokens: data.usage.output_tokens, total_tokens: data.usage.output_tokens, }; - } - if (data?.usage !== undefined) { - usageDataCopy.output_tokens += data.usage.output_tokens; - } return { chunk: new AIMessageChunk({ content: fields.coerceContentToString ? "" : [], additional_kwargs: { ...data.delta }, usage_metadata: usageMetadata, }), - usageData: usageDataCopy, }; } else if ( data.type === "content_block_start" && @@ -232,7 +215,6 @@ function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } else if ( data.type === "content_block_delta" && @@ -252,7 +234,6 @@ function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } } else if ( @@ -272,12 +253,9 @@ function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; - } else if ( - data.type === "content_block_start" && - data.content_block.type === "text" - ) { + } else if (data.type === "content_block_start" && + data.content_block.type === "text") { const content = data.content_block?.text; if (content !== undefined) { return { @@ -285,14 +263,13 @@ function _makeMessageChunkFromAnthropicEvent( content: fields.coerceContentToString ? content : [ - { - index: data.index, - ...data.content_block, - }, - ], + { + index: data.index, + ...data.content_block, + }, + ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } } @@ -367,11 +344,6 @@ export interface AnthropicInput { */ invocationKwargs?: Kwargs; - /** - * Whether or not to include token usage data in streamed chunks. - * @default true - */ - streamUsage?: boolean; } /** @@ -749,8 +721,6 @@ export class ChatAnthropicMessages< // Used for streaming requests protected streamingClient: Anthropic; - streamUsage = true; - constructor(fields?: Partial & BaseChatModelParams) { super(fields ?? {}); @@ -783,7 +753,6 @@ export class ChatAnthropicMessages< this.stopSequences = fields?.stopSequences ?? this.stopSequences; this.streaming = fields?.streaming ?? false; - this.streamUsage = fields?.streamUsage ?? this.streamUsage; } getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { @@ -922,7 +891,6 @@ export class ChatAnthropicMessages< ...formattedMessages, stream: true, }); - let usageData = { input_tokens: 0, output_tokens: 0 }; for await (const data of stream) { if (options.signal?.aborted) { @@ -931,15 +899,11 @@ export class ChatAnthropicMessages< } const result = _makeMessageChunkFromAnthropicEvent(data, { - streamUsage: !!(this.streamUsage || options.streamUsage), coerceContentToString, - usageData, }); if (!result) continue; - const { chunk, usageData: updatedUsageData } = result; - - usageData = updatedUsageData; + const { chunk } = result; const newToolCallChunk = extractToolCallChunk(chunk); @@ -962,23 +926,6 @@ export class ChatAnthropicMessages< await runManager?.handleLLMNewToken(token); } } - - let usageMetadata: UsageMetadata | undefined; - if (this.streamUsage || options.streamUsage) { - usageMetadata = { - input_tokens: usageData.input_tokens, - output_tokens: usageData.output_tokens, - total_tokens: usageData.input_tokens + usageData.output_tokens, - }; - } - yield new ChatGenerationChunk({ - message: new AIMessageChunk({ - content: coerceContentToString ? "" : [], - additional_kwargs: { usage: usageData }, - usage_metadata: usageMetadata, - }), - text: "", - }); } /** @ignore */ From f9e48255afcf3a64c4088c21841b9165f219b2af Mon Sep 17 00:00:00 2001 From: bracesproul Date: Mon, 29 Jul 2024 16:24:08 -0700 Subject: [PATCH 5/6] add back stream usage param --- libs/langchain-anthropic/src/chat_models.ts | 46 +++++++++++++-------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index d5aad1d5c47c..698e6dd1f814 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -67,7 +67,8 @@ type AnthropicMessageStreamEvent = Anthropic.MessageStreamEvent; type AnthropicRequestOptions = Anthropic.RequestOptions; export interface ChatAnthropicCallOptions - extends BaseChatModelCallOptions { + extends BaseChatModelCallOptions, + Pick { tools?: AnthropicToolTypes[]; /** * Whether or not to specify what tool the model should use @@ -156,12 +157,12 @@ function isAnthropicTool(tool: any): tool is AnthropicTool { function _makeMessageChunkFromAnthropicEvent( data: Anthropic.Messages.RawMessageStreamEvent, fields: { + streamUsage: boolean; coerceContentToString: boolean; } ): { chunk: AIMessageChunk; } | null { - if (data.type === "message_start") { // eslint-disable-next-line @typescript-eslint/no-unused-vars const { content, usage, ...additionalKwargs } = data.message; @@ -181,21 +182,21 @@ function _makeMessageChunkFromAnthropicEvent( chunk: new AIMessageChunk({ content: fields.coerceContentToString ? "" : [], additional_kwargs: filteredAdditionalKwargs, - usage_metadata: usageMetadata, + usage_metadata: fields.streamUsage ? usageMetadata : undefined, id: data.message.id, }), }; } else if (data.type === "message_delta") { const usageMetadata: UsageMetadata = { - input_tokens: 0, - output_tokens: data.usage.output_tokens, - total_tokens: data.usage.output_tokens, - }; + input_tokens: 0, + output_tokens: data.usage.output_tokens, + total_tokens: data.usage.output_tokens, + }; return { chunk: new AIMessageChunk({ content: fields.coerceContentToString ? "" : [], additional_kwargs: { ...data.delta }, - usage_metadata: usageMetadata, + usage_metadata: fields.streamUsage ? usageMetadata : undefined, }), }; } else if ( @@ -254,8 +255,10 @@ function _makeMessageChunkFromAnthropicEvent( additional_kwargs: {}, }), }; - } else if (data.type === "content_block_start" && - data.content_block.type === "text") { + } else if ( + data.type === "content_block_start" && + data.content_block.type === "text" + ) { const content = data.content_block?.text; if (content !== undefined) { return { @@ -263,11 +266,11 @@ function _makeMessageChunkFromAnthropicEvent( content: fields.coerceContentToString ? content : [ - { - index: data.index, - ...data.content_block, - }, - ], + { + index: data.index, + ...data.content_block, + }, + ], additional_kwargs: {}, }), }; @@ -344,6 +347,11 @@ export interface AnthropicInput { */ invocationKwargs?: Kwargs; + /** + * Whether or not to include token usage data in streamed chunks. + * @default true + */ + streamUsage?: boolean; } /** @@ -721,6 +729,8 @@ export class ChatAnthropicMessages< // Used for streaming requests protected streamingClient: Anthropic; + streamUsage = true; + constructor(fields?: Partial & BaseChatModelParams) { super(fields ?? {}); @@ -753,6 +763,7 @@ export class ChatAnthropicMessages< this.stopSequences = fields?.stopSequences ?? this.stopSequences; this.streaming = fields?.streaming ?? false; + this.streamUsage = fields?.streamUsage ?? this.streamUsage; } getLsParams(options: this["ParsedCallOptions"]): LangSmithParams { @@ -897,8 +908,9 @@ export class ChatAnthropicMessages< stream.controller.abort(); throw new Error("AbortError: User aborted the request."); } - + const shouldStreamUsage = this.streamUsage ?? options.streamUsage; const result = _makeMessageChunkFromAnthropicEvent(data, { + streamUsage: shouldStreamUsage, coerceContentToString, }); if (!result) continue; @@ -915,7 +927,7 @@ export class ChatAnthropicMessages< content: chunk.content, additional_kwargs: chunk.additional_kwargs, tool_call_chunks: newToolCallChunk ? [newToolCallChunk] : undefined, - usage_metadata: chunk.usage_metadata, + usage_metadata: shouldStreamUsage ? chunk.usage_metadata : undefined, response_metadata: chunk.response_metadata, id: chunk.id, }), From 9ace4ec80afce77502fc1abfcb07307a24f185ad Mon Sep 17 00:00:00 2001 From: bracesproul Date: Mon, 29 Jul 2024 16:32:47 -0700 Subject: [PATCH 6/6] fix issues with merge --- libs/langchain-anthropic/src/chat_models.ts | 12 +---- .../src/utils/message_outputs.ts | 44 +++++-------------- 2 files changed, 14 insertions(+), 42 deletions(-) diff --git a/libs/langchain-anthropic/src/chat_models.ts b/libs/langchain-anthropic/src/chat_models.ts index fef84e4e014f..fd725018298d 100644 --- a/libs/langchain-anthropic/src/chat_models.ts +++ b/libs/langchain-anthropic/src/chat_models.ts @@ -2,11 +2,7 @@ import { Anthropic, type ClientOptions } from "@anthropic-ai/sdk"; import type { Stream } from "@anthropic-ai/sdk/streaming"; import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; -import { - AIMessageChunk, - type BaseMessage, - UsageMetadata, -} from "@langchain/core/messages"; +import { AIMessageChunk, type BaseMessage } from "@langchain/core/messages"; import { ChatGenerationChunk, type ChatResult } from "@langchain/core/outputs"; import { getEnvironmentVariable } from "@langchain/core/utils/env"; import { @@ -431,7 +427,6 @@ export class ChatAnthropicMessages< ...formattedMessages, stream: true, }); - let usageData = { input_tokens: 0, output_tokens: 0 }; for await (const data of stream) { if (options.signal?.aborted) { @@ -442,13 +437,10 @@ export class ChatAnthropicMessages< const result = _makeMessageChunkFromAnthropicEvent(data, { streamUsage: shouldStreamUsage, coerceContentToString, - usageData, }); if (!result) continue; - const { chunk, usageData: updatedUsageData } = result; - - usageData = updatedUsageData; + const { chunk } = result; const newToolCallChunk = extractToolCallChunk(chunk); diff --git a/libs/langchain-anthropic/src/utils/message_outputs.ts b/libs/langchain-anthropic/src/utils/message_outputs.ts index df3a8762aa41..a04dfba44110 100644 --- a/libs/langchain-anthropic/src/utils/message_outputs.ts +++ b/libs/langchain-anthropic/src/utils/message_outputs.ts @@ -16,14 +16,10 @@ export function _makeMessageChunkFromAnthropicEvent( fields: { streamUsage: boolean; coerceContentToString: boolean; - usageData: { input_tokens: number; output_tokens: number }; } ): { chunk: AIMessageChunk; - usageData: { input_tokens: number; output_tokens: number }; } | null { - let usageDataCopy = { ...fields.usageData }; - if (data.type === "message_start") { // eslint-disable-next-line @typescript-eslint/no-unused-vars const { content, usage, ...additionalKwargs } = data.message; @@ -34,43 +30,31 @@ export function _makeMessageChunkFromAnthropicEvent( filteredAdditionalKwargs[key] = value; } } - usageDataCopy = usage; - let usageMetadata: UsageMetadata | undefined; - if (fields.streamUsage) { - usageMetadata = { - input_tokens: usage.input_tokens, - output_tokens: usage.output_tokens, - total_tokens: usage.input_tokens + usage.output_tokens, - }; - } + const usageMetadata: UsageMetadata = { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + total_tokens: usage.input_tokens + usage.output_tokens, + }; return { chunk: new AIMessageChunk({ content: fields.coerceContentToString ? "" : [], additional_kwargs: filteredAdditionalKwargs, - usage_metadata: usageMetadata, + usage_metadata: fields.streamUsage ? usageMetadata : undefined, id: data.message.id, }), - usageData: usageDataCopy, }; } else if (data.type === "message_delta") { - let usageMetadata: UsageMetadata | undefined; - if (fields.streamUsage) { - usageMetadata = { - input_tokens: data.usage.output_tokens, - output_tokens: 0, - total_tokens: data.usage.output_tokens, - }; - } - if (data?.usage !== undefined) { - usageDataCopy.output_tokens += data.usage.output_tokens; - } + const usageMetadata: UsageMetadata = { + input_tokens: 0, + output_tokens: data.usage.output_tokens, + total_tokens: data.usage.output_tokens, + }; return { chunk: new AIMessageChunk({ content: fields.coerceContentToString ? "" : [], additional_kwargs: { ...data.delta }, - usage_metadata: usageMetadata, + usage_metadata: fields.streamUsage ? usageMetadata : undefined, }), - usageData: usageDataCopy, }; } else if ( data.type === "content_block_start" && @@ -89,7 +73,6 @@ export function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } else if ( data.type === "content_block_delta" && @@ -109,7 +92,6 @@ export function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } } else if ( @@ -129,7 +111,6 @@ export function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } else if ( data.type === "content_block_start" && @@ -149,7 +130,6 @@ export function _makeMessageChunkFromAnthropicEvent( ], additional_kwargs: {}, }), - usageData: usageDataCopy, }; } }